From 1cca4270d4d2bcea16848ac2652c370cb1763a24 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:31 +0100 Subject: [PATCH 01/30] Sort imports --- conformance/src/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conformance/src/main.py b/conformance/src/main.py index 005b3bd98..8656b0669 100644 --- a/conformance/src/main.py +++ b/conformance/src/main.py @@ -3,9 +3,9 @@ """ import os -from pathlib import Path import re import sys +from pathlib import Path from time import time from typing import Sequence From e34c352eea8fcd9fc4e933254ec30f5734769bb8 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:31 +0100 Subject: [PATCH 02/30] Switch to `tomllib` from the stdlib --- conformance/pyproject.toml | 1 - conformance/src/main.py | 10 +++++----- conformance/src/reporting.py | 9 ++++----- conformance/src/test_groups.py | 6 +++--- conformance/uv.lock | 20 -------------------- 5 files changed, 12 insertions(+), 34 deletions(-) diff --git a/conformance/pyproject.toml b/conformance/pyproject.toml index e44dadf64..5d6133596 100644 --- a/conformance/pyproject.toml +++ b/conformance/pyproject.toml @@ -7,7 +7,6 @@ dependencies = [ "pycroscope", "pyrefly", "pyright", - "tomli", "tomlkit", "zuban", "ty", diff --git a/conformance/src/main.py b/conformance/src/main.py index 8656b0669..117eee2c4 100644 --- a/conformance/src/main.py +++ b/conformance/src/main.py @@ -5,11 +5,11 @@ import os import re import sys +import tomllib from pathlib import Path from time import time from typing import Sequence -import tomli import tomlkit from options import parse_options @@ -182,11 +182,11 @@ def update_output_for_test( # Read the existing results file if present. try: with open(results_file, "rb") as f: - existing_results = tomli.load(f) + existing_results = tomllib.load(f) except FileNotFoundError: should_write = True existing_results = {} - except tomli.TOMLDecodeError: + except tomllib.TOMLDecodeError: print(f"Error decoding {results_file}") existing_results = {} @@ -244,10 +244,10 @@ def update_type_checker_info(type_checker: TypeChecker, root_dir: Path): # Read the existing version file if present. try: with open(version_file, "rb") as f: - existing_info = tomli.load(f) + existing_info = tomllib.load(f) except FileNotFoundError: existing_info = {} - except tomli.TOMLDecodeError: + except tomllib.TOMLDecodeError: print(f"Error decoding {version_file}") existing_info = {} diff --git a/conformance/src/reporting.py b/conformance/src/reporting.py index cff2dc465..d6cd04ed0 100644 --- a/conformance/src/reporting.py +++ b/conformance/src/reporting.py @@ -2,10 +2,9 @@ Generates a summary of the type checker conformant tests. """ +import tomllib from pathlib import Path -import tomli - from test_groups import get_test_cases, get_test_groups from type_checker import TYPE_CHECKERS @@ -38,10 +37,10 @@ def generate_summary_html(root_dir: Path) -> str: try: with open(version_file, "rb") as f: - existing_info = tomli.load(f) + existing_info = tomllib.load(f) except FileNotFoundError: existing_info = {} - except tomli.TOMLDecodeError: + except tomllib.TOMLDecodeError: print(f"Error decoding {version_file}") existing_info = {} @@ -81,7 +80,7 @@ def generate_summary_html(root_dir: Path) -> str: / f"{test_case_name}.toml" ) with open(results_file, "rb") as f: - results = tomli.load(f) + results = tomllib.load(f) except FileNotFoundError: results = {} diff --git a/conformance/src/test_groups.py b/conformance/src/test_groups.py index b974d4016..ae8ab4bfa 100644 --- a/conformance/src/test_groups.py +++ b/conformance/src/test_groups.py @@ -3,13 +3,13 @@ conformance test suite. """ +import tomllib + from dataclasses import dataclass from itertools import chain from pathlib import Path from typing import Mapping, Sequence -import tomli - @dataclass class TestGroup: @@ -22,7 +22,7 @@ def get_test_groups(root_dir: Path) -> Mapping[str, TestGroup]: # group has a name that associated test cases must start with. test_group_file = root_dir / "src" / "test_groups.toml" with open(test_group_file, "rb") as f: - test_groups = tomli.load(f) + test_groups = tomllib.load(f) return { k: TestGroup(v.get("name", "unknown"), v.get("href", "")) diff --git a/conformance/uv.lock b/conformance/uv.lock index ac029a5b1..770b62411 100644 --- a/conformance/uv.lock +++ b/conformance/uv.lock @@ -151,24 +151,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/33/288b5868fa00846dacf249633719d747893e54aebd196b9968ac1878a5d3/pyright-1.1.410-py3-none-any.whl", hash = "sha256:5e961bed37cacf96b3f7cd7b1da39b350a9239aa2e69138d0e88f728cfaf296c", size = 6082448, upload-time = "2026-06-01T17:35:46.387Z" }, ] -[[package]] -name = "tomli" -version = "2.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/30/31573e9457673ab10aa432461bee537ce6cef177667deca369efb79df071/tomli-2.4.0.tar.gz", hash = "sha256:aa89c3f6c277dd275d8e243ad24f3b5e701491a860d5121f2cdd399fbb31fc9c", size = 17477, upload-time = "2026-01-11T11:22:38.165Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3c/43/7389a1869f2f26dba52404e1ef13b4784b6b37dac93bac53457e3ff24ca3/tomli-2.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:920b1de295e72887bafa3ad9f7a792f811847d57ea6b1215154030cf131f16b1", size = 154894, upload-time = "2026-01-11T11:21:56.07Z" }, - { url = "https://files.pythonhosted.org/packages/e9/05/2f9bf110b5294132b2edf13fe6ca6ae456204f3d749f623307cbb7a946f2/tomli-2.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7d6d9a4aee98fac3eab4952ad1d73aee87359452d1c086b5ceb43ed02ddb16b8", size = 149053, upload-time = "2026-01-11T11:21:57.467Z" }, - { url = "https://files.pythonhosted.org/packages/e8/41/1eda3ca1abc6f6154a8db4d714a4d35c4ad90adc0bcf700657291593fbf3/tomli-2.4.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:36b9d05b51e65b254ea6c2585b59d2c4cb91c8a3d91d0ed0f17591a29aaea54a", size = 243481, upload-time = "2026-01-11T11:21:58.661Z" }, - { url = "https://files.pythonhosted.org/packages/d2/6d/02ff5ab6c8868b41e7d4b987ce2b5f6a51d3335a70aa144edd999e055a01/tomli-2.4.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c8a885b370751837c029ef9bc014f27d80840e48bac415f3412e6593bbc18c1", size = 251720, upload-time = "2026-01-11T11:22:00.178Z" }, - { url = "https://files.pythonhosted.org/packages/7b/57/0405c59a909c45d5b6f146107c6d997825aa87568b042042f7a9c0afed34/tomli-2.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8768715ffc41f0008abe25d808c20c3d990f42b6e2e58305d5da280ae7d1fa3b", size = 247014, upload-time = "2026-01-11T11:22:01.238Z" }, - { url = "https://files.pythonhosted.org/packages/2c/0e/2e37568edd944b4165735687cbaf2fe3648129e440c26d02223672ee0630/tomli-2.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7b438885858efd5be02a9a133caf5812b8776ee0c969fea02c45e8e3f296ba51", size = 251820, upload-time = "2026-01-11T11:22:02.727Z" }, - { url = "https://files.pythonhosted.org/packages/5a/1c/ee3b707fdac82aeeb92d1a113f803cf6d0f37bdca0849cb489553e1f417a/tomli-2.4.0-cp312-cp312-win32.whl", hash = "sha256:0408e3de5ec77cc7f81960c362543cbbd91ef883e3138e81b729fc3eea5b9729", size = 97712, upload-time = "2026-01-11T11:22:03.777Z" }, - { url = "https://files.pythonhosted.org/packages/69/13/c07a9177d0b3bab7913299b9278845fc6eaaca14a02667c6be0b0a2270c8/tomli-2.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:685306e2cc7da35be4ee914fd34ab801a6acacb061b6a7abca922aaf9ad368da", size = 108296, upload-time = "2026-01-11T11:22:04.86Z" }, - { url = "https://files.pythonhosted.org/packages/18/27/e267a60bbeeee343bcc279bb9e8fbed0cbe224bc7b2a3dc2975f22809a09/tomli-2.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:5aa48d7c2356055feef06a43611fc401a07337d5b006be13a30f6c58f869e3c3", size = 94553, upload-time = "2026-01-11T11:22:05.854Z" }, - { url = "https://files.pythonhosted.org/packages/23/d1/136eb2cb77520a31e1f64cbae9d33ec6df0d78bdf4160398e86eec8a8754/tomli-2.4.0-py3-none-any.whl", hash = "sha256:1f776e7d669ebceb01dee46484485f43a4048746235e683bcdffacdf1fb4785a", size = 14477, upload-time = "2026-01-11T11:22:37.446Z" }, -] - [[package]] name = "tomlkit" version = "0.14.0" @@ -225,7 +207,6 @@ dependencies = [ { name = "pycroscope" }, { name = "pyrefly" }, { name = "pyright" }, - { name = "tomli" }, { name = "tomlkit" }, { name = "ty" }, { name = "zuban" }, @@ -237,7 +218,6 @@ requires-dist = [ { name = "pycroscope" }, { name = "pyrefly" }, { name = "pyright" }, - { name = "tomli" }, { name = "tomlkit" }, { name = "ty" }, { name = "zuban" }, From cb1407f8769df3f404b5ccb669c0650f3db8c875 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:31 +0100 Subject: [PATCH 03/30] Remove outdated comment in template --- conformance/results/results.html | 14 -------------- conformance/src/reporting.py | 2 +- conformance/src/results_template.html | 13 ------------- 3 files changed, 1 insertion(+), 28 deletions(-) diff --git a/conformance/results/results.html b/conformance/results/results.html index 733c4d22f..eb67476cb 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -1377,20 +1377,6 @@

Python Type System Conformance Test Results

Pass - - diff --git a/conformance/src/reporting.py b/conformance/src/reporting.py index d6cd04ed0..5d538264c 100644 --- a/conformance/src/reporting.py +++ b/conformance/src/reporting.py @@ -116,6 +116,6 @@ def generate_summary_html(root_dir: Path) -> str: summary_html.append("") - summary_html.append("\n") + summary_html.append("") return "\n".join(summary_html) diff --git a/conformance/src/results_template.html b/conformance/src/results_template.html index 214f778ed..d2de25b50 100644 --- a/conformance/src/results_template.html +++ b/conformance/src/results_template.html @@ -171,19 +171,6 @@

Python Type System Conformance Test Results

{{summary}} - From 8e947646eae660a422323fbd71bd2aca4d1de6cf Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:31 +0100 Subject: [PATCH 04/30] Move outer table tags to the template --- conformance/results/results.html | 8 ++++++-- conformance/src/reporting.py | 4 ++-- conformance/src/results_template.html | 6 +++++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/conformance/results/results.html b/conformance/results/results.html index eb67476cb..f1ce5c027 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -170,7 +170,9 @@

Python Type System Conformance Test Results

of many of the things users typically care about.

-
+
+
+ @@ -1376,7 +1378,9 @@

Python Type System Conformance Test Results

-
 
mypy 2.1.0
 Pass Pass
+ + + diff --git a/conformance/src/reporting.py b/conformance/src/reporting.py index 5d538264c..b02f6a78d 100644 --- a/conformance/src/reporting.py +++ b/conformance/src/reporting.py @@ -28,7 +28,7 @@ def generate_summary_html(root_dir: Path) -> str: test_groups = get_test_groups(root_dir) test_cases = get_test_cases(test_groups, root_dir / "tests") - summary_html = ['
'] + summary_html = [""] summary_html.append('') for type_checker in TYPE_CHECKERS: @@ -116,6 +116,6 @@ def generate_summary_html(root_dir: Path) -> str: summary_html.append("") - summary_html.append("
 
") + summary_html.append("") return "\n".join(summary_html) diff --git a/conformance/src/results_template.html b/conformance/src/results_template.html index d2de25b50..1e8306b72 100644 --- a/conformance/src/results_template.html +++ b/conformance/src/results_template.html @@ -170,7 +170,11 @@

Python Type System Conformance Test Results

of many of the things users typically care about.

- {{summary}} +
+ + {{summary}} +
+
From 2a07f12c08fd8735dbbc9f98cf5da6687848951a Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:31 +0100 Subject: [PATCH 05/30] Sort type checker columns by name --- conformance/results/results.html | 160 +++++++++++++++---------------- conformance/src/reporting.py | 8 +- 2 files changed, 85 insertions(+), 83 deletions(-) diff --git a/conformance/results/results.html b/conformance/results/results.html index f1ce5c027..97b3c3638 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -176,16 +176,16 @@

Python Type System Conformance Test Results

 
mypy 2.1.0
-
pyright 1.1.410
- -
zuban 0.8.2
+
pycroscope 0.4.0
pyrefly 1.1.0
-
pycroscope 0.4.0
+
pyright 1.1.410
ty 0.0.50
+
zuban 0.8.2
+ Type annotations @@ -200,11 +200,11 @@

Python Type System Conformance Test Results

     annotations_forward_refs
Partial

Does not report error for a forward reference that is not enclosed in quotes.

Does not report error for use of quoted type with "|" operator (runtime error).

Incorrectly generates error for quoted type defined in class scope.

-Pass -
Partial

Incorrectly generates error for quoted type defined in class scope.

-
Partial

Types in quotes incorrectly refer to shadowing class member.

Does not reject some type forms that require quotes.

Partial

Fails to reject "x" | int annotations that fail at runtime.

Rejects some valid quoted annotations.

+
Partial

Types in quotes incorrectly refer to shadowing class member.

Does not reject some type forms that require quotes.

+Pass
Partial

Resolves references in type annotations as referring to end-of-scope types (https://discuss.python.org/t/annotation-string-references-in-class-scope-in-conformance-tests/105439, https://github.com/python/typing/pull/2144)

+
Partial

Incorrectly generates error for quoted type defined in class scope.

     annotations_generators
Partial

Does not report incompatible Generator type in `yield from` statement.

@@ -216,18 +216,18 @@

Python Type System Conformance Test Results

     annotations_methods
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

-
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

Pass Pass +
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

Pass Pass      annotations_typeexpr Pass +
Partial

Fails to reject various weird annotations

Pass Pass Pass -
Partial

Fails to reject various weird annotations

Pass @@ -235,10 +235,10 @@

Python Type System Conformance Test Results

     typeforms_typeform
Partial

Does not support assigning Union and GenericAlias objects to their runtime types.

-Unsupported +
Partial

Fails to reject various weird annotations

Pass +Unsupported Pass -
Partial

Fails to reject various weird annotations

Pass @@ -254,10 +254,10 @@

Python Type System Conformance Test Results

     specialtypes_never Pass +
Partial

Does not enforce invariance in some contexts

Does not allow Any to be assigned to Never

Pass Pass Pass -
Partial

Does not enforce invariance in some contexts

Does not allow Any to be assigned to Never

Pass      specialtypes_none @@ -281,8 +281,8 @@

Python Type System Conformance Test Results

Pass Pass Pass -Pass
Partial

Allows arbitrary attributes to be accessed on `TA` where `TA = typing.Type[typing.Any]` or `TA = type[typing.Any]`.

Treats `type` equivalently to `type[object]` rather than `type[typing.Any]`.

+Pass Generics @@ -300,32 +300,32 @@

Python Type System Conformance Test Results

Pass Pass Pass -Pass
Partial

Incorrectly allows constrained type variables to be solved to a union of their constraints.

+Pass      generics_defaults
Partial

Does not detect a TypeVar with a default used after a TypeVarTuple.

Does not fully support defaults on TypeVarTuple and ParamSpec.

+
Partial

Numerous issues; does not support TypeVarTuple and ParamSpec defaults.

Pass Pass -Pass -
Partial

Numerous issues; does not support TypeVarTuple and ParamSpec defaults.

Partial

Does not forbid a `TypeVar` immediately following a `TypeVarTuple` in a parameter list from having a default.

Does not support `TypeVarTuple`.

+Pass      generics_defaults_referential
Partial

Does not correctly handle defaults referencing other TypeVars.

+
Partial

Fails to apply default specializations in some cases.

Pass Pass Pass -
Partial

Fails to apply default specializations in some cases.

Pass      generics_defaults_specialization
Partial

Does not correctly resolve defaults when classes are used directly.

-
Partial

Allows incorrect assignment to type[].

-Pass Pass Pass +
Partial

Allows incorrect assignment to type[].

Partial

Does not reject subscription of an already-specialized generic class.

+Pass      generics_paramspec_basic Pass @@ -340,14 +340,14 @@

Python Type System Conformance Test Results

Pass Pass Pass -Pass
Partial

Incorrectly allows using `*args: P.args` and `**kwargs: P.kwargs` when `P` has not been put into scope by any other parameter annotation or enclosing scope.

+Pass      generics_paramspec_semantics Pass -
Pass*

Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).

Pass Pass +
Pass*

Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).

Pass Pass @@ -362,18 +362,18 @@

Python Type System Conformance Test Results

     generics_scoping Pass Pass -Pass
Partial

Does not implement several scoping checks/restrictions for generics

Pass
Partial

Does not reject `list[T]()` in the global scope, where `T` is an unbound type variable.

Does nto reject `alias: TypeAlias = list[T]` in the body scope of a class generic over a type variable `T`.

+Pass      generics_self_advanced
Partial

Does not infer the type of an unannotated `self` parameter to be type `Self`.

Does not retain `Self` when calling method that returns `Self`.

Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`.

Does not retain `Self` when accessing attribute through `type[Self]`.

Pass -Pass
Pass*

Treats attributes not initialized on the class as instance-only

Pass Pass +Pass      generics_self_attributes Pass @@ -404,8 +404,8 @@

Python Type System Conformance Test Results

Pass Pass Pass -Pass
Partial

Does not reject `Self` used in a return annotation when `self` is annotated using another type variable.

+Pass      generics_syntax_compatibility Pass @@ -433,10 +433,10 @@

Python Type System Conformance Test Results

     generics_syntax_scoping
Partial

Does not following runtime scoping rules for type parameters in all cases.

+
Partial

Misses some details of scoping rules.

Pass Pass Pass -
Partial

Misses some details of scoping rules.

Pass      generics_type_erasure @@ -444,40 +444,40 @@

Python Type System Conformance Test Results

Pass Pass Pass -Pass
Partial

Does not reject access of generic instance variable from the class object.

+Pass      generics_typevartuple_args
Partial

Does not enforce that tuples captured by TypeVarTuple are of the same length.

-
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

-Pass Pass Pass +
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

Partial

Supports PEP-646 unpacked tuples but not TypeVarTuple.

+Pass      generics_typevartuple_basic
Partial

Does not enforce that tuples captured by TypeVarTuple are same length.

-
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

-Pass Pass Pass +
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

Unsupported +Pass      generics_typevartuple_callable Pass Pass Pass Pass -Pass Unsupported +Pass      generics_typevartuple_concat Pass Pass Pass Pass -Pass Unsupported +Pass      generics_typevartuple_overloads Pass @@ -492,16 +492,16 @@

Python Type System Conformance Test Results

Pass Pass Pass -Pass Unsupported +Pass      generics_typevartuple_unpack Pass Pass Pass Pass -Pass Unsupported +Pass      generics_upper_bound
Partial

Does not reject use of type variable within an upper bound.

@@ -532,10 +532,10 @@

Python Type System Conformance Test Results

     qualifiers_annotated
Partial

Does not allow ClassVar to be nested within Annotated.

Does not allow Final to be nested within Annotated.

Does not allow Required and NotRequired to be nested within Annotated.

Does not reject type[T] compatibility for type alias defined with Annotated.

Does not reject call of type alias defined with Annotated.

+
Partial

Fails to reject various weird annotations.

False positive on lambda in Annotated.

Pass Pass Pass -
Partial

Fails to reject various weird annotations.

False positive on lambda in Annotated.

Pass      qualifiers_final_annotation @@ -581,16 +581,16 @@

Python Type System Conformance Test Results

Pass Pass Pass -Pass
Partial

Does not emit a diagnostic if a type alias to a union, in which every element of the union is implicitly specialized with `Unknown`, is invalidly specialized again.

+Pass      aliases_implicit Pass +
Partial

Fails to handle various weird annotations.

Various bugs with resolving generic aliases.

Pass Pass -Pass -
Partial

Fails to handle various weird annotations.

Various bugs with resolving generic aliases.

Partial

Does not reject variables with `Any` or `Unknown` types when used as implicit type aliases.

Falls short on full syntactic validation of type aliases.

+Pass      aliases_newtype
Partial

`NewType`s are incorrectly considered to be classes.

@@ -605,24 +605,24 @@

Python Type System Conformance Test Results

Pass Pass Pass -Pass Unsupported +Pass      aliases_type_statement Pass +
Partial

Fails to reject various weird annotations.

Pass Pass -Pass -
Partial

Fails to reject various weird annotations.

Partial

Does not reject circular definitions of type aliases.

Does not support `type` statements generic over `TypeVarTuple`s.

+Pass      aliases_typealiastype
Partial

Incorrectly rejects some recursive type aliases using TypeAliasType.

Incorrectly rejects the use of a class-scoped TypeVar in a TypeAliasType definition.

+
Partial

Rejects valid ParamSpec specialization.

Pass Pass -Pass -
Partial

Rejects valid ParamSpec specialization.

Partial

Does not reject specializing a type parameter in a generic type alias with a type inconsistent with the parameter's upper bound.

Does not reject declaring a type alias with a type variable that is not in scope.

Does not reject declaring a type alias with a non-literal tuple passed to the `type_params` parameter.

Does not reject cyclically defined type aliases.

+Pass      aliases_variance Pass @@ -640,8 +640,8 @@

Python Type System Conformance Test Results

Pass Pass Pass -Pass
Pass*

Deliberately does not allow `str` to be narrowed to literal string types through equality or containment checks due to the possibility of `str` subclasses that could have unexpected equality semantics.

+Pass      literals_literalstring
Unsupported

Support for `LiteralString` is not implemented.

@@ -653,10 +653,10 @@

Python Type System Conformance Test Results

     literals_parameterizations
Partial

Does not reject tuple within Literal.

+
Partial

Fails to reject various invalid literal parameterizations.

Pass Pass Pass -
Partial

Fails to reject various invalid literal parameterizations.

Pass      literals_semantics @@ -672,67 +672,67 @@

Python Type System Conformance Test Results

     protocols_class_objects Pass +
Partial

Abstract type[Proto] still allows protocol class objects in some paths.

Pass Pass -Pass -
Partial

Abstract type[Proto] still allows protocol class objects in some paths.

Unsupported

`type[Proto]` is not yet supported.

`ClassVar` protocol members are not yet supported.

`@property` protocol members are only partially supported.

A class object `C` is only considered to inhabit a protocol type with a method member `f` if `f` exists as an attribute on the metaclass of `C`.

+Pass      protocols_definition
Partial

Does not detect protocol mismatch if concrete method is missing annotations.

Does not detect protocol mismatch if concrete method's parameters are position-only.

Pass Pass Pass -Pass
Partial

Does not reject implicit instance attributes in `Protocol` methods.

Does not support `ClassVar` protocol members.

Incorrectly considers `ClassVar` attributes on concrete classes as satisfying non-`ClassVar` attribute members on protocols.

Only has partial support for `@property` protocol members.

+Pass      protocols_explicit
Pass*

Does not report unimplemented attributes for class that explicitly derives from protocol until it is instantiated.

Pass Pass Pass -Pass
Unsupported

Allows implicitly abstract protocol methods to be called via `super()` on a protocol subclass.

Allows instantiation of abstract subclasses of protocol classes.

+Pass      protocols_generic Pass +
Partial

Fails to reject duplicate generic/protocol bases.

Treats global object as a literal.

Pass Pass -Pass -
Partial

Fails to reject duplicate generic/protocol bases.

Treats global object as a literal.

Partial

Only partially supports `@property` protocol members.

+Pass      protocols_merging Pass Pass Pass Pass -Pass
Partial

Does not reject attempted instantiation of abstract subclasses of protocols.

+Pass      protocols_modules Pass Pass Pass Pass -Pass
Partial

Never considers a module as satisfying a protocol with a method member due to the fact that the method will never exist on the class `types.ModuleType`, only on a given specific module instance.

+Pass      protocols_recursive Pass Pass Pass Pass -Pass
Partial

Fails to solve a type variable involving a recursive generic protocol.

+Pass      protocols_runtime_checkable
Partial

Does not report unsafe overlap for runtime_checkable protocol.

Pass Pass Pass -Pass
Partial

Does not reject `isinstance()` or `issubclass()` calls against runtime-checkable protocols where there is an unsafe overlap between the type of the first argument and the protocol.

+Pass      protocols_self Pass @@ -755,8 +755,8 @@

Python Type System Conformance Test Results

Pass Pass Pass -Pass Unsupported +Pass Callables @@ -764,10 +764,10 @@

Python Type System Conformance Test Results

     callables_annotation
Partial

Incorrectly treats "*args: T, **kwargs: T" as "..." when T is specialized to Any.

Does not treat "*args: Any, **kargs: Any" as "..." when separated by keyword parameter.

Pass -Pass
Partial

Parameter names are lost when resolving ParamSpec

Pass
Partial

Infers a callback protocol as being a gradual type if the callback has signature `__call__[T](self, *args: T, **kwargs: T)` and `T` has been explicitly specialized to `Any`.

+Pass      callables_kwargs
Partial

Allows callable without kwargs to be assigned to callable with unpacked kwargs

@@ -801,8 +801,8 @@

Python Type System Conformance Test Results

Pass Pass Pass -Pass
Partial

Does not reject invalid argument types to an inherited constructor in a specialized subclass of a generic superclass.

Does not reject class-scoped type variables used in the `self` annotation.

+Pass      constructors_call_metaclass
Unsupported

Does not honor metaclass __call__ method when evaluating constructor call.

Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

@@ -825,16 +825,16 @@

Python Type System Conformance Test Results

Pass Pass Pass -Pass
Partial

Has overly lenient handling of calls to `type[T]` if `T` is a type variable without an upper bound.

+Pass      constructors_callable
Partial

Does not generate a union type for __new__ and __init__ when converting class to callable.

Does not ignore __init__ based on __new__ return type when converting class to callable.

Does not support __new__ return type that is different from class being constructed.

Pass -Pass
Partial

Converting constructor to callable does not preserve class-scoped type params.

Pass
Partial

Does not include `__init__` when `__new__` returns `Self`.

Does not ignore `__init__` when `__new__` returns `Any`.

Unions overload return types.

+Pass      constructors_consistency
Pass*

Does not report inconsistency between __new__ and __init__ (optional).

@@ -849,10 +849,10 @@

Python Type System Conformance Test Results

     overloads_basic Pass +
Partial

Does less literal promotion than the test asks for.

Pass Pass Pass -
Partial

Does less literal promotion than the test asks for.

Pass      overloads_consistency @@ -873,17 +873,17 @@

Python Type System Conformance Test Results

     overloads_definitions_stub
Partial

Allows @override to appear in a stub file not on the first overload.

+
Unsupported

Does not support checking stubs.

Pass Pass Pass -
Unsupported

Does not support checking stubs.

Pass      overloads_evaluation
Partial

Does not expand boolean arguments to Literal[True] and Literal[False].

Does not expand enum arguments to literal variants.

Does not expand tuple arguments to possible combinations.

Does not evaluate Any in some cases where overload is ambiguous.

Evaluates Any in some cases where overload is not ambiguous.

-
Partial

Does not evaluate Any in some cases where overload is ambiguous.

Pass Pass +
Partial

Does not evaluate Any in some cases where overload is ambiguous.

Pass Pass @@ -893,21 +893,21 @@

Python Type System Conformance Test Results

     exceptions_context_managers Pass Pass -Pass
Partial

Some error suppressing context managers are not detected

Pass Unsupported +Pass Dataclasses      dataclasses_descriptors
Partial

Assumes descriptor behavior only when field is assigned in class body.

Does not correctly evaluate type of descriptor access.

-Pass -Pass -
Partial

* Assumes descriptor behavior only when field is assigned in class body

* Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types

Partial

Conformance suite is questionable; see https://github.com/python/typing/issues/2259

+
Partial

* Assumes descriptor behavior only when field is assigned in class body

* Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types

+Pass
Partial

Only infers a descriptor `__get__` method as being called when a descriptor attribute is accessed on an instance if the descriptor attribute is present in the class namespace.

+Pass      dataclasses_final
Partial

Wrongly requires a Final dataclass field to be initialized at class level.

Doesn't support Final nested inside ClassVar.

@@ -930,8 +930,8 @@

Python Type System Conformance Test Results

Pass Pass Pass -Pass
Partial

Understands the `Hashable` protocol as equivalent to `object`.

+Pass      dataclasses_inheritance Pass @@ -978,8 +978,8 @@

Python Type System Conformance Test Results

Pass Pass Pass -Pass
Partial

Synthesizes a `__slots__` attribute but does not validate attribute assignments against `__slots__`.

+Pass      dataclasses_transform_class Pass @@ -1042,9 +1042,9 @@

Python Type System Conformance Test Results

     typeddicts_class_syntax
Partial

Does not support version-conditional items in TypedDict definitions.

-
Partial

Does not support version-conditional items in TypedDict definitions.

Pass Pass +
Partial

Does not support version-conditional items in TypedDict definitions.

Pass Pass @@ -1130,10 +1130,10 @@

Python Type System Conformance Test Results

     typeddicts_type_consistency Pass +
Partial

Considers TypedDicts to be assignable to plain dict types.

Pass Pass Pass -
Partial

Considers TypedDicts to be assignable to plain dict types.

Pass      typeddicts_usage @@ -1176,10 +1176,10 @@

Python Type System Conformance Test Results

     namedtuples_define_class
Partial

Does not reject override of named tuple attribute in child class.

Does not support version-conditional fields.

+
Partial

Does not support precise type inference for slices over namedtuples.

Pass Pass Pass -
Partial

Does not support precise type inference for slices over namedtuples.

Pass      namedtuples_define_functional @@ -1219,10 +1219,10 @@

Python Type System Conformance Test Results

     enums_definition Pass +
Partial

Does not allow enum members to be conditional on version/platform checks.

Pass Pass Pass -
Partial

Does not allow enum members to be conditional on version/platform checks.

Pass      enums_expansion @@ -1230,8 +1230,8 @@

Python Type System Conformance Test Results

Pass Pass Pass -Pass
Partial

Does not support `enum.Flag`.

+Pass      enums_member_names
Pass*

Does not support special-cased handling of member name literal types in some cases (optional).

@@ -1251,9 +1251,9 @@

Python Type System Conformance Test Results

     enums_members
Partial

Does not treat attribute with annotation and no assignment as non-member.

Does not treat callables as non-members.

Does not honor `enum.member` as method decorator.

Does not properly handle aliased enum members.

Does not support `_ignore_` mechanism (optional).

Does not treat attributes with private names as non-members.

-
Pass*

Does not support `_ignore_` mechanism (optional).

Pass Pass +
Pass*

Does not support `_ignore_` mechanism (optional).

Pass Pass @@ -1273,8 +1273,8 @@

Python Type System Conformance Test Results

Pass Pass Pass -Pass
Partial

Intersects the pre-existing type with the top materialization of the bracketed type rather than the bracketed type itself.

+Pass Type checker directives @@ -1300,22 +1300,22 @@

Python Type System Conformance Test Results

Pass Pass Pass -Pass
Partial

Does not detect calls to deprecated overloads.

Does not detect implicit calls to deprecated dunder methods, for example via operators.

Does not detect accesses of, or attempts to set, deprecated properties.

+Pass      directives_disjoint_base Pass -
Unsupported

Does not support PEP 800 disjoint-base semantics.

+
Partial

Does not reject invalid class definitions due to disjoint bases, but uses disjoint base information in type narrowing.

Pass +
Unsupported

Does not support PEP 800 disjoint-base semantics.

Pass -
Partial

Does not reject invalid class definitions due to disjoint bases, but uses disjoint base information in type narrowing.

Pass      directives_no_type_check
Partial

Does not honor `@no_type_check` class decorator (allowed).

Does not reject invalid call of `@no_type_check` function.

-
Pass*

Does not honor `@no_type_check` class decorator (allowed).

Pass Pass +
Pass*

Does not honor `@no_type_check` class decorator (allowed).

Pass Pass diff --git a/conformance/src/reporting.py b/conformance/src/reporting.py index b02f6a78d..76b5332b6 100644 --- a/conformance/src/reporting.py +++ b/conformance/src/reporting.py @@ -2,6 +2,7 @@ Generates a summary of the type checker conformant tests. """ +import operator import tomllib from pathlib import Path @@ -24,14 +25,15 @@ def generate_summary(root_dir: Path): def generate_summary_html(root_dir: Path) -> str: - column_count = len(TYPE_CHECKERS) + 1 + type_checkers = sorted(TYPE_CHECKERS, key=operator.attrgetter("name")) + column_count = len(type_checkers) + 1 test_groups = get_test_groups(root_dir) test_cases = get_test_cases(test_groups, root_dir / "tests") summary_html = [""] summary_html.append(' ') - for type_checker in TYPE_CHECKERS: + for type_checker in type_checkers: # Load the version file for the type checker. version_file = root_dir / "results" / type_checker.name / "version.toml" @@ -71,7 +73,7 @@ def generate_summary_html(root_dir: Path) -> str: summary_html.append(f'     {test_case_name}') - for type_checker in TYPE_CHECKERS: + for type_checker in type_checkers: try: results_file = ( root_dir From 903ff1afa91ae9bc3fef7fa5190842e6fab34f98 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:31 +0100 Subject: [PATCH 06/30] Fix validation errors in HTML - `` cannot be a parent of `

` - `#bottom` must be unique --- conformance/results/results.html | 280 +++++++++++++------------- conformance/src/reporting.py | 2 +- conformance/src/results_template.html | 4 +- 3 files changed, 143 insertions(+), 143 deletions(-) diff --git a/conformance/results/results.html b/conformance/results/results.html index 97b3c3638..acbda3631 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -135,12 +135,12 @@ margin: 4px; } - #bottom { + .hover-text .tooltip-text { top: 25px; left: -80%; } - #bottom::before { + .hover-text .tooltip-text::before { top: -5%; left: 94%; } @@ -199,15 +199,15 @@

Python Type System Conformance Test Results

Pass      annotations_forward_refs -
Partial

Does not report error for a forward reference that is not enclosed in quotes.

Does not report error for use of quoted type with "|" operator (runtime error).

Incorrectly generates error for quoted type defined in class scope.

-
Partial

Fails to reject "x" | int annotations that fail at runtime.

Rejects some valid quoted annotations.

-
Partial

Types in quotes incorrectly refer to shadowing class member.

Does not reject some type forms that require quotes.

+
Partial

Does not report error for a forward reference that is not enclosed in quotes.

Does not report error for use of quoted type with "|" operator (runtime error).

Incorrectly generates error for quoted type defined in class scope.

+
Partial

Fails to reject "x" | int annotations that fail at runtime.

Rejects some valid quoted annotations.

+
Partial

Types in quotes incorrectly refer to shadowing class member.

Does not reject some type forms that require quotes.

Pass -
Partial

Resolves references in type annotations as referring to end-of-scope types (https://discuss.python.org/t/annotation-string-references-in-class-scope-in-conformance-tests/105439, https://github.com/python/typing/pull/2144)

-
Partial

Incorrectly generates error for quoted type defined in class scope.

+
Partial

Resolves references in type annotations as referring to end-of-scope types (https://discuss.python.org/t/annotation-string-references-in-class-scope-in-conformance-tests/105439, https://github.com/python/typing/pull/2144)

+
Partial

Incorrectly generates error for quoted type defined in class scope.

     annotations_generators -
Partial

Does not report incompatible Generator type in `yield from` statement.

+
Partial

Does not report incompatible Generator type in `yield from` statement.

Pass Pass Pass @@ -215,16 +215,16 @@

Python Type System Conformance Test Results

Pass      annotations_methods -
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

+
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

Pass Pass -
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

+
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

Pass Pass      annotations_typeexpr Pass -
Partial

Fails to reject various weird annotations

+
Partial

Fails to reject various weird annotations

Pass Pass Pass @@ -234,8 +234,8 @@

Python Type System Conformance Test Results

Type forms      typeforms_typeform -
Partial

Does not support assigning Union and GenericAlias objects to their runtime types.

-
Partial

Fails to reject various weird annotations

+
Partial

Does not support assigning Union and GenericAlias objects to their runtime types.

+
Partial

Fails to reject various weird annotations

Pass Unsupported Pass @@ -254,7 +254,7 @@

Python Type System Conformance Test Results

     specialtypes_never Pass -
Partial

Does not enforce invariance in some contexts

Does not allow Any to be assigned to Never

+
Partial

Does not enforce invariance in some contexts

Does not allow Any to be assigned to Never

Pass Pass Pass @@ -277,18 +277,18 @@

Python Type System Conformance Test Results

Pass      specialtypes_type -
Partial

Does not treat `type` same as `type[Any]` for assert_type.

Does not allow access to unknown attributes from object of type `type[Any]`.

+
Partial

Does not treat `type` same as `type[Any]` for assert_type.

Does not allow access to unknown attributes from object of type `type[Any]`.

Pass Pass Pass -
Partial

Allows arbitrary attributes to be accessed on `TA` where `TA = typing.Type[typing.Any]` or `TA = type[typing.Any]`.

Treats `type` equivalently to `type[object]` rather than `type[typing.Any]`.

+
Partial

Allows arbitrary attributes to be accessed on `TA` where `TA = typing.Type[typing.Any]` or `TA = type[typing.Any]`.

Treats `type` equivalently to `type[object]` rather than `type[typing.Any]`.

Pass Generics      generics_base_class -
Partial

Does not detect inconsistent type variable ordering.

+
Partial

Does not detect inconsistent type variable ordering.

Pass Pass Pass @@ -300,31 +300,31 @@

Python Type System Conformance Test Results

Pass Pass Pass -
Partial

Incorrectly allows constrained type variables to be solved to a union of their constraints.

+
Partial

Incorrectly allows constrained type variables to be solved to a union of their constraints.

Pass      generics_defaults -
Partial

Does not detect a TypeVar with a default used after a TypeVarTuple.

Does not fully support defaults on TypeVarTuple and ParamSpec.

-
Partial

Numerous issues; does not support TypeVarTuple and ParamSpec defaults.

+
Partial

Does not detect a TypeVar with a default used after a TypeVarTuple.

Does not fully support defaults on TypeVarTuple and ParamSpec.

+
Partial

Numerous issues; does not support TypeVarTuple and ParamSpec defaults.

Pass Pass -
Partial

Does not forbid a `TypeVar` immediately following a `TypeVarTuple` in a parameter list from having a default.

Does not support `TypeVarTuple`.

+
Partial

Does not forbid a `TypeVar` immediately following a `TypeVarTuple` in a parameter list from having a default.

Does not support `TypeVarTuple`.

Pass      generics_defaults_referential -
Partial

Does not correctly handle defaults referencing other TypeVars.

-
Partial

Fails to apply default specializations in some cases.

+
Partial

Does not correctly handle defaults referencing other TypeVars.

+
Partial

Fails to apply default specializations in some cases.

Pass Pass Pass Pass      generics_defaults_specialization -
Partial

Does not correctly resolve defaults when classes are used directly.

+
Partial

Does not correctly resolve defaults when classes are used directly.

Pass Pass -
Partial

Allows incorrect assignment to type[].

-
Partial

Does not reject subscription of an already-specialized generic class.

+
Partial

Allows incorrect assignment to type[].

+
Partial

Does not reject subscription of an already-specialized generic class.

Pass      generics_paramspec_basic @@ -340,14 +340,14 @@

Python Type System Conformance Test Results

Pass Pass Pass -
Partial

Incorrectly allows using `*args: P.args` and `**kwargs: P.kwargs` when `P` has not been put into scope by any other parameter annotation or enclosing scope.

+
Partial

Incorrectly allows using `*args: P.args` and `**kwargs: P.kwargs` when `P` has not been put into scope by any other parameter annotation or enclosing scope.

Pass      generics_paramspec_semantics Pass Pass Pass -
Pass*

Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).

+
Pass*

Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).

Pass Pass @@ -362,15 +362,15 @@

Python Type System Conformance Test Results

     generics_scoping Pass Pass -
Partial

Does not implement several scoping checks/restrictions for generics

+
Partial

Does not implement several scoping checks/restrictions for generics

Pass -
Partial

Does not reject `list[T]()` in the global scope, where `T` is an unbound type variable.

Does nto reject `alias: TypeAlias = list[T]` in the body scope of a class generic over a type variable `T`.

+
Partial

Does not reject `list[T]()` in the global scope, where `T` is an unbound type variable.

Does nto reject `alias: TypeAlias = list[T]` in the body scope of a class generic over a type variable `T`.

Pass      generics_self_advanced -
Partial

Does not infer the type of an unannotated `self` parameter to be type `Self`.

Does not retain `Self` when calling method that returns `Self`.

Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`.

Does not retain `Self` when accessing attribute through `type[Self]`.

+
Partial

Does not infer the type of an unannotated `self` parameter to be type `Self`.

Does not retain `Self` when calling method that returns `Self`.

Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`.

Does not retain `Self` when accessing attribute through `type[Self]`.

Pass -
Pass*

Treats attributes not initialized on the class as instance-only

+
Pass*

Treats attributes not initialized on the class as instance-only

Pass Pass Pass @@ -404,7 +404,7 @@

Python Type System Conformance Test Results

Pass Pass Pass -
Partial

Does not reject `Self` used in a return annotation when `self` is annotated using another type variable.

+
Partial

Does not reject `Self` used in a return annotation when `self` is annotated using another type variable.

Pass      generics_syntax_compatibility @@ -424,7 +424,7 @@

Python Type System Conformance Test Results

Pass      generics_syntax_infer_variance -
Unsupported

Type parameter syntax not yet supported.

+
Unsupported

Type parameter syntax not yet supported.

Pass Pass Pass @@ -432,34 +432,34 @@

Python Type System Conformance Test Results

Pass      generics_syntax_scoping -
Partial

Does not following runtime scoping rules for type parameters in all cases.

-
Partial

Misses some details of scoping rules.

+
Partial

Does not following runtime scoping rules for type parameters in all cases.

+
Partial

Misses some details of scoping rules.

Pass Pass Pass Pass      generics_type_erasure -
Partial

Infers Node[Never] instead of Node[Any] when argument is not provided.

False negative on instance attribute access on type(node).

+
Partial

Infers Node[Never] instead of Node[Any] when argument is not provided.

False negative on instance attribute access on type(node).

Pass Pass Pass -
Partial

Does not reject access of generic instance variable from the class object.

+
Partial

Does not reject access of generic instance variable from the class object.

Pass      generics_typevartuple_args -
Partial

Does not enforce that tuples captured by TypeVarTuple are of the same length.

+
Partial

Does not enforce that tuples captured by TypeVarTuple are of the same length.

Pass Pass -
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

-
Partial

Supports PEP-646 unpacked tuples but not TypeVarTuple.

+
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

+
Partial

Supports PEP-646 unpacked tuples but not TypeVarTuple.

Pass      generics_typevartuple_basic -
Partial

Does not enforce that tuples captured by TypeVarTuple are same length.

+
Partial

Does not enforce that tuples captured by TypeVarTuple are same length.

Pass Pass -
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

+
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

Unsupported Pass @@ -488,7 +488,7 @@

Python Type System Conformance Test Results

Pass      generics_typevartuple_specialization -
Partial

Incorrectly specializes generic alias that includes a TypeVar and TypeVarTuple if no type arguments are provided.

Rejects specialization of generic type alias defined as a tuple containing a TypeVar.

+
Partial

Incorrectly specializes generic alias that includes a TypeVar and TypeVarTuple if no type arguments are provided.

Rejects specialization of generic type alias defined as a tuple containing a TypeVar.

Pass Pass Pass @@ -504,7 +504,7 @@

Python Type System Conformance Test Results

Pass      generics_upper_bound -
Partial

Does not reject use of type variable within an upper bound.

+
Partial

Does not reject use of type variable within an upper bound.

Pass Pass Pass @@ -512,7 +512,7 @@

Python Type System Conformance Test Results

Pass      generics_variance -
Partial

Does not reject use of class-scoped TypeVar used in a base class when variance is incompatible.

+
Partial

Does not reject use of class-scoped TypeVar used in a base class when variance is incompatible.

Pass Pass Pass @@ -531,15 +531,15 @@

Python Type System Conformance Test Results

Type qualifiers      qualifiers_annotated -
Partial

Does not allow ClassVar to be nested within Annotated.

Does not allow Final to be nested within Annotated.

Does not allow Required and NotRequired to be nested within Annotated.

Does not reject type[T] compatibility for type alias defined with Annotated.

Does not reject call of type alias defined with Annotated.

-
Partial

Fails to reject various weird annotations.

False positive on lambda in Annotated.

+
Partial

Does not allow ClassVar to be nested within Annotated.

Does not allow Final to be nested within Annotated.

Does not allow Required and NotRequired to be nested within Annotated.

Does not reject type[T] compatibility for type alias defined with Annotated.

Does not reject call of type alias defined with Annotated.

+
Partial

Fails to reject various weird annotations.

False positive on lambda in Annotated.

Pass Pass Pass Pass      qualifiers_final_annotation -
Partial

Does not treat use of Final name as if it was replaced by the literal in NamedTuple definition.

Does not allow conditional assignment of Final instance variable in __init__ method.

Does not allow redefinition of private class variable that is marked Final in parent class.

Does not report modification of local Final variable via "for" statement.

+
Partial

Does not treat use of Final name as if it was replaced by the literal in NamedTuple definition.

Does not allow conditional assignment of Final instance variable in __init__ method.

Does not allow redefinition of private class variable that is marked Final in parent class.

Does not report modification of local Final variable via "for" statement.

Pass Pass Pass @@ -558,7 +558,7 @@

Python Type System Conformance Test Results

Class type compatibility      classes_classvar -
Partial

Internal error if TypeVarTuple is used in ClassVar.

Does not reject use of ParamSpec in ClassVar.

Rejects ClassVar nested in Annotated.

Does not reject use of ClassVar in TypeAlias definition.

+
Partial

Internal error if TypeVarTuple is used in ClassVar.

Does not reject use of ParamSpec in ClassVar.

Rejects ClassVar nested in Annotated.

Does not reject use of ClassVar in TypeAlias definition.

Pass Pass Pass @@ -577,23 +577,23 @@

Python Type System Conformance Test Results

Type aliases      aliases_explicit -
Partial

Does not reject specialization of type alias that has already been implicitly specialized.

+
Partial

Does not reject specialization of type alias that has already been implicitly specialized.

Pass Pass Pass -
Partial

Does not emit a diagnostic if a type alias to a union, in which every element of the union is implicitly specialized with `Unknown`, is invalidly specialized again.

+
Partial

Does not emit a diagnostic if a type alias to a union, in which every element of the union is implicitly specialized with `Unknown`, is invalidly specialized again.

Pass      aliases_implicit Pass -
Partial

Fails to handle various weird annotations.

Various bugs with resolving generic aliases.

+
Partial

Fails to handle various weird annotations.

Various bugs with resolving generic aliases.

Pass Pass -
Partial

Does not reject variables with `Any` or `Unknown` types when used as implicit type aliases.

Falls short on full syntactic validation of type aliases.

+
Partial

Does not reject variables with `Any` or `Unknown` types when used as implicit type aliases.

Falls short on full syntactic validation of type aliases.

Pass      aliases_newtype -
Partial

`NewType`s are incorrectly considered to be classes.

+
Partial

`NewType`s are incorrectly considered to be classes.

Pass Pass Pass @@ -610,18 +610,18 @@

Python Type System Conformance Test Results

     aliases_type_statement Pass -
Partial

Fails to reject various weird annotations.

+
Partial

Fails to reject various weird annotations.

Pass Pass -
Partial

Does not reject circular definitions of type aliases.

Does not support `type` statements generic over `TypeVarTuple`s.

+
Partial

Does not reject circular definitions of type aliases.

Does not support `type` statements generic over `TypeVarTuple`s.

Pass      aliases_typealiastype -
Partial

Incorrectly rejects some recursive type aliases using TypeAliasType.

Incorrectly rejects the use of a class-scoped TypeVar in a TypeAliasType definition.

-
Partial

Rejects valid ParamSpec specialization.

+
Partial

Incorrectly rejects some recursive type aliases using TypeAliasType.

Incorrectly rejects the use of a class-scoped TypeVar in a TypeAliasType definition.

+
Partial

Rejects valid ParamSpec specialization.

Pass Pass -
Partial

Does not reject specializing a type parameter in a generic type alias with a type inconsistent with the parameter's upper bound.

Does not reject declaring a type alias with a type variable that is not in scope.

Does not reject declaring a type alias with a non-literal tuple passed to the `type_params` parameter.

Does not reject cyclically defined type aliases.

+
Partial

Does not reject specializing a type parameter in a generic type alias with a type inconsistent with the parameter's upper bound.

Does not reject declaring a type alias with a type variable that is not in scope.

Does not reject declaring a type alias with a non-literal tuple passed to the `type_params` parameter.

Does not reject cyclically defined type aliases.

Pass      aliases_variance @@ -636,15 +636,15 @@

Python Type System Conformance Test Results

Literals      literals_interactions -
Pass*

Does not narrow `str` or `LiteralString` types to `Literal` string types via equality or containment checks.

+
Pass*

Does not narrow `str` or `LiteralString` types to `Literal` string types via equality or containment checks.

Pass Pass Pass -
Pass*

Deliberately does not allow `str` to be narrowed to literal string types through equality or containment checks due to the possibility of `str` subclasses that could have unexpected equality semantics.

+
Pass*

Deliberately does not allow `str` to be narrowed to literal string types through equality or containment checks due to the possibility of `str` subclasses that could have unexpected equality semantics.

Pass      literals_literalstring -
Unsupported

Support for `LiteralString` is not implemented.

+
Unsupported

Support for `LiteralString` is not implemented.

Pass Pass Pass @@ -652,8 +652,8 @@

Python Type System Conformance Test Results

Pass      literals_parameterizations -
Partial

Does not reject tuple within Literal.

-
Partial

Fails to reject various invalid literal parameterizations.

+
Partial

Does not reject tuple within Literal.

+
Partial

Fails to reject various invalid literal parameterizations.

Pass Pass Pass @@ -672,34 +672,34 @@

Python Type System Conformance Test Results

     protocols_class_objects Pass -
Partial

Abstract type[Proto] still allows protocol class objects in some paths.

+
Partial

Abstract type[Proto] still allows protocol class objects in some paths.

Pass Pass -
Unsupported

`type[Proto]` is not yet supported.

`ClassVar` protocol members are not yet supported.

`@property` protocol members are only partially supported.

A class object `C` is only considered to inhabit a protocol type with a method member `f` if `f` exists as an attribute on the metaclass of `C`.

+
Unsupported

`type[Proto]` is not yet supported.

`ClassVar` protocol members are not yet supported.

`@property` protocol members are only partially supported.

A class object `C` is only considered to inhabit a protocol type with a method member `f` if `f` exists as an attribute on the metaclass of `C`.

Pass      protocols_definition -
Partial

Does not detect protocol mismatch if concrete method is missing annotations.

Does not detect protocol mismatch if concrete method's parameters are position-only.

+
Partial

Does not detect protocol mismatch if concrete method is missing annotations.

Does not detect protocol mismatch if concrete method's parameters are position-only.

Pass Pass Pass -
Partial

Does not reject implicit instance attributes in `Protocol` methods.

Does not support `ClassVar` protocol members.

Incorrectly considers `ClassVar` attributes on concrete classes as satisfying non-`ClassVar` attribute members on protocols.

Only has partial support for `@property` protocol members.

+
Partial

Does not reject implicit instance attributes in `Protocol` methods.

Does not support `ClassVar` protocol members.

Incorrectly considers `ClassVar` attributes on concrete classes as satisfying non-`ClassVar` attribute members on protocols.

Only has partial support for `@property` protocol members.

Pass      protocols_explicit -
Pass*

Does not report unimplemented attributes for class that explicitly derives from protocol until it is instantiated.

+
Pass*

Does not report unimplemented attributes for class that explicitly derives from protocol until it is instantiated.

Pass Pass Pass -
Unsupported

Allows implicitly abstract protocol methods to be called via `super()` on a protocol subclass.

Allows instantiation of abstract subclasses of protocol classes.

+
Unsupported

Allows implicitly abstract protocol methods to be called via `super()` on a protocol subclass.

Allows instantiation of abstract subclasses of protocol classes.

Pass      protocols_generic Pass -
Partial

Fails to reject duplicate generic/protocol bases.

Treats global object as a literal.

+
Partial

Fails to reject duplicate generic/protocol bases.

Treats global object as a literal.

Pass Pass -
Partial

Only partially supports `@property` protocol members.

+
Partial

Only partially supports `@property` protocol members.

Pass      protocols_merging @@ -707,7 +707,7 @@

Python Type System Conformance Test Results

Pass Pass Pass -
Partial

Does not reject attempted instantiation of abstract subclasses of protocols.

+
Partial

Does not reject attempted instantiation of abstract subclasses of protocols.

Pass      protocols_modules @@ -715,7 +715,7 @@

Python Type System Conformance Test Results

Pass Pass Pass -
Partial

Never considers a module as satisfying a protocol with a method member due to the fact that the method will never exist on the class `types.ModuleType`, only on a given specific module instance.

+
Partial

Never considers a module as satisfying a protocol with a method member due to the fact that the method will never exist on the class `types.ModuleType`, only on a given specific module instance.

Pass      protocols_recursive @@ -723,15 +723,15 @@

Python Type System Conformance Test Results

Pass Pass Pass -
Partial

Fails to solve a type variable involving a recursive generic protocol.

+
Partial

Fails to solve a type variable involving a recursive generic protocol.

Pass      protocols_runtime_checkable -
Partial

Does not report unsafe overlap for runtime_checkable protocol.

+
Partial

Does not report unsafe overlap for runtime_checkable protocol.

Pass Pass Pass -
Partial

Does not reject `isinstance()` or `issubclass()` calls against runtime-checkable protocols where there is an unsafe overlap between the type of the first argument and the protocol.

+
Partial

Does not reject `isinstance()` or `issubclass()` calls against runtime-checkable protocols where there is an unsafe overlap between the type of the first argument and the protocol.

Pass      protocols_self @@ -762,15 +762,15 @@

Python Type System Conformance Test Results

Callables      callables_annotation -
Partial

Incorrectly treats "*args: T, **kwargs: T" as "..." when T is specialized to Any.

Does not treat "*args: Any, **kargs: Any" as "..." when separated by keyword parameter.

+
Partial

Incorrectly treats "*args: T, **kwargs: T" as "..." when T is specialized to Any.

Does not treat "*args: Any, **kargs: Any" as "..." when separated by keyword parameter.

Pass -
Partial

Parameter names are lost when resolving ParamSpec

+
Partial

Parameter names are lost when resolving ParamSpec

Pass -
Partial

Infers a callback protocol as being a gradual type if the callback has signature `__call__[T](self, *args: T, **kwargs: T)` and `T` has been explicitly specialized to `Any`.

+
Partial

Infers a callback protocol as being a gradual type if the callback has signature `__call__[T](self, *args: T, **kwargs: T)` and `T` has been explicitly specialized to `Any`.

Pass      callables_kwargs -
Partial

Allows callable without kwargs to be assigned to callable with unpacked kwargs

+
Partial

Allows callable without kwargs to be assigned to callable with unpacked kwargs

Pass Pass Pass @@ -797,15 +797,15 @@

Python Type System Conformance Test Results

Constructors      constructors_call_init -
Partial

Does not report errors during binding to self parameter of __init__ method.

Does not reject use of class-scoped type variables in annotation of self parameter in __init__ method.

+
Partial

Does not report errors during binding to self parameter of __init__ method.

Does not reject use of class-scoped type variables in annotation of self parameter in __init__ method.

Pass Pass Pass -
Partial

Does not reject invalid argument types to an inherited constructor in a specialized subclass of a generic superclass.

Does not reject class-scoped type variables used in the `self` annotation.

+
Partial

Does not reject invalid argument types to an inherited constructor in a specialized subclass of a generic superclass.

Does not reject class-scoped type variables used in the `self` annotation.

Pass      constructors_call_metaclass -
Unsupported

Does not honor metaclass __call__ method when evaluating constructor call.

Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

+
Unsupported

Does not honor metaclass __call__ method when evaluating constructor call.

Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

Pass Pass Pass @@ -813,7 +813,7 @@

Python Type System Conformance Test Results

Pass      constructors_call_new -
Partial

Does not support __new__ return type that is not a subclass of the class being constructed.

Does not skip evaluation of __init__ based on __new__ return type.

Does not report errors during binding to cls parameter of __new__ method.

+
Partial

Does not support __new__ return type that is not a subclass of the class being constructed.

Does not skip evaluation of __init__ based on __new__ return type.

Does not report errors during binding to cls parameter of __new__ method.

Pass Pass Pass @@ -821,23 +821,23 @@

Python Type System Conformance Test Results

Pass      constructors_call_type -
Partial

Does not validate call to custom metaclass __call__ method through type[T].

+
Partial

Does not validate call to custom metaclass __call__ method through type[T].

Pass Pass Pass -
Partial

Has overly lenient handling of calls to `type[T]` if `T` is a type variable without an upper bound.

+
Partial

Has overly lenient handling of calls to `type[T]` if `T` is a type variable without an upper bound.

Pass      constructors_callable -
Partial

Does not generate a union type for __new__ and __init__ when converting class to callable.

Does not ignore __init__ based on __new__ return type when converting class to callable.

Does not support __new__ return type that is different from class being constructed.

+
Partial

Does not generate a union type for __new__ and __init__ when converting class to callable.

Does not ignore __init__ based on __new__ return type when converting class to callable.

Does not support __new__ return type that is different from class being constructed.

Pass -
Partial

Converting constructor to callable does not preserve class-scoped type params.

+
Partial

Converting constructor to callable does not preserve class-scoped type params.

Pass -
Partial

Does not include `__init__` when `__new__` returns `Self`.

Does not ignore `__init__` when `__new__` returns `Any`.

Unions overload return types.

+
Partial

Does not include `__init__` when `__new__` returns `Self`.

Does not ignore `__init__` when `__new__` returns `Any`.

Unions overload return types.

Pass      constructors_consistency -
Pass*

Does not report inconsistency between __new__ and __init__ (optional).

+
Pass*

Does not report inconsistency between __new__ and __init__ (optional).

Pass Pass Pass @@ -849,7 +849,7 @@

Python Type System Conformance Test Results

     overloads_basic Pass -
Partial

Does less literal promotion than the test asks for.

+
Partial

Does less literal promotion than the test asks for.

Pass Pass Pass @@ -864,7 +864,7 @@

Python Type System Conformance Test Results

Pass      overloads_definitions -
Partial

Allows @override to be on all overloads and implementation, instead of just implementation.

+
Partial

Allows @override to be on all overloads and implementation, instead of just implementation.

Pass Pass Pass @@ -872,18 +872,18 @@

Python Type System Conformance Test Results

Pass      overloads_definitions_stub -
Partial

Allows @override to appear in a stub file not on the first overload.

-
Unsupported

Does not support checking stubs.

+
Partial

Allows @override to appear in a stub file not on the first overload.

+
Unsupported

Does not support checking stubs.

Pass Pass Pass Pass      overloads_evaluation -
Partial

Does not expand boolean arguments to Literal[True] and Literal[False].

Does not expand enum arguments to literal variants.

Does not expand tuple arguments to possible combinations.

Does not evaluate Any in some cases where overload is ambiguous.

Evaluates Any in some cases where overload is not ambiguous.

+
Partial

Does not expand boolean arguments to Literal[True] and Literal[False].

Does not expand enum arguments to literal variants.

Does not expand tuple arguments to possible combinations.

Does not evaluate Any in some cases where overload is ambiguous.

Evaluates Any in some cases where overload is not ambiguous.

Pass Pass -
Partial

Does not evaluate Any in some cases where overload is ambiguous.

+
Partial

Does not evaluate Any in some cases where overload is ambiguous.

Pass Pass @@ -893,7 +893,7 @@

Python Type System Conformance Test Results

     exceptions_context_managers Pass Pass -
Partial

Some error suppressing context managers are not detected

+
Partial

Some error suppressing context managers are not detected

Pass Unsupported Pass @@ -902,15 +902,15 @@

Python Type System Conformance Test Results

Dataclasses      dataclasses_descriptors -
Partial

Assumes descriptor behavior only when field is assigned in class body.

Does not correctly evaluate type of descriptor access.

-
Partial

Conformance suite is questionable; see https://github.com/python/typing/issues/2259

-
Partial

* Assumes descriptor behavior only when field is assigned in class body

* Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types

+
Partial

Assumes descriptor behavior only when field is assigned in class body.

Does not correctly evaluate type of descriptor access.

+
Partial

Conformance suite is questionable; see https://github.com/python/typing/issues/2259

+
Partial

* Assumes descriptor behavior only when field is assigned in class body

* Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types

Pass -
Partial

Only infers a descriptor `__get__` method as being called when a descriptor attribute is accessed on an instance if the descriptor attribute is present in the class namespace.

+
Partial

Only infers a descriptor `__get__` method as being called when a descriptor attribute is accessed on an instance if the descriptor attribute is present in the class namespace.

Pass      dataclasses_final -
Partial

Wrongly requires a Final dataclass field to be initialized at class level.

Doesn't support Final nested inside ClassVar.

+
Partial

Wrongly requires a Final dataclass field to be initialized at class level.

Doesn't support Final nested inside ClassVar.

Pass Pass Pass @@ -926,11 +926,11 @@

Python Type System Conformance Test Results

Pass      dataclasses_hash -
Unsupported

Does not synthesize `__hash__ = None` as a class attribute for unhashable dataclasses.

Does not report when an unhashable dataclass has `__hash__` called directly on an instance.

Does not report when dataclass is not compatible with Hashable protocol.

+
Unsupported

Does not synthesize `__hash__ = None` as a class attribute for unhashable dataclasses.

Does not report when an unhashable dataclass has `__hash__` called directly on an instance.

Does not report when dataclass is not compatible with Hashable protocol.

Pass Pass Pass -
Partial

Understands the `Hashable` protocol as equivalent to `object`.

+
Partial

Understands the `Hashable` protocol as equivalent to `object`.

Pass      dataclasses_inheritance @@ -974,11 +974,11 @@

Python Type System Conformance Test Results

Pass      dataclasses_slots -
Partial

Does not reject write to instance variable that is not defined in __slots__.

+
Partial

Does not reject write to instance variable that is not defined in __slots__.

Pass Pass Pass -
Partial

Synthesizes a `__slots__` attribute but does not validate attribute assignments against `__slots__`.

+
Partial

Synthesizes a `__slots__` attribute but does not validate attribute assignments against `__slots__`.

Pass      dataclasses_transform_class @@ -990,7 +990,7 @@

Python Type System Conformance Test Results

Pass      dataclasses_transform_converter -
Unsupported

Converter parameter not yet supported.

+
Unsupported

Converter parameter not yet supported.

Pass Pass Pass @@ -998,7 +998,7 @@

Python Type System Conformance Test Results

Pass      dataclasses_transform_field -
Partial

Does not properly handle field constructor that has default value for `kw_only` or `init` parameter.

+
Partial

Does not properly handle field constructor that has default value for `kw_only` or `init` parameter.

Pass Pass Pass @@ -1006,7 +1006,7 @@

Python Type System Conformance Test Results

Pass      dataclasses_transform_func -
Partial

Does not handle `kw_only=False` override when `kw_only_default=True`.

Does not report error when `order=False` and comparison operators are used.

+
Partial

Does not handle `kw_only=False` override when `kw_only_default=True`.

Does not report error when `order=False` and comparison operators are used.

Pass Pass Pass @@ -1022,7 +1022,7 @@

Python Type System Conformance Test Results

Pass      dataclasses_usage -
Pass*

Does not detect unannotated usage of `dataclasses.field()`.

+
Pass*

Does not detect unannotated usage of `dataclasses.field()`.

Pass Pass Pass @@ -1033,7 +1033,7 @@

Python Type System Conformance Test Results

Typed dictionaries      typeddicts_alt_syntax -
Pass*

Does not support keyword-argument form of alternative syntax (deprecated in 3.11).

+
Pass*

Does not support keyword-argument form of alternative syntax (deprecated in 3.11).

Pass Pass Pass @@ -1041,15 +1041,15 @@

Python Type System Conformance Test Results

Pass      typeddicts_class_syntax -
Partial

Does not support version-conditional items in TypedDict definitions.

+
Partial

Does not support version-conditional items in TypedDict definitions.

Pass Pass -
Partial

Does not support version-conditional items in TypedDict definitions.

+
Partial

Does not support version-conditional items in TypedDict definitions.

Pass Pass      typeddicts_extra_items -
Unsupported

Not supported.

+
Unsupported

Not supported.

Pass Pass Pass @@ -1097,7 +1097,7 @@

Python Type System Conformance Test Results

Pass      typeddicts_readonly_inheritance -
Partial

Incorrectly rejects non-ReadOnly override of ReadOnly item.

Incorrectly rejects override of ReadOnly item with another ReadOnly item with narrower type.

Incorrectly rejects override of NotRequired ReadOnly item with a Required ReadOnly item.

+
Partial

Incorrectly rejects non-ReadOnly override of ReadOnly item.

Incorrectly rejects override of ReadOnly item with another ReadOnly item with narrower type.

Incorrectly rejects override of NotRequired ReadOnly item with a Required ReadOnly item.

Pass Pass Pass @@ -1113,7 +1113,7 @@

Python Type System Conformance Test Results

Pass      typeddicts_readonly_update -
Partial

Incorrectly allows update of ReadOnly item.

Incorrectly rejects update involving an item with Never type.

+
Partial

Incorrectly allows update of ReadOnly item.

Incorrectly rejects update involving an item with Never type.

Pass Pass Pass @@ -1130,7 +1130,7 @@

Python Type System Conformance Test Results

     typeddicts_type_consistency Pass -
Partial

Considers TypedDicts to be assignable to plain dict types.

+
Partial

Considers TypedDicts to be assignable to plain dict types.

Pass Pass Pass @@ -1148,7 +1148,7 @@

Python Type System Conformance Test Results

Tuples      tuples_type_compat -
Partial

Does not support tuple narrowing based on `len()` type guard (optional).

Incorrectly marks a match case as unreachable.

+
Partial

Does not support tuple narrowing based on `len()` type guard (optional).

Incorrectly marks a match case as unreachable.

Pass Pass Pass @@ -1164,7 +1164,7 @@

Python Type System Conformance Test Results

Pass      tuples_unpacked -
Partial

"More than one unpack" error is missing in some cases.

+
Partial

"More than one unpack" error is missing in some cases.

Pass Pass Pass @@ -1175,8 +1175,8 @@

Python Type System Conformance Test Results

Named tuples      namedtuples_define_class -
Partial

Does not reject override of named tuple attribute in child class.

Does not support version-conditional fields.

-
Partial

Does not support precise type inference for slices over namedtuples.

+
Partial

Does not reject override of named tuple attribute in child class.

Does not support version-conditional fields.

+
Partial

Does not support precise type inference for slices over namedtuples.

Pass Pass Pass @@ -1199,7 +1199,7 @@

Python Type System Conformance Test Results

Pass      namedtuples_usage -
Partial

Does not reject attempt to delete named tuple field by name.

+
Partial

Does not reject attempt to delete named tuple field by name.

Pass Pass Pass @@ -1219,22 +1219,22 @@

Python Type System Conformance Test Results

     enums_definition Pass -
Partial

Does not allow enum members to be conditional on version/platform checks.

+
Partial

Does not allow enum members to be conditional on version/platform checks.

Pass Pass Pass Pass      enums_expansion -
Partial

Improperly applies narrowing to Flag subclass.

+
Partial

Improperly applies narrowing to Flag subclass.

Pass Pass Pass -
Partial

Does not support `enum.Flag`.

+
Partial

Does not support `enum.Flag`.

Pass      enums_member_names -
Pass*

Does not support special-cased handling of member name literal types in some cases (optional).

+
Pass*

Does not support special-cased handling of member name literal types in some cases (optional).

Pass Pass Pass @@ -1242,7 +1242,7 @@

Python Type System Conformance Test Results

Pass      enums_member_values -
Partial

Does not enforce declared type of `_value_`.

Does not enforce assigned tuple types for enum members (optional).

+
Partial

Does not enforce declared type of `_value_`.

Does not enforce assigned tuple types for enum members (optional).

Pass Pass Pass @@ -1250,10 +1250,10 @@

Python Type System Conformance Test Results

Pass      enums_members -
Partial

Does not treat attribute with annotation and no assignment as non-member.

Does not treat callables as non-members.

Does not honor `enum.member` as method decorator.

Does not properly handle aliased enum members.

Does not support `_ignore_` mechanism (optional).

Does not treat attributes with private names as non-members.

+
Partial

Does not treat attribute with annotation and no assignment as non-member.

Does not treat callables as non-members.

Does not honor `enum.member` as method decorator.

Does not properly handle aliased enum members.

Does not support `_ignore_` mechanism (optional).

Does not treat attributes with private names as non-members.

Pass Pass -
Pass*

Does not support `_ignore_` mechanism (optional).

+
Pass*

Does not support `_ignore_` mechanism (optional).

Pass Pass @@ -1273,7 +1273,7 @@

Python Type System Conformance Test Results

Pass Pass Pass -
Partial

Intersects the pre-existing type with the top materialization of the bracketed type rather than the bracketed type itself.

+
Partial

Intersects the pre-existing type with the top materialization of the bracketed type rather than the bracketed type itself.

Pass @@ -1300,22 +1300,22 @@

Python Type System Conformance Test Results

Pass Pass Pass -
Partial

Does not detect calls to deprecated overloads.

Does not detect implicit calls to deprecated dunder methods, for example via operators.

Does not detect accesses of, or attempts to set, deprecated properties.

+
Partial

Does not detect calls to deprecated overloads.

Does not detect implicit calls to deprecated dunder methods, for example via operators.

Does not detect accesses of, or attempts to set, deprecated properties.

Pass      directives_disjoint_base Pass -
Partial

Does not reject invalid class definitions due to disjoint bases, but uses disjoint base information in type narrowing.

+
Partial

Does not reject invalid class definitions due to disjoint bases, but uses disjoint base information in type narrowing.

Pass -
Unsupported

Does not support PEP 800 disjoint-base semantics.

+
Unsupported

Does not support PEP 800 disjoint-base semantics.

Pass Pass      directives_no_type_check -
Partial

Does not honor `@no_type_check` class decorator (allowed).

Does not reject invalid call of `@no_type_check` function.

+
Partial

Does not honor `@no_type_check` class decorator (allowed).

Does not reject invalid call of `@no_type_check` function.

Pass Pass -
Pass*

Does not honor `@no_type_check` class decorator (allowed).

+
Pass*

Does not honor `@no_type_check` class decorator (allowed).

Pass Pass @@ -1336,7 +1336,7 @@

Python Type System Conformance Test Results

Pass      directives_type_ignore -
Partial

Does not honor "# type: ignore" comment if comment includes additional text.

+
Partial

Does not honor "# type: ignore" comment if comment includes additional text.

Pass Pass Pass @@ -1360,7 +1360,7 @@

Python Type System Conformance Test Results

Pass      directives_version_platform -
Pass*

Does not understand three-element form of sys.version checks.

Does not understand os.name checks.

+
Pass*

Does not understand three-element form of sys.version checks.

Does not understand os.name checks.

Pass Pass Pass @@ -1371,7 +1371,7 @@

Python Type System Conformance Test Results

Historical and deprecated features      historical_positional -
Partial

Does not reject positional-only parameter after non-positional-only parameter.

Treats keyword-only parameter as positional-only.

Applies legacy positional-only rules when PEP 570 syntax is used.

+
Partial

Does not reject positional-only parameter after non-positional-only parameter.

Treats keyword-only parameter as positional-only.

Applies legacy positional-only rules when PEP 570 syntax is used.

Pass Pass Pass diff --git a/conformance/src/reporting.py b/conformance/src/reporting.py index 76b5332b6..ae30dcd55 100644 --- a/conformance/src/reporting.py +++ b/conformance/src/reporting.py @@ -112,7 +112,7 @@ def generate_summary_html(root_dir: Path) -> str: conformance_cell = f"{conformance}" if raw_notes != "": - conformance_cell = f'
{conformance_cell}{notes}
' + conformance_cell = f'
{conformance_cell}
{notes}
' summary_html.append(f'{conformance_cell}') diff --git a/conformance/src/results_template.html b/conformance/src/results_template.html index 1e8306b72..1da1e3b1c 100644 --- a/conformance/src/results_template.html +++ b/conformance/src/results_template.html @@ -135,12 +135,12 @@ margin: 4px; } - #bottom { + .hover-text .tooltip-text { top: 25px; left: -80%; } - #bottom::before { + .hover-text .tooltip-text::before { top: -5%; left: 94%; } From 74457f87743015ccb611ef3c3b3a8de76673b90f Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:31 +0100 Subject: [PATCH 07/30] Use `` instead of `` for non-header cells --- conformance/results/results.html | 1692 +++++++++++++++--------------- conformance/src/reporting.py | 2 +- 2 files changed, 847 insertions(+), 847 deletions(-) diff --git a/conformance/results/results.html b/conformance/results/results.html index acbda3631..4c4c67a72 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -191,1192 +191,1192 @@

Python Type System Conformance Test Results

Type annotations      annotations_coroutines -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      annotations_forward_refs -
Partial

Does not report error for a forward reference that is not enclosed in quotes.

Does not report error for use of quoted type with "|" operator (runtime error).

Incorrectly generates error for quoted type defined in class scope.

-
Partial

Fails to reject "x" | int annotations that fail at runtime.

Rejects some valid quoted annotations.

-
Partial

Types in quotes incorrectly refer to shadowing class member.

Does not reject some type forms that require quotes.

-Pass -
Partial

Resolves references in type annotations as referring to end-of-scope types (https://discuss.python.org/t/annotation-string-references-in-class-scope-in-conformance-tests/105439, https://github.com/python/typing/pull/2144)

-
Partial

Incorrectly generates error for quoted type defined in class scope.

+
Partial

Does not report error for a forward reference that is not enclosed in quotes.

Does not report error for use of quoted type with "|" operator (runtime error).

Incorrectly generates error for quoted type defined in class scope.

+
Partial

Fails to reject "x" | int annotations that fail at runtime.

Rejects some valid quoted annotations.

+
Partial

Types in quotes incorrectly refer to shadowing class member.

Does not reject some type forms that require quotes.

+Pass +
Partial

Resolves references in type annotations as referring to end-of-scope types (https://discuss.python.org/t/annotation-string-references-in-class-scope-in-conformance-tests/105439, https://github.com/python/typing/pull/2144)

+
Partial

Incorrectly generates error for quoted type defined in class scope.

     annotations_generators -
Partial

Does not report incompatible Generator type in `yield from` statement.

-Pass -Pass -Pass -Pass -Pass +
Partial

Does not report incompatible Generator type in `yield from` statement.

+Pass +Pass +Pass +Pass +Pass      annotations_methods -
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

-Pass -Pass -
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

-Pass -Pass +
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

+Pass +Pass +
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

+Pass +Pass      annotations_typeexpr -Pass -
Partial

Fails to reject various weird annotations

-Pass -Pass -Pass -Pass +Pass +
Partial

Fails to reject various weird annotations

+Pass +Pass +Pass +Pass Type forms      typeforms_typeform -
Partial

Does not support assigning Union and GenericAlias objects to their runtime types.

-
Partial

Fails to reject various weird annotations

-Pass -Unsupported -Pass -Pass +
Partial

Does not support assigning Union and GenericAlias objects to their runtime types.

+
Partial

Fails to reject various weird annotations

+Pass +Unsupported +Pass +Pass Special types in annotations      specialtypes_any -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      specialtypes_never -Pass -
Partial

Does not enforce invariance in some contexts

Does not allow Any to be assigned to Never

-Pass -Pass -Pass -Pass +Pass +
Partial

Does not enforce invariance in some contexts

Does not allow Any to be assigned to Never

+Pass +Pass +Pass +Pass      specialtypes_none -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      specialtypes_promotions -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      specialtypes_type -
Partial

Does not treat `type` same as `type[Any]` for assert_type.

Does not allow access to unknown attributes from object of type `type[Any]`.

-Pass -Pass -Pass -
Partial

Allows arbitrary attributes to be accessed on `TA` where `TA = typing.Type[typing.Any]` or `TA = type[typing.Any]`.

Treats `type` equivalently to `type[object]` rather than `type[typing.Any]`.

-Pass +
Partial

Does not treat `type` same as `type[Any]` for assert_type.

Does not allow access to unknown attributes from object of type `type[Any]`.

+Pass +Pass +Pass +
Partial

Allows arbitrary attributes to be accessed on `TA` where `TA = typing.Type[typing.Any]` or `TA = type[typing.Any]`.

Treats `type` equivalently to `type[object]` rather than `type[typing.Any]`.

+Pass Generics      generics_base_class -
Partial

Does not detect inconsistent type variable ordering.

-Pass -Pass -Pass -Pass -Pass +
Partial

Does not detect inconsistent type variable ordering.

+Pass +Pass +Pass +Pass +Pass      generics_basic -Pass -Pass -Pass -Pass -
Partial

Incorrectly allows constrained type variables to be solved to a union of their constraints.

-Pass +Pass +Pass +Pass +Pass +
Partial

Incorrectly allows constrained type variables to be solved to a union of their constraints.

+Pass      generics_defaults -
Partial

Does not detect a TypeVar with a default used after a TypeVarTuple.

Does not fully support defaults on TypeVarTuple and ParamSpec.

-
Partial

Numerous issues; does not support TypeVarTuple and ParamSpec defaults.

-Pass -Pass -
Partial

Does not forbid a `TypeVar` immediately following a `TypeVarTuple` in a parameter list from having a default.

Does not support `TypeVarTuple`.

-Pass +
Partial

Does not detect a TypeVar with a default used after a TypeVarTuple.

Does not fully support defaults on TypeVarTuple and ParamSpec.

+
Partial

Numerous issues; does not support TypeVarTuple and ParamSpec defaults.

+Pass +Pass +
Partial

Does not forbid a `TypeVar` immediately following a `TypeVarTuple` in a parameter list from having a default.

Does not support `TypeVarTuple`.

+Pass      generics_defaults_referential -
Partial

Does not correctly handle defaults referencing other TypeVars.

-
Partial

Fails to apply default specializations in some cases.

-Pass -Pass -Pass -Pass +
Partial

Does not correctly handle defaults referencing other TypeVars.

+
Partial

Fails to apply default specializations in some cases.

+Pass +Pass +Pass +Pass      generics_defaults_specialization -
Partial

Does not correctly resolve defaults when classes are used directly.

-Pass -Pass -
Partial

Allows incorrect assignment to type[].

-
Partial

Does not reject subscription of an already-specialized generic class.

-Pass +
Partial

Does not correctly resolve defaults when classes are used directly.

+Pass +Pass +
Partial

Allows incorrect assignment to type[].

+
Partial

Does not reject subscription of an already-specialized generic class.

+Pass      generics_paramspec_basic -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      generics_paramspec_components -Pass -Pass -Pass -Pass -
Partial

Incorrectly allows using `*args: P.args` and `**kwargs: P.kwargs` when `P` has not been put into scope by any other parameter annotation or enclosing scope.

-Pass +Pass +Pass +Pass +Pass +
Partial

Incorrectly allows using `*args: P.args` and `**kwargs: P.kwargs` when `P` has not been put into scope by any other parameter annotation or enclosing scope.

+Pass      generics_paramspec_semantics -Pass -Pass -Pass -
Pass*

Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).

-Pass -Pass +Pass +Pass +Pass +
Pass*

Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).

+Pass +Pass      generics_paramspec_specialization -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      generics_scoping -Pass -Pass -
Partial

Does not implement several scoping checks/restrictions for generics

-Pass -
Partial

Does not reject `list[T]()` in the global scope, where `T` is an unbound type variable.

Does nto reject `alias: TypeAlias = list[T]` in the body scope of a class generic over a type variable `T`.

-Pass +Pass +Pass +
Partial

Does not implement several scoping checks/restrictions for generics

+Pass +
Partial

Does not reject `list[T]()` in the global scope, where `T` is an unbound type variable.

Does nto reject `alias: TypeAlias = list[T]` in the body scope of a class generic over a type variable `T`.

+Pass      generics_self_advanced -
Partial

Does not infer the type of an unannotated `self` parameter to be type `Self`.

Does not retain `Self` when calling method that returns `Self`.

Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`.

Does not retain `Self` when accessing attribute through `type[Self]`.

-Pass -
Pass*

Treats attributes not initialized on the class as instance-only

-Pass -Pass -Pass +
Partial

Does not infer the type of an unannotated `self` parameter to be type `Self`.

Does not retain `Self` when calling method that returns `Self`.

Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`.

Does not retain `Self` when accessing attribute through `type[Self]`.

+Pass +
Pass*

Treats attributes not initialized on the class as instance-only

+Pass +Pass +Pass      generics_self_attributes -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      generics_self_basic -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      generics_self_protocols -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      generics_self_usage -Pass -Pass -Pass -Pass -
Partial

Does not reject `Self` used in a return annotation when `self` is annotated using another type variable.

-Pass +Pass +Pass +Pass +Pass +
Partial

Does not reject `Self` used in a return annotation when `self` is annotated using another type variable.

+Pass      generics_syntax_compatibility -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      generics_syntax_declarations -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      generics_syntax_infer_variance -
Unsupported

Type parameter syntax not yet supported.

-Pass -Pass -Pass -Pass -Pass +
Unsupported

Type parameter syntax not yet supported.

+Pass +Pass +Pass +Pass +Pass      generics_syntax_scoping -
Partial

Does not following runtime scoping rules for type parameters in all cases.

-
Partial

Misses some details of scoping rules.

-Pass -Pass -Pass -Pass +
Partial

Does not following runtime scoping rules for type parameters in all cases.

+
Partial

Misses some details of scoping rules.

+Pass +Pass +Pass +Pass      generics_type_erasure -
Partial

Infers Node[Never] instead of Node[Any] when argument is not provided.

False negative on instance attribute access on type(node).

-Pass -Pass -Pass -
Partial

Does not reject access of generic instance variable from the class object.

-Pass +
Partial

Infers Node[Never] instead of Node[Any] when argument is not provided.

False negative on instance attribute access on type(node).

+Pass +Pass +Pass +
Partial

Does not reject access of generic instance variable from the class object.

+Pass      generics_typevartuple_args -
Partial

Does not enforce that tuples captured by TypeVarTuple are of the same length.

-Pass -Pass -
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

-
Partial

Supports PEP-646 unpacked tuples but not TypeVarTuple.

-Pass +
Partial

Does not enforce that tuples captured by TypeVarTuple are of the same length.

+Pass +Pass +
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

+
Partial

Supports PEP-646 unpacked tuples but not TypeVarTuple.

+Pass      generics_typevartuple_basic -
Partial

Does not enforce that tuples captured by TypeVarTuple are same length.

-Pass -Pass -
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

-Unsupported -Pass +
Partial

Does not enforce that tuples captured by TypeVarTuple are same length.

+Pass +Pass +
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

+Unsupported +Pass      generics_typevartuple_callable -Pass -Pass -Pass -Pass -Unsupported -Pass +Pass +Pass +Pass +Pass +Unsupported +Pass      generics_typevartuple_concat -Pass -Pass -Pass -Pass -Unsupported -Pass +Pass +Pass +Pass +Pass +Unsupported +Pass      generics_typevartuple_overloads -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      generics_typevartuple_specialization -
Partial

Incorrectly specializes generic alias that includes a TypeVar and TypeVarTuple if no type arguments are provided.

Rejects specialization of generic type alias defined as a tuple containing a TypeVar.

-Pass -Pass -Pass -Unsupported -Pass +
Partial

Incorrectly specializes generic alias that includes a TypeVar and TypeVarTuple if no type arguments are provided.

Rejects specialization of generic type alias defined as a tuple containing a TypeVar.

+Pass +Pass +Pass +Unsupported +Pass      generics_typevartuple_unpack -Pass -Pass -Pass -Pass -Unsupported -Pass +Pass +Pass +Pass +Pass +Unsupported +Pass      generics_upper_bound -
Partial

Does not reject use of type variable within an upper bound.

-Pass -Pass -Pass -Pass -Pass +
Partial

Does not reject use of type variable within an upper bound.

+Pass +Pass +Pass +Pass +Pass      generics_variance -
Partial

Does not reject use of class-scoped TypeVar used in a base class when variance is incompatible.

-Pass -Pass -Pass -Pass -Pass +
Partial

Does not reject use of class-scoped TypeVar used in a base class when variance is incompatible.

+Pass +Pass +Pass +Pass +Pass      generics_variance_inference -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass Type qualifiers      qualifiers_annotated -
Partial

Does not allow ClassVar to be nested within Annotated.

Does not allow Final to be nested within Annotated.

Does not allow Required and NotRequired to be nested within Annotated.

Does not reject type[T] compatibility for type alias defined with Annotated.

Does not reject call of type alias defined with Annotated.

-
Partial

Fails to reject various weird annotations.

False positive on lambda in Annotated.

-Pass -Pass -Pass -Pass +
Partial

Does not allow ClassVar to be nested within Annotated.

Does not allow Final to be nested within Annotated.

Does not allow Required and NotRequired to be nested within Annotated.

Does not reject type[T] compatibility for type alias defined with Annotated.

Does not reject call of type alias defined with Annotated.

+
Partial

Fails to reject various weird annotations.

False positive on lambda in Annotated.

+Pass +Pass +Pass +Pass      qualifiers_final_annotation -
Partial

Does not treat use of Final name as if it was replaced by the literal in NamedTuple definition.

Does not allow conditional assignment of Final instance variable in __init__ method.

Does not allow redefinition of private class variable that is marked Final in parent class.

Does not report modification of local Final variable via "for" statement.

-Pass -Pass -Pass -Pass -Pass +
Partial

Does not treat use of Final name as if it was replaced by the literal in NamedTuple definition.

Does not allow conditional assignment of Final instance variable in __init__ method.

Does not allow redefinition of private class variable that is marked Final in parent class.

Does not report modification of local Final variable via "for" statement.

+Pass +Pass +Pass +Pass +Pass      qualifiers_final_decorator -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass Class type compatibility      classes_classvar -
Partial

Internal error if TypeVarTuple is used in ClassVar.

Does not reject use of ParamSpec in ClassVar.

Rejects ClassVar nested in Annotated.

Does not reject use of ClassVar in TypeAlias definition.

-Pass -Pass -Pass -Pass -Pass +
Partial

Internal error if TypeVarTuple is used in ClassVar.

Does not reject use of ParamSpec in ClassVar.

Rejects ClassVar nested in Annotated.

Does not reject use of ClassVar in TypeAlias definition.

+Pass +Pass +Pass +Pass +Pass      classes_override -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass Type aliases      aliases_explicit -
Partial

Does not reject specialization of type alias that has already been implicitly specialized.

-Pass -Pass -Pass -
Partial

Does not emit a diagnostic if a type alias to a union, in which every element of the union is implicitly specialized with `Unknown`, is invalidly specialized again.

-Pass +
Partial

Does not reject specialization of type alias that has already been implicitly specialized.

+Pass +Pass +Pass +
Partial

Does not emit a diagnostic if a type alias to a union, in which every element of the union is implicitly specialized with `Unknown`, is invalidly specialized again.

+Pass      aliases_implicit -Pass -
Partial

Fails to handle various weird annotations.

Various bugs with resolving generic aliases.

-Pass -Pass -
Partial

Does not reject variables with `Any` or `Unknown` types when used as implicit type aliases.

Falls short on full syntactic validation of type aliases.

-Pass +Pass +
Partial

Fails to handle various weird annotations.

Various bugs with resolving generic aliases.

+Pass +Pass +
Partial

Does not reject variables with `Any` or `Unknown` types when used as implicit type aliases.

Falls short on full syntactic validation of type aliases.

+Pass      aliases_newtype -
Partial

`NewType`s are incorrectly considered to be classes.

-Pass -Pass -Pass -Pass -Pass +
Partial

`NewType`s are incorrectly considered to be classes.

+Pass +Pass +Pass +Pass +Pass      aliases_recursive -Pass -Pass -Pass -Pass -Unsupported -Pass +Pass +Pass +Pass +Pass +Unsupported +Pass      aliases_type_statement -Pass -
Partial

Fails to reject various weird annotations.

-Pass -Pass -
Partial

Does not reject circular definitions of type aliases.

Does not support `type` statements generic over `TypeVarTuple`s.

-Pass +Pass +
Partial

Fails to reject various weird annotations.

+Pass +Pass +
Partial

Does not reject circular definitions of type aliases.

Does not support `type` statements generic over `TypeVarTuple`s.

+Pass      aliases_typealiastype -
Partial

Incorrectly rejects some recursive type aliases using TypeAliasType.

Incorrectly rejects the use of a class-scoped TypeVar in a TypeAliasType definition.

-
Partial

Rejects valid ParamSpec specialization.

-Pass -Pass -
Partial

Does not reject specializing a type parameter in a generic type alias with a type inconsistent with the parameter's upper bound.

Does not reject declaring a type alias with a type variable that is not in scope.

Does not reject declaring a type alias with a non-literal tuple passed to the `type_params` parameter.

Does not reject cyclically defined type aliases.

-Pass +
Partial

Incorrectly rejects some recursive type aliases using TypeAliasType.

Incorrectly rejects the use of a class-scoped TypeVar in a TypeAliasType definition.

+
Partial

Rejects valid ParamSpec specialization.

+Pass +Pass +
Partial

Does not reject specializing a type parameter in a generic type alias with a type inconsistent with the parameter's upper bound.

Does not reject declaring a type alias with a type variable that is not in scope.

Does not reject declaring a type alias with a non-literal tuple passed to the `type_params` parameter.

Does not reject cyclically defined type aliases.

+Pass      aliases_variance -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass Literals      literals_interactions -
Pass*

Does not narrow `str` or `LiteralString` types to `Literal` string types via equality or containment checks.

-Pass -Pass -Pass -
Pass*

Deliberately does not allow `str` to be narrowed to literal string types through equality or containment checks due to the possibility of `str` subclasses that could have unexpected equality semantics.

-Pass +
Pass*

Does not narrow `str` or `LiteralString` types to `Literal` string types via equality or containment checks.

+Pass +Pass +Pass +
Pass*

Deliberately does not allow `str` to be narrowed to literal string types through equality or containment checks due to the possibility of `str` subclasses that could have unexpected equality semantics.

+Pass      literals_literalstring -
Unsupported

Support for `LiteralString` is not implemented.

-Pass -Pass -Pass -Pass -Pass +
Unsupported

Support for `LiteralString` is not implemented.

+Pass +Pass +Pass +Pass +Pass      literals_parameterizations -
Partial

Does not reject tuple within Literal.

-
Partial

Fails to reject various invalid literal parameterizations.

-Pass -Pass -Pass -Pass +
Partial

Does not reject tuple within Literal.

+
Partial

Fails to reject various invalid literal parameterizations.

+Pass +Pass +Pass +Pass      literals_semantics -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass Protocols      protocols_class_objects -Pass -
Partial

Abstract type[Proto] still allows protocol class objects in some paths.

-Pass -Pass -
Unsupported

`type[Proto]` is not yet supported.

`ClassVar` protocol members are not yet supported.

`@property` protocol members are only partially supported.

A class object `C` is only considered to inhabit a protocol type with a method member `f` if `f` exists as an attribute on the metaclass of `C`.

-Pass +Pass +
Partial

Abstract type[Proto] still allows protocol class objects in some paths.

+Pass +Pass +
Unsupported

`type[Proto]` is not yet supported.

`ClassVar` protocol members are not yet supported.

`@property` protocol members are only partially supported.

A class object `C` is only considered to inhabit a protocol type with a method member `f` if `f` exists as an attribute on the metaclass of `C`.

+Pass      protocols_definition -
Partial

Does not detect protocol mismatch if concrete method is missing annotations.

Does not detect protocol mismatch if concrete method's parameters are position-only.

-Pass -Pass -Pass -
Partial

Does not reject implicit instance attributes in `Protocol` methods.

Does not support `ClassVar` protocol members.

Incorrectly considers `ClassVar` attributes on concrete classes as satisfying non-`ClassVar` attribute members on protocols.

Only has partial support for `@property` protocol members.

-Pass +
Partial

Does not detect protocol mismatch if concrete method is missing annotations.

Does not detect protocol mismatch if concrete method's parameters are position-only.

+Pass +Pass +Pass +
Partial

Does not reject implicit instance attributes in `Protocol` methods.

Does not support `ClassVar` protocol members.

Incorrectly considers `ClassVar` attributes on concrete classes as satisfying non-`ClassVar` attribute members on protocols.

Only has partial support for `@property` protocol members.

+Pass      protocols_explicit -
Pass*

Does not report unimplemented attributes for class that explicitly derives from protocol until it is instantiated.

-Pass -Pass -Pass -
Unsupported

Allows implicitly abstract protocol methods to be called via `super()` on a protocol subclass.

Allows instantiation of abstract subclasses of protocol classes.

-Pass +
Pass*

Does not report unimplemented attributes for class that explicitly derives from protocol until it is instantiated.

+Pass +Pass +Pass +
Unsupported

Allows implicitly abstract protocol methods to be called via `super()` on a protocol subclass.

Allows instantiation of abstract subclasses of protocol classes.

+Pass      protocols_generic -Pass -
Partial

Fails to reject duplicate generic/protocol bases.

Treats global object as a literal.

-Pass -Pass -
Partial

Only partially supports `@property` protocol members.

-Pass +Pass +
Partial

Fails to reject duplicate generic/protocol bases.

Treats global object as a literal.

+Pass +Pass +
Partial

Only partially supports `@property` protocol members.

+Pass      protocols_merging -Pass -Pass -Pass -Pass -
Partial

Does not reject attempted instantiation of abstract subclasses of protocols.

-Pass +Pass +Pass +Pass +Pass +
Partial

Does not reject attempted instantiation of abstract subclasses of protocols.

+Pass      protocols_modules -Pass -Pass -Pass -Pass -
Partial

Never considers a module as satisfying a protocol with a method member due to the fact that the method will never exist on the class `types.ModuleType`, only on a given specific module instance.

-Pass +Pass +Pass +Pass +Pass +
Partial

Never considers a module as satisfying a protocol with a method member due to the fact that the method will never exist on the class `types.ModuleType`, only on a given specific module instance.

+Pass      protocols_recursive -Pass -Pass -Pass -Pass -
Partial

Fails to solve a type variable involving a recursive generic protocol.

-Pass +Pass +Pass +Pass +Pass +
Partial

Fails to solve a type variable involving a recursive generic protocol.

+Pass      protocols_runtime_checkable -
Partial

Does not report unsafe overlap for runtime_checkable protocol.

-Pass -Pass -Pass -
Partial

Does not reject `isinstance()` or `issubclass()` calls against runtime-checkable protocols where there is an unsafe overlap between the type of the first argument and the protocol.

-Pass +
Partial

Does not report unsafe overlap for runtime_checkable protocol.

+Pass +Pass +Pass +
Partial

Does not reject `isinstance()` or `issubclass()` calls against runtime-checkable protocols where there is an unsafe overlap between the type of the first argument and the protocol.

+Pass      protocols_self -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      protocols_subtyping -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      protocols_variance -Pass -Pass -Pass -Pass -Unsupported -Pass +Pass +Pass +Pass +Pass +Unsupported +Pass Callables      callables_annotation -
Partial

Incorrectly treats "*args: T, **kwargs: T" as "..." when T is specialized to Any.

Does not treat "*args: Any, **kargs: Any" as "..." when separated by keyword parameter.

-Pass -
Partial

Parameter names are lost when resolving ParamSpec

-Pass -
Partial

Infers a callback protocol as being a gradual type if the callback has signature `__call__[T](self, *args: T, **kwargs: T)` and `T` has been explicitly specialized to `Any`.

-Pass +
Partial

Incorrectly treats "*args: T, **kwargs: T" as "..." when T is specialized to Any.

Does not treat "*args: Any, **kargs: Any" as "..." when separated by keyword parameter.

+Pass +
Partial

Parameter names are lost when resolving ParamSpec

+Pass +
Partial

Infers a callback protocol as being a gradual type if the callback has signature `__call__[T](self, *args: T, **kwargs: T)` and `T` has been explicitly specialized to `Any`.

+Pass      callables_kwargs -
Partial

Allows callable without kwargs to be assigned to callable with unpacked kwargs

-Pass -Pass -Pass -Pass -Pass +
Partial

Allows callable without kwargs to be assigned to callable with unpacked kwargs

+Pass +Pass +Pass +Pass +Pass      callables_protocol -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      callables_subtyping -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass Constructors      constructors_call_init -
Partial

Does not report errors during binding to self parameter of __init__ method.

Does not reject use of class-scoped type variables in annotation of self parameter in __init__ method.

-Pass -Pass -Pass -
Partial

Does not reject invalid argument types to an inherited constructor in a specialized subclass of a generic superclass.

Does not reject class-scoped type variables used in the `self` annotation.

-Pass +
Partial

Does not report errors during binding to self parameter of __init__ method.

Does not reject use of class-scoped type variables in annotation of self parameter in __init__ method.

+Pass +Pass +Pass +
Partial

Does not reject invalid argument types to an inherited constructor in a specialized subclass of a generic superclass.

Does not reject class-scoped type variables used in the `self` annotation.

+Pass      constructors_call_metaclass -
Unsupported

Does not honor metaclass __call__ method when evaluating constructor call.

Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

-Pass -Pass -Pass -Pass -Pass +
Unsupported

Does not honor metaclass __call__ method when evaluating constructor call.

Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

+Pass +Pass +Pass +Pass +Pass      constructors_call_new -
Partial

Does not support __new__ return type that is not a subclass of the class being constructed.

Does not skip evaluation of __init__ based on __new__ return type.

Does not report errors during binding to cls parameter of __new__ method.

-Pass -Pass -Pass -Pass -Pass +
Partial

Does not support __new__ return type that is not a subclass of the class being constructed.

Does not skip evaluation of __init__ based on __new__ return type.

Does not report errors during binding to cls parameter of __new__ method.

+Pass +Pass +Pass +Pass +Pass      constructors_call_type -
Partial

Does not validate call to custom metaclass __call__ method through type[T].

-Pass -Pass -Pass -
Partial

Has overly lenient handling of calls to `type[T]` if `T` is a type variable without an upper bound.

-Pass +
Partial

Does not validate call to custom metaclass __call__ method through type[T].

+Pass +Pass +Pass +
Partial

Has overly lenient handling of calls to `type[T]` if `T` is a type variable without an upper bound.

+Pass      constructors_callable -
Partial

Does not generate a union type for __new__ and __init__ when converting class to callable.

Does not ignore __init__ based on __new__ return type when converting class to callable.

Does not support __new__ return type that is different from class being constructed.

-Pass -
Partial

Converting constructor to callable does not preserve class-scoped type params.

-Pass -
Partial

Does not include `__init__` when `__new__` returns `Self`.

Does not ignore `__init__` when `__new__` returns `Any`.

Unions overload return types.

-Pass +
Partial

Does not generate a union type for __new__ and __init__ when converting class to callable.

Does not ignore __init__ based on __new__ return type when converting class to callable.

Does not support __new__ return type that is different from class being constructed.

+Pass +
Partial

Converting constructor to callable does not preserve class-scoped type params.

+Pass +
Partial

Does not include `__init__` when `__new__` returns `Self`.

Does not ignore `__init__` when `__new__` returns `Any`.

Unions overload return types.

+Pass      constructors_consistency -
Pass*

Does not report inconsistency between __new__ and __init__ (optional).

-Pass -Pass -Pass -Pass -Pass +
Pass*

Does not report inconsistency between __new__ and __init__ (optional).

+Pass +Pass +Pass +Pass +Pass Overloads      overloads_basic -Pass -
Partial

Does less literal promotion than the test asks for.

-Pass -Pass -Pass -Pass +Pass +
Partial

Does less literal promotion than the test asks for.

+Pass +Pass +Pass +Pass      overloads_consistency -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      overloads_definitions -
Partial

Allows @override to be on all overloads and implementation, instead of just implementation.

-Pass -Pass -Pass -Pass -Pass +
Partial

Allows @override to be on all overloads and implementation, instead of just implementation.

+Pass +Pass +Pass +Pass +Pass      overloads_definitions_stub -
Partial

Allows @override to appear in a stub file not on the first overload.

-
Unsupported

Does not support checking stubs.

-Pass -Pass -Pass -Pass +
Partial

Allows @override to appear in a stub file not on the first overload.

+
Unsupported

Does not support checking stubs.

+Pass +Pass +Pass +Pass      overloads_evaluation -
Partial

Does not expand boolean arguments to Literal[True] and Literal[False].

Does not expand enum arguments to literal variants.

Does not expand tuple arguments to possible combinations.

Does not evaluate Any in some cases where overload is ambiguous.

Evaluates Any in some cases where overload is not ambiguous.

-Pass -Pass -
Partial

Does not evaluate Any in some cases where overload is ambiguous.

-Pass -Pass +
Partial

Does not expand boolean arguments to Literal[True] and Literal[False].

Does not expand enum arguments to literal variants.

Does not expand tuple arguments to possible combinations.

Does not evaluate Any in some cases where overload is ambiguous.

Evaluates Any in some cases where overload is not ambiguous.

+Pass +Pass +
Partial

Does not evaluate Any in some cases where overload is ambiguous.

+Pass +Pass Exceptions      exceptions_context_managers -Pass -Pass -
Partial

Some error suppressing context managers are not detected

-Pass -Unsupported -Pass +Pass +Pass +
Partial

Some error suppressing context managers are not detected

+Pass +Unsupported +Pass Dataclasses      dataclasses_descriptors -
Partial

Assumes descriptor behavior only when field is assigned in class body.

Does not correctly evaluate type of descriptor access.

-
Partial

Conformance suite is questionable; see https://github.com/python/typing/issues/2259

-
Partial

* Assumes descriptor behavior only when field is assigned in class body

* Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types

-Pass -
Partial

Only infers a descriptor `__get__` method as being called when a descriptor attribute is accessed on an instance if the descriptor attribute is present in the class namespace.

-Pass +
Partial

Assumes descriptor behavior only when field is assigned in class body.

Does not correctly evaluate type of descriptor access.

+
Partial

Conformance suite is questionable; see https://github.com/python/typing/issues/2259

+
Partial

* Assumes descriptor behavior only when field is assigned in class body

* Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types

+Pass +
Partial

Only infers a descriptor `__get__` method as being called when a descriptor attribute is accessed on an instance if the descriptor attribute is present in the class namespace.

+Pass      dataclasses_final -
Partial

Wrongly requires a Final dataclass field to be initialized at class level.

Doesn't support Final nested inside ClassVar.

-Pass -Pass -Pass -Pass -Pass +
Partial

Wrongly requires a Final dataclass field to be initialized at class level.

Doesn't support Final nested inside ClassVar.

+Pass +Pass +Pass +Pass +Pass      dataclasses_frozen -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      dataclasses_hash -
Unsupported

Does not synthesize `__hash__ = None` as a class attribute for unhashable dataclasses.

Does not report when an unhashable dataclass has `__hash__` called directly on an instance.

Does not report when dataclass is not compatible with Hashable protocol.

-Pass -Pass -Pass -
Partial

Understands the `Hashable` protocol as equivalent to `object`.

-Pass +
Unsupported

Does not synthesize `__hash__ = None` as a class attribute for unhashable dataclasses.

Does not report when an unhashable dataclass has `__hash__` called directly on an instance.

Does not report when dataclass is not compatible with Hashable protocol.

+Pass +Pass +Pass +
Partial

Understands the `Hashable` protocol as equivalent to `object`.

+Pass      dataclasses_inheritance -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      dataclasses_kwonly -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      dataclasses_match_args -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      dataclasses_order -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      dataclasses_postinit -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      dataclasses_slots -
Partial

Does not reject write to instance variable that is not defined in __slots__.

-Pass -Pass -Pass -
Partial

Synthesizes a `__slots__` attribute but does not validate attribute assignments against `__slots__`.

-Pass +
Partial

Does not reject write to instance variable that is not defined in __slots__.

+Pass +Pass +Pass +
Partial

Synthesizes a `__slots__` attribute but does not validate attribute assignments against `__slots__`.

+Pass      dataclasses_transform_class -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      dataclasses_transform_converter -
Unsupported

Converter parameter not yet supported.

-Pass -Pass -Pass -Pass -Pass +
Unsupported

Converter parameter not yet supported.

+Pass +Pass +Pass +Pass +Pass      dataclasses_transform_field -
Partial

Does not properly handle field constructor that has default value for `kw_only` or `init` parameter.

-Pass -Pass -Pass -Pass -Pass +
Partial

Does not properly handle field constructor that has default value for `kw_only` or `init` parameter.

+Pass +Pass +Pass +Pass +Pass      dataclasses_transform_func -
Partial

Does not handle `kw_only=False` override when `kw_only_default=True`.

Does not report error when `order=False` and comparison operators are used.

-Pass -Pass -Pass -Pass -Pass +
Partial

Does not handle `kw_only=False` override when `kw_only_default=True`.

Does not report error when `order=False` and comparison operators are used.

+Pass +Pass +Pass +Pass +Pass      dataclasses_transform_meta -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      dataclasses_usage -
Pass*

Does not detect unannotated usage of `dataclasses.field()`.

-Pass -Pass -Pass -Pass -Pass +
Pass*

Does not detect unannotated usage of `dataclasses.field()`.

+Pass +Pass +Pass +Pass +Pass Typed dictionaries      typeddicts_alt_syntax -
Pass*

Does not support keyword-argument form of alternative syntax (deprecated in 3.11).

-Pass -Pass -Pass -Pass -Pass +
Pass*

Does not support keyword-argument form of alternative syntax (deprecated in 3.11).

+Pass +Pass +Pass +Pass +Pass      typeddicts_class_syntax -
Partial

Does not support version-conditional items in TypedDict definitions.

-Pass -Pass -
Partial

Does not support version-conditional items in TypedDict definitions.

-Pass -Pass +
Partial

Does not support version-conditional items in TypedDict definitions.

+Pass +Pass +
Partial

Does not support version-conditional items in TypedDict definitions.

+Pass +Pass      typeddicts_extra_items -
Unsupported

Not supported.

-Pass -Pass -Pass -Pass -Pass +
Unsupported

Not supported.

+Pass +Pass +Pass +Pass +Pass      typeddicts_final -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      typeddicts_inheritance -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      typeddicts_operations -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      typeddicts_readonly -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      typeddicts_readonly_consistency -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      typeddicts_readonly_inheritance -
Partial

Incorrectly rejects non-ReadOnly override of ReadOnly item.

Incorrectly rejects override of ReadOnly item with another ReadOnly item with narrower type.

Incorrectly rejects override of NotRequired ReadOnly item with a Required ReadOnly item.

-Pass -Pass -Pass -Pass -Pass +
Partial

Incorrectly rejects non-ReadOnly override of ReadOnly item.

Incorrectly rejects override of ReadOnly item with another ReadOnly item with narrower type.

Incorrectly rejects override of NotRequired ReadOnly item with a Required ReadOnly item.

+Pass +Pass +Pass +Pass +Pass      typeddicts_readonly_kwargs -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      typeddicts_readonly_update -
Partial

Incorrectly allows update of ReadOnly item.

Incorrectly rejects update involving an item with Never type.

-Pass -Pass -Pass -Pass -Pass +
Partial

Incorrectly allows update of ReadOnly item.

Incorrectly rejects update involving an item with Never type.

+Pass +Pass +Pass +Pass +Pass      typeddicts_required -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      typeddicts_type_consistency -Pass -
Partial

Considers TypedDicts to be assignable to plain dict types.

-Pass -Pass -Pass -Pass +Pass +
Partial

Considers TypedDicts to be assignable to plain dict types.

+Pass +Pass +Pass +Pass      typeddicts_usage -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass Tuples      tuples_type_compat -
Partial

Does not support tuple narrowing based on `len()` type guard (optional).

Incorrectly marks a match case as unreachable.

-Pass -Pass -Pass -Pass -Pass +
Partial

Does not support tuple narrowing based on `len()` type guard (optional).

Incorrectly marks a match case as unreachable.

+Pass +Pass +Pass +Pass +Pass      tuples_type_form -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      tuples_unpacked -
Partial

"More than one unpack" error is missing in some cases.

-Pass -Pass -Pass -Pass -Pass +
Partial

"More than one unpack" error is missing in some cases.

+Pass +Pass +Pass +Pass +Pass Named tuples      namedtuples_define_class -
Partial

Does not reject override of named tuple attribute in child class.

Does not support version-conditional fields.

-
Partial

Does not support precise type inference for slices over namedtuples.

-Pass -Pass -Pass -Pass +
Partial

Does not reject override of named tuple attribute in child class.

Does not support version-conditional fields.

+
Partial

Does not support precise type inference for slices over namedtuples.

+Pass +Pass +Pass +Pass      namedtuples_define_functional -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      namedtuples_type_compat -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      namedtuples_usage -
Partial

Does not reject attempt to delete named tuple field by name.

-Pass -Pass -Pass -Pass -Pass +
Partial

Does not reject attempt to delete named tuple field by name.

+Pass +Pass +Pass +Pass +Pass Enumerations      enums_behaviors -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      enums_definition -Pass -
Partial

Does not allow enum members to be conditional on version/platform checks.

-Pass -Pass -Pass -Pass +Pass +
Partial

Does not allow enum members to be conditional on version/platform checks.

+Pass +Pass +Pass +Pass      enums_expansion -
Partial

Improperly applies narrowing to Flag subclass.

-Pass -Pass -Pass -
Partial

Does not support `enum.Flag`.

-Pass +
Partial

Improperly applies narrowing to Flag subclass.

+Pass +Pass +Pass +
Partial

Does not support `enum.Flag`.

+Pass      enums_member_names -
Pass*

Does not support special-cased handling of member name literal types in some cases (optional).

-Pass -Pass -Pass -Pass -Pass +
Pass*

Does not support special-cased handling of member name literal types in some cases (optional).

+Pass +Pass +Pass +Pass +Pass      enums_member_values -
Partial

Does not enforce declared type of `_value_`.

Does not enforce assigned tuple types for enum members (optional).

-Pass -Pass -Pass -Pass -Pass +
Partial

Does not enforce declared type of `_value_`.

Does not enforce assigned tuple types for enum members (optional).

+Pass +Pass +Pass +Pass +Pass      enums_members -
Partial

Does not treat attribute with annotation and no assignment as non-member.

Does not treat callables as non-members.

Does not honor `enum.member` as method decorator.

Does not properly handle aliased enum members.

Does not support `_ignore_` mechanism (optional).

Does not treat attributes with private names as non-members.

-Pass -Pass -
Pass*

Does not support `_ignore_` mechanism (optional).

-Pass -Pass +
Partial

Does not treat attribute with annotation and no assignment as non-member.

Does not treat callables as non-members.

Does not honor `enum.member` as method decorator.

Does not properly handle aliased enum members.

Does not support `_ignore_` mechanism (optional).

Does not treat attributes with private names as non-members.

+Pass +Pass +
Pass*

Does not support `_ignore_` mechanism (optional).

+Pass +Pass Type narrowing      narrowing_typeguard -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      narrowing_typeis -Pass -Pass -Pass -Pass -
Partial

Intersects the pre-existing type with the top materialization of the bracketed type rather than the bracketed type itself.

-Pass +Pass +Pass +Pass +Pass +
Partial

Intersects the pre-existing type with the top materialization of the bracketed type rather than the bracketed type itself.

+Pass Type checker directives      directives_assert_type -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      directives_cast -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      directives_deprecated -Pass -Pass -Pass -Pass -
Partial

Does not detect calls to deprecated overloads.

Does not detect implicit calls to deprecated dunder methods, for example via operators.

Does not detect accesses of, or attempts to set, deprecated properties.

-Pass +Pass +Pass +Pass +Pass +
Partial

Does not detect calls to deprecated overloads.

Does not detect implicit calls to deprecated dunder methods, for example via operators.

Does not detect accesses of, or attempts to set, deprecated properties.

+Pass      directives_disjoint_base -Pass -
Partial

Does not reject invalid class definitions due to disjoint bases, but uses disjoint base information in type narrowing.

-Pass -
Unsupported

Does not support PEP 800 disjoint-base semantics.

-Pass -Pass +Pass +
Partial

Does not reject invalid class definitions due to disjoint bases, but uses disjoint base information in type narrowing.

+Pass +
Unsupported

Does not support PEP 800 disjoint-base semantics.

+Pass +Pass      directives_no_type_check -
Partial

Does not honor `@no_type_check` class decorator (allowed).

Does not reject invalid call of `@no_type_check` function.

-Pass -Pass -
Pass*

Does not honor `@no_type_check` class decorator (allowed).

-Pass -Pass +
Partial

Does not honor `@no_type_check` class decorator (allowed).

Does not reject invalid call of `@no_type_check` function.

+Pass +Pass +
Pass*

Does not honor `@no_type_check` class decorator (allowed).

+Pass +Pass      directives_reveal_type -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      directives_type_checking -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      directives_type_ignore -
Partial

Does not honor "# type: ignore" comment if comment includes additional text.

-Pass -Pass -Pass -Pass -Pass +
Partial

Does not honor "# type: ignore" comment if comment includes additional text.

+Pass +Pass +Pass +Pass +Pass      directives_type_ignore_file1 -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      directives_type_ignore_file2 -Pass -Pass -Pass -Pass -Pass -Pass +Pass +Pass +Pass +Pass +Pass +Pass      directives_version_platform -
Pass*

Does not understand three-element form of sys.version checks.

Does not understand os.name checks.

-Pass -Pass -Pass -Pass -Pass +
Pass*

Does not understand three-element form of sys.version checks.

Does not understand os.name checks.

+Pass +Pass +Pass +Pass +Pass Historical and deprecated features      historical_positional -
Partial

Does not reject positional-only parameter after non-positional-only parameter.

Treats keyword-only parameter as positional-only.

Applies legacy positional-only rules when PEP 570 syntax is used.

-Pass -Pass -Pass -Pass -Pass +
Partial

Does not reject positional-only parameter after non-positional-only parameter.

Treats keyword-only parameter as positional-only.

Applies legacy positional-only rules when PEP 570 syntax is used.

+Pass +Pass +Pass +Pass +Pass diff --git a/conformance/src/reporting.py b/conformance/src/reporting.py index ae30dcd55..591f31106 100644 --- a/conformance/src/reporting.py +++ b/conformance/src/reporting.py @@ -114,7 +114,7 @@ def generate_summary_html(root_dir: Path) -> str: if raw_notes != "": conformance_cell = f'
{conformance_cell}
{notes}
' - summary_html.append(f'{conformance_cell}') + summary_html.append(f'{conformance_cell}') summary_html.append("") From 37f4f5795ba5f4af4be924aff9231078dd6fef5f Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:31 +0100 Subject: [PATCH 08/30] Add a `` for header row --- conformance/results/results.html | 4 ++-- conformance/src/reporting.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conformance/results/results.html b/conformance/results/results.html index 4c4c67a72..fea078eab 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -172,7 +172,7 @@

Python Type System Conformance Test Results

- + @@ -186,7 +186,7 @@

Python Type System Conformance Test Results

- + diff --git a/conformance/src/reporting.py b/conformance/src/reporting.py index 591f31106..18b25017d 100644 --- a/conformance/src/reporting.py +++ b/conformance/src/reporting.py @@ -30,7 +30,7 @@ def generate_summary_html(root_dir: Path) -> str: test_groups = get_test_groups(root_dir) test_cases = get_test_cases(test_groups, root_dir / "tests") - summary_html = [""] + summary_html = [""] summary_html.append('') for type_checker in type_checkers: @@ -51,7 +51,7 @@ def generate_summary_html(root_dir: Path) -> str: summary_html.append(f"") - summary_html.append("") + summary_html.append("") for test_group_name, test_group in test_groups.items(): tests_in_group = [ From c227f71d97ed86853d9d5928b1f1b7d647fbc692 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:31 +0100 Subject: [PATCH 09/30] Simplify applying styles to header row --- conformance/results/results.html | 37 +++++++++++---------------- conformance/src/reporting.py | 8 +++--- conformance/src/results_template.html | 12 +++------ 3 files changed, 22 insertions(+), 35 deletions(-) diff --git a/conformance/results/results.html b/conformance/results/results.html index fea078eab..06e33cd1e 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -82,15 +82,11 @@ text-align: center; } - .tc-header { - padding-top: 4px; - text-align: center; - } - - .tc-name { + thead > tr > th { font-size: 16px; - margin-top: 8px; - margin-bottom: 8px; + padding-bottom: 8px; + padding-top: 12px; + text-align: center; } .tc-time { @@ -173,20 +169,17 @@

Python Type System Conformance Test Results

 
mypy 2.1.0
zuban 0.8.2
Type annotations
 
{version}
") summary_html.append("
- - - - - - - - + + + + + + + + + + + diff --git a/conformance/src/reporting.py b/conformance/src/reporting.py index 18b25017d..4424e24a3 100644 --- a/conformance/src/reporting.py +++ b/conformance/src/reporting.py @@ -30,8 +30,7 @@ def generate_summary_html(root_dir: Path) -> str: test_groups = get_test_groups(root_dir) test_cases = get_test_cases(test_groups, root_dir / "tests") - summary_html = [""] - summary_html.append('') + summary_html = ["", "", ""] for type_checker in type_checkers: # Load the version file for the type checker. @@ -48,10 +47,9 @@ def generate_summary_html(root_dir: Path) -> str: version = existing_info["version"] or "Unknown version" - summary_html.append(f"") + summary_html.append(f"") - summary_html.append("") + summary_html.extend(["", "", ""]) for test_group_name, test_group in test_groups.items(): tests_in_group = [ diff --git a/conformance/src/results_template.html b/conformance/src/results_template.html index 1da1e3b1c..4cfd4e666 100644 --- a/conformance/src/results_template.html +++ b/conformance/src/results_template.html @@ -82,15 +82,11 @@ text-align: center; } - .tc-header { - padding-top: 4px; - text-align: center; - } - - .tc-name { + thead > tr > th { font-size: 16px; - margin-top: 8px; - margin-bottom: 8px; + padding-bottom: 8px; + padding-top: 12px; + text-align: center; } .tc-time { From 3f5bd6fea086b951947334017b9bbd1f6f261c55 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:31 +0100 Subject: [PATCH 10/30] Simplify applying styles to body columns --- conformance/results/results.html | 2299 +++++++++++++------------ conformance/src/reporting.py | 6 +- conformance/src/results_template.html | 3 +- 3 files changed, 1155 insertions(+), 1153 deletions(-) diff --git a/conformance/results/results.html b/conformance/results/results.html index 06e33cd1e..2bed4e459 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -52,7 +52,8 @@ margin-left: 20px; } - .column { + tbody > tr > th, + tbody > tr > td { padding-left: 8px; padding-right: 8px; padding-top: 4px; @@ -180,1196 +181,1196 @@

Python Type System Conformance Test Results

 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - + + + + + + +
 
mypy 2.1.0
-
pycroscope 0.4.0
-
pyrefly 1.1.0
-
pyright 1.1.410
-
ty 0.0.50
-
zuban 0.8.2
-
mypy 2.1.0pycroscope 0.4.0pyrefly 1.1.0pyright 1.1.410ty 0.0.50zuban 0.8.2
Type annotations
 
{version}
") - summary_html.append("		{version}
+
Type annotations
     annotations_coroutinesPassPassPassPassPassPass
     annotations_forward_refs
Partial

Does not report error for a forward reference that is not enclosed in quotes.

Does not report error for use of quoted type with "|" operator (runtime error).

Incorrectly generates error for quoted type defined in class scope.

Partial

Fails to reject "x" | int annotations that fail at runtime.

Rejects some valid quoted annotations.

Partial

Types in quotes incorrectly refer to shadowing class member.

Does not reject some type forms that require quotes.

Pass
Partial

Resolves references in type annotations as referring to end-of-scope types (https://discuss.python.org/t/annotation-string-references-in-class-scope-in-conformance-tests/105439, https://github.com/python/typing/pull/2144)

Partial

Incorrectly generates error for quoted type defined in class scope.

     annotations_generators
Partial

Does not report incompatible Generator type in `yield from` statement.

PassPassPassPassPass
     annotations_methods
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

PassPass
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

PassPass
     annotations_typeexprPass
Partial

Fails to reject various weird annotations

PassPassPassPass
+
     annotations_coroutinesPassPassPassPassPassPass
     annotations_forward_refs
Partial

Does not report error for a forward reference that is not enclosed in quotes.

Does not report error for use of quoted type with "|" operator (runtime error).

Incorrectly generates error for quoted type defined in class scope.

Partial

Fails to reject "x" | int annotations that fail at runtime.

Rejects some valid quoted annotations.

Partial

Types in quotes incorrectly refer to shadowing class member.

Does not reject some type forms that require quotes.

Pass
Partial

Resolves references in type annotations as referring to end-of-scope types (https://discuss.python.org/t/annotation-string-references-in-class-scope-in-conformance-tests/105439, https://github.com/python/typing/pull/2144)

Partial

Incorrectly generates error for quoted type defined in class scope.

     annotations_generators
Partial

Does not report incompatible Generator type in `yield from` statement.

PassPassPassPassPass
     annotations_methods
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

PassPass
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

PassPass
     annotations_typeexprPass
Partial

Fails to reject various weird annotations

PassPassPassPass
Type forms
     typeforms_typeform
Partial

Does not support assigning Union and GenericAlias objects to their runtime types.

Partial

Fails to reject various weird annotations

PassUnsupportedPassPass
+
     typeforms_typeform
Partial

Does not support assigning Union and GenericAlias objects to their runtime types.

Partial

Fails to reject various weird annotations

PassUnsupportedPassPass
Special types in annotations
     specialtypes_anyPassPassPassPassPassPass
     specialtypes_neverPass
Partial

Does not enforce invariance in some contexts

Does not allow Any to be assigned to Never

PassPassPassPass
     specialtypes_nonePassPassPassPassPassPass
     specialtypes_promotionsPassPassPassPassPassPass
     specialtypes_type
Partial

Does not treat `type` same as `type[Any]` for assert_type.

Does not allow access to unknown attributes from object of type `type[Any]`.

PassPassPass
Partial

Allows arbitrary attributes to be accessed on `TA` where `TA = typing.Type[typing.Any]` or `TA = type[typing.Any]`.

Treats `type` equivalently to `type[object]` rather than `type[typing.Any]`.

Pass
+
     specialtypes_anyPassPassPassPassPassPass
     specialtypes_neverPass
Partial

Does not enforce invariance in some contexts

Does not allow Any to be assigned to Never

PassPassPassPass
     specialtypes_nonePassPassPassPassPassPass
     specialtypes_promotionsPassPassPassPassPassPass
     specialtypes_type
Partial

Does not treat `type` same as `type[Any]` for assert_type.

Does not allow access to unknown attributes from object of type `type[Any]`.

PassPassPass
Partial

Allows arbitrary attributes to be accessed on `TA` where `TA = typing.Type[typing.Any]` or `TA = type[typing.Any]`.

Treats `type` equivalently to `type[object]` rather than `type[typing.Any]`.

Pass
Generics
     generics_base_class
Partial

Does not detect inconsistent type variable ordering.

PassPassPassPassPass
     generics_basicPassPassPassPass
Partial

Incorrectly allows constrained type variables to be solved to a union of their constraints.

Pass
     generics_defaults
Partial

Does not detect a TypeVar with a default used after a TypeVarTuple.

Does not fully support defaults on TypeVarTuple and ParamSpec.

Partial

Numerous issues; does not support TypeVarTuple and ParamSpec defaults.

PassPass
Partial

Does not forbid a `TypeVar` immediately following a `TypeVarTuple` in a parameter list from having a default.

Does not support `TypeVarTuple`.

Pass
     generics_defaults_referential
Partial

Does not correctly handle defaults referencing other TypeVars.

Partial

Fails to apply default specializations in some cases.

PassPassPassPass
     generics_defaults_specialization
Partial

Does not correctly resolve defaults when classes are used directly.

PassPass
Partial

Allows incorrect assignment to type[].

Partial

Does not reject subscription of an already-specialized generic class.

Pass
     generics_paramspec_basicPassPassPassPassPassPass
     generics_paramspec_componentsPassPassPassPass
Partial

Incorrectly allows using `*args: P.args` and `**kwargs: P.kwargs` when `P` has not been put into scope by any other parameter annotation or enclosing scope.

Pass
     generics_paramspec_semanticsPassPassPass
Pass*

Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).

PassPass
     generics_paramspec_specializationPassPassPassPassPassPass
     generics_scopingPassPass
Partial

Does not implement several scoping checks/restrictions for generics

Pass
Partial

Does not reject `list[T]()` in the global scope, where `T` is an unbound type variable.

Does nto reject `alias: TypeAlias = list[T]` in the body scope of a class generic over a type variable `T`.

Pass
     generics_self_advanced
Partial

Does not infer the type of an unannotated `self` parameter to be type `Self`.

Does not retain `Self` when calling method that returns `Self`.

Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`.

Does not retain `Self` when accessing attribute through `type[Self]`.

Pass
Pass*

Treats attributes not initialized on the class as instance-only

PassPassPass
     generics_self_attributesPassPassPassPassPassPass
     generics_self_basicPassPassPassPassPassPass
     generics_self_protocolsPassPassPassPassPassPass
     generics_self_usagePassPassPassPass
Partial

Does not reject `Self` used in a return annotation when `self` is annotated using another type variable.

Pass
     generics_syntax_compatibilityPassPassPassPassPassPass
     generics_syntax_declarationsPassPassPassPassPassPass
     generics_syntax_infer_variance
Unsupported

Type parameter syntax not yet supported.

PassPassPassPassPass
     generics_syntax_scoping
Partial

Does not following runtime scoping rules for type parameters in all cases.

Partial

Misses some details of scoping rules.

PassPassPassPass
     generics_type_erasure
Partial

Infers Node[Never] instead of Node[Any] when argument is not provided.

False negative on instance attribute access on type(node).

PassPassPass
Partial

Does not reject access of generic instance variable from the class object.

Pass
     generics_typevartuple_args
Partial

Does not enforce that tuples captured by TypeVarTuple are of the same length.

PassPass
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

Partial

Supports PEP-646 unpacked tuples but not TypeVarTuple.

Pass
     generics_typevartuple_basic
Partial

Does not enforce that tuples captured by TypeVarTuple are same length.

PassPass
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

UnsupportedPass
     generics_typevartuple_callablePassPassPassPassUnsupportedPass
     generics_typevartuple_concatPassPassPassPassUnsupportedPass
     generics_typevartuple_overloadsPassPassPassPassPassPass
     generics_typevartuple_specialization
Partial

Incorrectly specializes generic alias that includes a TypeVar and TypeVarTuple if no type arguments are provided.

Rejects specialization of generic type alias defined as a tuple containing a TypeVar.

PassPassPassUnsupportedPass
     generics_typevartuple_unpackPassPassPassPassUnsupportedPass
     generics_upper_bound
Partial

Does not reject use of type variable within an upper bound.

PassPassPassPassPass
     generics_variance
Partial

Does not reject use of class-scoped TypeVar used in a base class when variance is incompatible.

PassPassPassPassPass
     generics_variance_inferencePassPassPassPassPassPass
+
     generics_base_class
Partial

Does not detect inconsistent type variable ordering.

PassPassPassPassPass
     generics_basicPassPassPassPass
Partial

Incorrectly allows constrained type variables to be solved to a union of their constraints.

Pass
     generics_defaults
Partial

Does not detect a TypeVar with a default used after a TypeVarTuple.

Does not fully support defaults on TypeVarTuple and ParamSpec.

Partial

Numerous issues; does not support TypeVarTuple and ParamSpec defaults.

PassPass
Partial

Does not forbid a `TypeVar` immediately following a `TypeVarTuple` in a parameter list from having a default.

Does not support `TypeVarTuple`.

Pass
     generics_defaults_referential
Partial

Does not correctly handle defaults referencing other TypeVars.

Partial

Fails to apply default specializations in some cases.

PassPassPassPass
     generics_defaults_specialization
Partial

Does not correctly resolve defaults when classes are used directly.

PassPass
Partial

Allows incorrect assignment to type[].

Partial

Does not reject subscription of an already-specialized generic class.

Pass
     generics_paramspec_basicPassPassPassPassPassPass
     generics_paramspec_componentsPassPassPassPass
Partial

Incorrectly allows using `*args: P.args` and `**kwargs: P.kwargs` when `P` has not been put into scope by any other parameter annotation or enclosing scope.

Pass
     generics_paramspec_semanticsPassPassPass
Pass*

Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).

PassPass
     generics_paramspec_specializationPassPassPassPassPassPass
     generics_scopingPassPass
Partial

Does not implement several scoping checks/restrictions for generics

Pass
Partial

Does not reject `list[T]()` in the global scope, where `T` is an unbound type variable.

Does nto reject `alias: TypeAlias = list[T]` in the body scope of a class generic over a type variable `T`.

Pass
     generics_self_advanced
Partial

Does not infer the type of an unannotated `self` parameter to be type `Self`.

Does not retain `Self` when calling method that returns `Self`.

Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`.

Does not retain `Self` when accessing attribute through `type[Self]`.

Pass
Pass*

Treats attributes not initialized on the class as instance-only

PassPassPass
     generics_self_attributesPassPassPassPassPassPass
     generics_self_basicPassPassPassPassPassPass
     generics_self_protocolsPassPassPassPassPassPass
     generics_self_usagePassPassPassPass
Partial

Does not reject `Self` used in a return annotation when `self` is annotated using another type variable.

Pass
     generics_syntax_compatibilityPassPassPassPassPassPass
     generics_syntax_declarationsPassPassPassPassPassPass
     generics_syntax_infer_variance
Unsupported

Type parameter syntax not yet supported.

PassPassPassPassPass
     generics_syntax_scoping
Partial

Does not following runtime scoping rules for type parameters in all cases.

Partial

Misses some details of scoping rules.

PassPassPassPass
     generics_type_erasure
Partial

Infers Node[Never] instead of Node[Any] when argument is not provided.

False negative on instance attribute access on type(node).

PassPassPass
Partial

Does not reject access of generic instance variable from the class object.

Pass
     generics_typevartuple_args
Partial

Does not enforce that tuples captured by TypeVarTuple are of the same length.

PassPass
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

Partial

Supports PEP-646 unpacked tuples but not TypeVarTuple.

Pass
     generics_typevartuple_basic
Partial

Does not enforce that tuples captured by TypeVarTuple are same length.

PassPass
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

UnsupportedPass
     generics_typevartuple_callablePassPassPassPassUnsupportedPass
     generics_typevartuple_concatPassPassPassPassUnsupportedPass
     generics_typevartuple_overloadsPassPassPassPassPassPass
     generics_typevartuple_specialization
Partial

Incorrectly specializes generic alias that includes a TypeVar and TypeVarTuple if no type arguments are provided.

Rejects specialization of generic type alias defined as a tuple containing a TypeVar.

PassPassPassUnsupportedPass
     generics_typevartuple_unpackPassPassPassPassUnsupportedPass
     generics_upper_bound
Partial

Does not reject use of type variable within an upper bound.

PassPassPassPassPass
     generics_variance
Partial

Does not reject use of class-scoped TypeVar used in a base class when variance is incompatible.

PassPassPassPassPass
     generics_variance_inferencePassPassPassPassPassPass
Type qualifiers
     qualifiers_annotated
Partial

Does not allow ClassVar to be nested within Annotated.

Does not allow Final to be nested within Annotated.

Does not allow Required and NotRequired to be nested within Annotated.

Does not reject type[T] compatibility for type alias defined with Annotated.

Does not reject call of type alias defined with Annotated.

Partial

Fails to reject various weird annotations.

False positive on lambda in Annotated.

PassPassPassPass
     qualifiers_final_annotation
Partial

Does not treat use of Final name as if it was replaced by the literal in NamedTuple definition.

Does not allow conditional assignment of Final instance variable in __init__ method.

Does not allow redefinition of private class variable that is marked Final in parent class.

Does not report modification of local Final variable via "for" statement.

PassPassPassPassPass
     qualifiers_final_decoratorPassPassPassPassPassPass
+
     qualifiers_annotated
Partial

Does not allow ClassVar to be nested within Annotated.

Does not allow Final to be nested within Annotated.

Does not allow Required and NotRequired to be nested within Annotated.

Does not reject type[T] compatibility for type alias defined with Annotated.

Does not reject call of type alias defined with Annotated.

Partial

Fails to reject various weird annotations.

False positive on lambda in Annotated.

PassPassPassPass
     qualifiers_final_annotation
Partial

Does not treat use of Final name as if it was replaced by the literal in NamedTuple definition.

Does not allow conditional assignment of Final instance variable in __init__ method.

Does not allow redefinition of private class variable that is marked Final in parent class.

Does not report modification of local Final variable via "for" statement.

PassPassPassPassPass
     qualifiers_final_decoratorPassPassPassPassPassPass
Class type compatibility
     classes_classvar
Partial

Internal error if TypeVarTuple is used in ClassVar.

Does not reject use of ParamSpec in ClassVar.

Rejects ClassVar nested in Annotated.

Does not reject use of ClassVar in TypeAlias definition.

PassPassPassPassPass
     classes_overridePassPassPassPassPassPass
+
     classes_classvar
Partial

Internal error if TypeVarTuple is used in ClassVar.

Does not reject use of ParamSpec in ClassVar.

Rejects ClassVar nested in Annotated.

Does not reject use of ClassVar in TypeAlias definition.

PassPassPassPassPass
     classes_overridePassPassPassPassPassPass
Type aliases
     aliases_explicit
Partial

Does not reject specialization of type alias that has already been implicitly specialized.

PassPassPass
Partial

Does not emit a diagnostic if a type alias to a union, in which every element of the union is implicitly specialized with `Unknown`, is invalidly specialized again.

Pass
     aliases_implicitPass
Partial

Fails to handle various weird annotations.

Various bugs with resolving generic aliases.

PassPass
Partial

Does not reject variables with `Any` or `Unknown` types when used as implicit type aliases.

Falls short on full syntactic validation of type aliases.

Pass
     aliases_newtype
Partial

`NewType`s are incorrectly considered to be classes.

PassPassPassPassPass
     aliases_recursivePassPassPassPassUnsupportedPass
     aliases_type_statementPass
Partial

Fails to reject various weird annotations.

PassPass
Partial

Does not reject circular definitions of type aliases.

Does not support `type` statements generic over `TypeVarTuple`s.

Pass
     aliases_typealiastype
Partial

Incorrectly rejects some recursive type aliases using TypeAliasType.

Incorrectly rejects the use of a class-scoped TypeVar in a TypeAliasType definition.

Partial

Rejects valid ParamSpec specialization.

PassPass
Partial

Does not reject specializing a type parameter in a generic type alias with a type inconsistent with the parameter's upper bound.

Does not reject declaring a type alias with a type variable that is not in scope.

Does not reject declaring a type alias with a non-literal tuple passed to the `type_params` parameter.

Does not reject cyclically defined type aliases.

Pass
     aliases_variancePassPassPassPassPassPass
+
     aliases_explicit
Partial

Does not reject specialization of type alias that has already been implicitly specialized.

PassPassPass
Partial

Does not emit a diagnostic if a type alias to a union, in which every element of the union is implicitly specialized with `Unknown`, is invalidly specialized again.

Pass
     aliases_implicitPass
Partial

Fails to handle various weird annotations.

Various bugs with resolving generic aliases.

PassPass
Partial

Does not reject variables with `Any` or `Unknown` types when used as implicit type aliases.

Falls short on full syntactic validation of type aliases.

Pass
     aliases_newtype
Partial

`NewType`s are incorrectly considered to be classes.

PassPassPassPassPass
     aliases_recursivePassPassPassPassUnsupportedPass
     aliases_type_statementPass
Partial

Fails to reject various weird annotations.

PassPass
Partial

Does not reject circular definitions of type aliases.

Does not support `type` statements generic over `TypeVarTuple`s.

Pass
     aliases_typealiastype
Partial

Incorrectly rejects some recursive type aliases using TypeAliasType.

Incorrectly rejects the use of a class-scoped TypeVar in a TypeAliasType definition.

Partial

Rejects valid ParamSpec specialization.

PassPass
Partial

Does not reject specializing a type parameter in a generic type alias with a type inconsistent with the parameter's upper bound.

Does not reject declaring a type alias with a type variable that is not in scope.

Does not reject declaring a type alias with a non-literal tuple passed to the `type_params` parameter.

Does not reject cyclically defined type aliases.

Pass
     aliases_variancePassPassPassPassPassPass
Literals
     literals_interactions
Pass*

Does not narrow `str` or `LiteralString` types to `Literal` string types via equality or containment checks.

PassPassPass
Pass*

Deliberately does not allow `str` to be narrowed to literal string types through equality or containment checks due to the possibility of `str` subclasses that could have unexpected equality semantics.

Pass
     literals_literalstring
Unsupported

Support for `LiteralString` is not implemented.

PassPassPassPassPass
     literals_parameterizations
Partial

Does not reject tuple within Literal.

Partial

Fails to reject various invalid literal parameterizations.

PassPassPassPass
     literals_semanticsPassPassPassPassPassPass
+
     literals_interactions
Pass*

Does not narrow `str` or `LiteralString` types to `Literal` string types via equality or containment checks.

PassPassPass
Pass*

Deliberately does not allow `str` to be narrowed to literal string types through equality or containment checks due to the possibility of `str` subclasses that could have unexpected equality semantics.

Pass
     literals_literalstring
Unsupported

Support for `LiteralString` is not implemented.

PassPassPassPassPass
     literals_parameterizations
Partial

Does not reject tuple within Literal.

Partial

Fails to reject various invalid literal parameterizations.

PassPassPassPass
     literals_semanticsPassPassPassPassPassPass
Protocols
     protocols_class_objectsPass
Partial

Abstract type[Proto] still allows protocol class objects in some paths.

PassPass
Unsupported

`type[Proto]` is not yet supported.

`ClassVar` protocol members are not yet supported.

`@property` protocol members are only partially supported.

A class object `C` is only considered to inhabit a protocol type with a method member `f` if `f` exists as an attribute on the metaclass of `C`.

Pass
     protocols_definition
Partial

Does not detect protocol mismatch if concrete method is missing annotations.

Does not detect protocol mismatch if concrete method's parameters are position-only.

PassPassPass
Partial

Does not reject implicit instance attributes in `Protocol` methods.

Does not support `ClassVar` protocol members.

Incorrectly considers `ClassVar` attributes on concrete classes as satisfying non-`ClassVar` attribute members on protocols.

Only has partial support for `@property` protocol members.

Pass
     protocols_explicit
Pass*

Does not report unimplemented attributes for class that explicitly derives from protocol until it is instantiated.

PassPassPass
Unsupported

Allows implicitly abstract protocol methods to be called via `super()` on a protocol subclass.

Allows instantiation of abstract subclasses of protocol classes.

Pass
     protocols_genericPass
Partial

Fails to reject duplicate generic/protocol bases.

Treats global object as a literal.

PassPass
Partial

Only partially supports `@property` protocol members.

Pass
     protocols_mergingPassPassPassPass
Partial

Does not reject attempted instantiation of abstract subclasses of protocols.

Pass
     protocols_modulesPassPassPassPass
Partial

Never considers a module as satisfying a protocol with a method member due to the fact that the method will never exist on the class `types.ModuleType`, only on a given specific module instance.

Pass
     protocols_recursivePassPassPassPass
Partial

Fails to solve a type variable involving a recursive generic protocol.

Pass
     protocols_runtime_checkable
Partial

Does not report unsafe overlap for runtime_checkable protocol.

PassPassPass
Partial

Does not reject `isinstance()` or `issubclass()` calls against runtime-checkable protocols where there is an unsafe overlap between the type of the first argument and the protocol.

Pass
     protocols_selfPassPassPassPassPassPass
     protocols_subtypingPassPassPassPassPassPass
     protocols_variancePassPassPassPassUnsupportedPass
+
     protocols_class_objectsPass
Partial

Abstract type[Proto] still allows protocol class objects in some paths.

PassPass
Unsupported

`type[Proto]` is not yet supported.

`ClassVar` protocol members are not yet supported.

`@property` protocol members are only partially supported.

A class object `C` is only considered to inhabit a protocol type with a method member `f` if `f` exists as an attribute on the metaclass of `C`.

Pass
     protocols_definition
Partial

Does not detect protocol mismatch if concrete method is missing annotations.

Does not detect protocol mismatch if concrete method's parameters are position-only.

PassPassPass
Partial

Does not reject implicit instance attributes in `Protocol` methods.

Does not support `ClassVar` protocol members.

Incorrectly considers `ClassVar` attributes on concrete classes as satisfying non-`ClassVar` attribute members on protocols.

Only has partial support for `@property` protocol members.

Pass
     protocols_explicit
Pass*

Does not report unimplemented attributes for class that explicitly derives from protocol until it is instantiated.

PassPassPass
Unsupported

Allows implicitly abstract protocol methods to be called via `super()` on a protocol subclass.

Allows instantiation of abstract subclasses of protocol classes.

Pass
     protocols_genericPass
Partial

Fails to reject duplicate generic/protocol bases.

Treats global object as a literal.

PassPass
Partial

Only partially supports `@property` protocol members.

Pass
     protocols_mergingPassPassPassPass
Partial

Does not reject attempted instantiation of abstract subclasses of protocols.

Pass
     protocols_modulesPassPassPassPass
Partial

Never considers a module as satisfying a protocol with a method member due to the fact that the method will never exist on the class `types.ModuleType`, only on a given specific module instance.

Pass
     protocols_recursivePassPassPassPass
Partial

Fails to solve a type variable involving a recursive generic protocol.

Pass
     protocols_runtime_checkable
Partial

Does not report unsafe overlap for runtime_checkable protocol.

PassPassPass
Partial

Does not reject `isinstance()` or `issubclass()` calls against runtime-checkable protocols where there is an unsafe overlap between the type of the first argument and the protocol.

Pass
     protocols_selfPassPassPassPassPassPass
     protocols_subtypingPassPassPassPassPassPass
     protocols_variancePassPassPassPassUnsupportedPass
Callables
     callables_annotation
Partial

Incorrectly treats "*args: T, **kwargs: T" as "..." when T is specialized to Any.

Does not treat "*args: Any, **kargs: Any" as "..." when separated by keyword parameter.

Pass
Partial

Parameter names are lost when resolving ParamSpec

Pass
Partial

Infers a callback protocol as being a gradual type if the callback has signature `__call__[T](self, *args: T, **kwargs: T)` and `T` has been explicitly specialized to `Any`.

Pass
     callables_kwargs
Partial

Allows callable without kwargs to be assigned to callable with unpacked kwargs

PassPassPassPassPass
     callables_protocolPassPassPassPassPassPass
     callables_subtypingPassPassPassPassPassPass
+
     callables_annotation
Partial

Incorrectly treats "*args: T, **kwargs: T" as "..." when T is specialized to Any.

Does not treat "*args: Any, **kargs: Any" as "..." when separated by keyword parameter.

Pass
Partial

Parameter names are lost when resolving ParamSpec

Pass
Partial

Infers a callback protocol as being a gradual type if the callback has signature `__call__[T](self, *args: T, **kwargs: T)` and `T` has been explicitly specialized to `Any`.

Pass
     callables_kwargs
Partial

Allows callable without kwargs to be assigned to callable with unpacked kwargs

PassPassPassPassPass
     callables_protocolPassPassPassPassPassPass
     callables_subtypingPassPassPassPassPassPass
Constructors
     constructors_call_init
Partial

Does not report errors during binding to self parameter of __init__ method.

Does not reject use of class-scoped type variables in annotation of self parameter in __init__ method.

PassPassPass
Partial

Does not reject invalid argument types to an inherited constructor in a specialized subclass of a generic superclass.

Does not reject class-scoped type variables used in the `self` annotation.

Pass
     constructors_call_metaclass
Unsupported

Does not honor metaclass __call__ method when evaluating constructor call.

Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

PassPassPassPassPass
     constructors_call_new
Partial

Does not support __new__ return type that is not a subclass of the class being constructed.

Does not skip evaluation of __init__ based on __new__ return type.

Does not report errors during binding to cls parameter of __new__ method.

PassPassPassPassPass
     constructors_call_type
Partial

Does not validate call to custom metaclass __call__ method through type[T].

PassPassPass
Partial

Has overly lenient handling of calls to `type[T]` if `T` is a type variable without an upper bound.

Pass
     constructors_callable
Partial

Does not generate a union type for __new__ and __init__ when converting class to callable.

Does not ignore __init__ based on __new__ return type when converting class to callable.

Does not support __new__ return type that is different from class being constructed.

Pass
Partial

Converting constructor to callable does not preserve class-scoped type params.

Pass
Partial

Does not include `__init__` when `__new__` returns `Self`.

Does not ignore `__init__` when `__new__` returns `Any`.

Unions overload return types.

Pass
     constructors_consistency
Pass*

Does not report inconsistency between __new__ and __init__ (optional).

PassPassPassPassPass
+
     constructors_call_init
Partial

Does not report errors during binding to self parameter of __init__ method.

Does not reject use of class-scoped type variables in annotation of self parameter in __init__ method.

PassPassPass
Partial

Does not reject invalid argument types to an inherited constructor in a specialized subclass of a generic superclass.

Does not reject class-scoped type variables used in the `self` annotation.

Pass
     constructors_call_metaclass
Unsupported

Does not honor metaclass __call__ method when evaluating constructor call.

Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

PassPassPassPassPass
     constructors_call_new
Partial

Does not support __new__ return type that is not a subclass of the class being constructed.

Does not skip evaluation of __init__ based on __new__ return type.

Does not report errors during binding to cls parameter of __new__ method.

PassPassPassPassPass
     constructors_call_type
Partial

Does not validate call to custom metaclass __call__ method through type[T].

PassPassPass
Partial

Has overly lenient handling of calls to `type[T]` if `T` is a type variable without an upper bound.

Pass
     constructors_callable
Partial

Does not generate a union type for __new__ and __init__ when converting class to callable.

Does not ignore __init__ based on __new__ return type when converting class to callable.

Does not support __new__ return type that is different from class being constructed.

Pass
Partial

Converting constructor to callable does not preserve class-scoped type params.

Pass
Partial

Does not include `__init__` when `__new__` returns `Self`.

Does not ignore `__init__` when `__new__` returns `Any`.

Unions overload return types.

Pass
     constructors_consistency
Pass*

Does not report inconsistency between __new__ and __init__ (optional).

PassPassPassPassPass
Overloads
     overloads_basicPass
Partial

Does less literal promotion than the test asks for.

PassPassPassPass
     overloads_consistencyPassPassPassPassPassPass
     overloads_definitions
Partial

Allows @override to be on all overloads and implementation, instead of just implementation.

PassPassPassPassPass
     overloads_definitions_stub
Partial

Allows @override to appear in a stub file not on the first overload.

Unsupported

Does not support checking stubs.

PassPassPassPass
     overloads_evaluation
Partial

Does not expand boolean arguments to Literal[True] and Literal[False].

Does not expand enum arguments to literal variants.

Does not expand tuple arguments to possible combinations.

Does not evaluate Any in some cases where overload is ambiguous.

Evaluates Any in some cases where overload is not ambiguous.

PassPass
Partial

Does not evaluate Any in some cases where overload is ambiguous.

PassPass
+
     overloads_basicPass
Partial

Does less literal promotion than the test asks for.

PassPassPassPass
     overloads_consistencyPassPassPassPassPassPass
     overloads_definitions
Partial

Allows @override to be on all overloads and implementation, instead of just implementation.

PassPassPassPassPass
     overloads_definitions_stub
Partial

Allows @override to appear in a stub file not on the first overload.

Unsupported

Does not support checking stubs.

PassPassPassPass
     overloads_evaluation
Partial

Does not expand boolean arguments to Literal[True] and Literal[False].

Does not expand enum arguments to literal variants.

Does not expand tuple arguments to possible combinations.

Does not evaluate Any in some cases where overload is ambiguous.

Evaluates Any in some cases where overload is not ambiguous.

PassPass
Partial

Does not evaluate Any in some cases where overload is ambiguous.

PassPass
Exceptions
     exceptions_context_managersPassPass
Partial

Some error suppressing context managers are not detected

PassUnsupportedPass
+
     exceptions_context_managersPassPass
Partial

Some error suppressing context managers are not detected

PassUnsupportedPass
Dataclasses
     dataclasses_descriptors
Partial

Assumes descriptor behavior only when field is assigned in class body.

Does not correctly evaluate type of descriptor access.

Partial

Conformance suite is questionable; see https://github.com/python/typing/issues/2259

Partial

* Assumes descriptor behavior only when field is assigned in class body

* Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types

Pass
Partial

Only infers a descriptor `__get__` method as being called when a descriptor attribute is accessed on an instance if the descriptor attribute is present in the class namespace.

Pass
     dataclasses_final
Partial

Wrongly requires a Final dataclass field to be initialized at class level.

Doesn't support Final nested inside ClassVar.

PassPassPassPassPass
     dataclasses_frozenPassPassPassPassPassPass
     dataclasses_hash
Unsupported

Does not synthesize `__hash__ = None` as a class attribute for unhashable dataclasses.

Does not report when an unhashable dataclass has `__hash__` called directly on an instance.

Does not report when dataclass is not compatible with Hashable protocol.

PassPassPass
Partial

Understands the `Hashable` protocol as equivalent to `object`.

Pass
     dataclasses_inheritancePassPassPassPassPassPass
     dataclasses_kwonlyPassPassPassPassPassPass
     dataclasses_match_argsPassPassPassPassPassPass
     dataclasses_orderPassPassPassPassPassPass
     dataclasses_postinitPassPassPassPassPassPass
     dataclasses_slots
Partial

Does not reject write to instance variable that is not defined in __slots__.

PassPassPass
Partial

Synthesizes a `__slots__` attribute but does not validate attribute assignments against `__slots__`.

Pass
     dataclasses_transform_classPassPassPassPassPassPass
     dataclasses_transform_converter
Unsupported

Converter parameter not yet supported.

PassPassPassPassPass
     dataclasses_transform_field
Partial

Does not properly handle field constructor that has default value for `kw_only` or `init` parameter.

PassPassPassPassPass
     dataclasses_transform_func
Partial

Does not handle `kw_only=False` override when `kw_only_default=True`.

Does not report error when `order=False` and comparison operators are used.

PassPassPassPassPass
     dataclasses_transform_metaPassPassPassPassPassPass
     dataclasses_usage
Pass*

Does not detect unannotated usage of `dataclasses.field()`.

PassPassPassPassPass
+
     dataclasses_descriptors
Partial

Assumes descriptor behavior only when field is assigned in class body.

Does not correctly evaluate type of descriptor access.

Partial

Conformance suite is questionable; see https://github.com/python/typing/issues/2259

Partial

* Assumes descriptor behavior only when field is assigned in class body

* Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types

Pass
Partial

Only infers a descriptor `__get__` method as being called when a descriptor attribute is accessed on an instance if the descriptor attribute is present in the class namespace.

Pass
     dataclasses_final
Partial

Wrongly requires a Final dataclass field to be initialized at class level.

Doesn't support Final nested inside ClassVar.

PassPassPassPassPass
     dataclasses_frozenPassPassPassPassPassPass
     dataclasses_hash
Unsupported

Does not synthesize `__hash__ = None` as a class attribute for unhashable dataclasses.

Does not report when an unhashable dataclass has `__hash__` called directly on an instance.

Does not report when dataclass is not compatible with Hashable protocol.

PassPassPass
Partial

Understands the `Hashable` protocol as equivalent to `object`.

Pass
     dataclasses_inheritancePassPassPassPassPassPass
     dataclasses_kwonlyPassPassPassPassPassPass
     dataclasses_match_argsPassPassPassPassPassPass
     dataclasses_orderPassPassPassPassPassPass
     dataclasses_postinitPassPassPassPassPassPass
     dataclasses_slots
Partial

Does not reject write to instance variable that is not defined in __slots__.

PassPassPass
Partial

Synthesizes a `__slots__` attribute but does not validate attribute assignments against `__slots__`.

Pass
     dataclasses_transform_classPassPassPassPassPassPass
     dataclasses_transform_converter
Unsupported

Converter parameter not yet supported.

PassPassPassPassPass
     dataclasses_transform_field
Partial

Does not properly handle field constructor that has default value for `kw_only` or `init` parameter.

PassPassPassPassPass
     dataclasses_transform_func
Partial

Does not handle `kw_only=False` override when `kw_only_default=True`.

Does not report error when `order=False` and comparison operators are used.

PassPassPassPassPass
     dataclasses_transform_metaPassPassPassPassPassPass
     dataclasses_usage
Pass*

Does not detect unannotated usage of `dataclasses.field()`.

PassPassPassPassPass
Typed dictionaries
     typeddicts_alt_syntax
Pass*

Does not support keyword-argument form of alternative syntax (deprecated in 3.11).

PassPassPassPassPass
     typeddicts_class_syntax
Partial

Does not support version-conditional items in TypedDict definitions.

PassPass
Partial

Does not support version-conditional items in TypedDict definitions.

PassPass
     typeddicts_extra_items
Unsupported

Not supported.

PassPassPassPassPass
     typeddicts_finalPassPassPassPassPassPass
     typeddicts_inheritancePassPassPassPassPassPass
     typeddicts_operationsPassPassPassPassPassPass
     typeddicts_readonlyPassPassPassPassPassPass
     typeddicts_readonly_consistencyPassPassPassPassPassPass
     typeddicts_readonly_inheritance
Partial

Incorrectly rejects non-ReadOnly override of ReadOnly item.

Incorrectly rejects override of ReadOnly item with another ReadOnly item with narrower type.

Incorrectly rejects override of NotRequired ReadOnly item with a Required ReadOnly item.

PassPassPassPassPass
     typeddicts_readonly_kwargsPassPassPassPassPassPass
     typeddicts_readonly_update
Partial

Incorrectly allows update of ReadOnly item.

Incorrectly rejects update involving an item with Never type.

PassPassPassPassPass
     typeddicts_requiredPassPassPassPassPassPass
     typeddicts_type_consistencyPass
Partial

Considers TypedDicts to be assignable to plain dict types.

PassPassPassPass
     typeddicts_usagePassPassPassPassPassPass
+
     typeddicts_alt_syntax
Pass*

Does not support keyword-argument form of alternative syntax (deprecated in 3.11).

PassPassPassPassPass
     typeddicts_class_syntax
Partial

Does not support version-conditional items in TypedDict definitions.

PassPass
Partial

Does not support version-conditional items in TypedDict definitions.

PassPass
     typeddicts_extra_items
Unsupported

Not supported.

PassPassPassPassPass
     typeddicts_finalPassPassPassPassPassPass
     typeddicts_inheritancePassPassPassPassPassPass
     typeddicts_operationsPassPassPassPassPassPass
     typeddicts_readonlyPassPassPassPassPassPass
     typeddicts_readonly_consistencyPassPassPassPassPassPass
     typeddicts_readonly_inheritance
Partial

Incorrectly rejects non-ReadOnly override of ReadOnly item.

Incorrectly rejects override of ReadOnly item with another ReadOnly item with narrower type.

Incorrectly rejects override of NotRequired ReadOnly item with a Required ReadOnly item.

PassPassPassPassPass
     typeddicts_readonly_kwargsPassPassPassPassPassPass
     typeddicts_readonly_update
Partial

Incorrectly allows update of ReadOnly item.

Incorrectly rejects update involving an item with Never type.

PassPassPassPassPass
     typeddicts_requiredPassPassPassPassPassPass
     typeddicts_type_consistencyPass
Partial

Considers TypedDicts to be assignable to plain dict types.

PassPassPassPass
     typeddicts_usagePassPassPassPassPassPass
Tuples
     tuples_type_compat
Partial

Does not support tuple narrowing based on `len()` type guard (optional).

Incorrectly marks a match case as unreachable.

PassPassPassPassPass
     tuples_type_formPassPassPassPassPassPass
     tuples_unpacked
Partial

"More than one unpack" error is missing in some cases.

PassPassPassPassPass
+
     tuples_type_compat
Partial

Does not support tuple narrowing based on `len()` type guard (optional).

Incorrectly marks a match case as unreachable.

PassPassPassPassPass
     tuples_type_formPassPassPassPassPassPass
     tuples_unpacked
Partial

"More than one unpack" error is missing in some cases.

PassPassPassPassPass
Named tuples
     namedtuples_define_class
Partial

Does not reject override of named tuple attribute in child class.

Does not support version-conditional fields.

Partial

Does not support precise type inference for slices over namedtuples.

PassPassPassPass
     namedtuples_define_functionalPassPassPassPassPassPass
     namedtuples_type_compatPassPassPassPassPassPass
     namedtuples_usage
Partial

Does not reject attempt to delete named tuple field by name.

PassPassPassPassPass
+
     namedtuples_define_class
Partial

Does not reject override of named tuple attribute in child class.

Does not support version-conditional fields.

Partial

Does not support precise type inference for slices over namedtuples.

PassPassPassPass
     namedtuples_define_functionalPassPassPassPassPassPass
     namedtuples_type_compatPassPassPassPassPassPass
     namedtuples_usage
Partial

Does not reject attempt to delete named tuple field by name.

PassPassPassPassPass
Enumerations
     enums_behaviorsPassPassPassPassPassPass
     enums_definitionPass
Partial

Does not allow enum members to be conditional on version/platform checks.

PassPassPassPass
     enums_expansion
Partial

Improperly applies narrowing to Flag subclass.

PassPassPass
Partial

Does not support `enum.Flag`.

Pass
     enums_member_names
Pass*

Does not support special-cased handling of member name literal types in some cases (optional).

PassPassPassPassPass
     enums_member_values
Partial

Does not enforce declared type of `_value_`.

Does not enforce assigned tuple types for enum members (optional).

PassPassPassPassPass
     enums_members
Partial

Does not treat attribute with annotation and no assignment as non-member.

Does not treat callables as non-members.

Does not honor `enum.member` as method decorator.

Does not properly handle aliased enum members.

Does not support `_ignore_` mechanism (optional).

Does not treat attributes with private names as non-members.

PassPass
Pass*

Does not support `_ignore_` mechanism (optional).

PassPass
+
     enums_behaviorsPassPassPassPassPassPass
     enums_definitionPass
Partial

Does not allow enum members to be conditional on version/platform checks.

PassPassPassPass
     enums_expansion
Partial

Improperly applies narrowing to Flag subclass.

PassPassPass
Partial

Does not support `enum.Flag`.

Pass
     enums_member_names
Pass*

Does not support special-cased handling of member name literal types in some cases (optional).

PassPassPassPassPass
     enums_member_values
Partial

Does not enforce declared type of `_value_`.

Does not enforce assigned tuple types for enum members (optional).

PassPassPassPassPass
     enums_members
Partial

Does not treat attribute with annotation and no assignment as non-member.

Does not treat callables as non-members.

Does not honor `enum.member` as method decorator.

Does not properly handle aliased enum members.

Does not support `_ignore_` mechanism (optional).

Does not treat attributes with private names as non-members.

PassPass
Pass*

Does not support `_ignore_` mechanism (optional).

PassPass
Type narrowing
     narrowing_typeguardPassPassPassPassPassPass
     narrowing_typeisPassPassPassPass
Partial

Intersects the pre-existing type with the top materialization of the bracketed type rather than the bracketed type itself.

Pass
+
     narrowing_typeguardPassPassPassPassPassPass
     narrowing_typeisPassPassPassPass
Partial

Intersects the pre-existing type with the top materialization of the bracketed type rather than the bracketed type itself.

Pass
Type checker directives
     directives_assert_typePassPassPassPassPassPass
     directives_castPassPassPassPassPassPass
     directives_deprecatedPassPassPassPass
Partial

Does not detect calls to deprecated overloads.

Does not detect implicit calls to deprecated dunder methods, for example via operators.

Does not detect accesses of, or attempts to set, deprecated properties.

Pass
     directives_disjoint_basePass
Partial

Does not reject invalid class definitions due to disjoint bases, but uses disjoint base information in type narrowing.

Pass
Unsupported

Does not support PEP 800 disjoint-base semantics.

PassPass
     directives_no_type_check
Partial

Does not honor `@no_type_check` class decorator (allowed).

Does not reject invalid call of `@no_type_check` function.

PassPass
Pass*

Does not honor `@no_type_check` class decorator (allowed).

PassPass
     directives_reveal_typePassPassPassPassPassPass
     directives_type_checkingPassPassPassPassPassPass
     directives_type_ignore
Partial

Does not honor "# type: ignore" comment if comment includes additional text.

PassPassPassPassPass
     directives_type_ignore_file1PassPassPassPassPassPass
     directives_type_ignore_file2PassPassPassPassPassPass
     directives_version_platform
Pass*

Does not understand three-element form of sys.version checks.

Does not understand os.name checks.

PassPassPassPassPass
+
     directives_assert_typePassPassPassPassPassPass
     directives_castPassPassPassPassPassPass
     directives_deprecatedPassPassPassPass
Partial

Does not detect calls to deprecated overloads.

Does not detect implicit calls to deprecated dunder methods, for example via operators.

Does not detect accesses of, or attempts to set, deprecated properties.

Pass
     directives_disjoint_basePass
Partial

Does not reject invalid class definitions due to disjoint bases, but uses disjoint base information in type narrowing.

Pass
Unsupported

Does not support PEP 800 disjoint-base semantics.

PassPass
     directives_no_type_check
Partial

Does not honor `@no_type_check` class decorator (allowed).

Does not reject invalid call of `@no_type_check` function.

PassPass
Pass*

Does not honor `@no_type_check` class decorator (allowed).

PassPass
     directives_reveal_typePassPassPassPassPassPass
     directives_type_checkingPassPassPassPassPassPass
     directives_type_ignore
Partial

Does not honor "# type: ignore" comment if comment includes additional text.

PassPassPassPassPass
     directives_type_ignore_file1PassPassPassPassPassPass
     directives_type_ignore_file2PassPassPassPassPassPass
     directives_version_platform
Pass*

Does not understand three-element form of sys.version checks.

Does not understand os.name checks.

PassPassPassPassPass
Historical and deprecated features
     historical_positional
Partial

Does not reject positional-only parameter after non-positional-only parameter.

Treats keyword-only parameter as positional-only.

Applies legacy positional-only rules when PEP 570 syntax is used.

PassPassPassPassPass
     historical_positional
Partial

Does not reject positional-only parameter after non-positional-only parameter.

Treats keyword-only parameter as positional-only.

Applies legacy positional-only rules when PEP 570 syntax is used.

PassPassPassPassPass
diff --git a/conformance/src/reporting.py b/conformance/src/reporting.py index 4424e24a3..149527f20 100644 --- a/conformance/src/reporting.py +++ b/conformance/src/reporting.py @@ -60,7 +60,7 @@ def generate_summary_html(root_dir: Path) -> str: # Are there any test cases in this group? if len(tests_in_group) > 0: - summary_html.append(f'') + summary_html.append(f'') summary_html.append( f'{test_group.name}' ) @@ -69,7 +69,7 @@ def generate_summary_html(root_dir: Path) -> str: for test_case in tests_in_group: test_case_name = test_case.stem - summary_html.append(f'     {test_case_name}') + summary_html.append(f'     {test_case_name}') for type_checker in type_checkers: try: @@ -112,7 +112,7 @@ def generate_summary_html(root_dir: Path) -> str: if raw_notes != "": conformance_cell = f'
{conformance_cell}
{notes}
' - summary_html.append(f'{conformance_cell}') + summary_html.append(f'{conformance_cell}') summary_html.append("") diff --git a/conformance/src/results_template.html b/conformance/src/results_template.html index 4cfd4e666..df70a0ca7 100644 --- a/conformance/src/results_template.html +++ b/conformance/src/results_template.html @@ -52,7 +52,8 @@ margin-left: 20px; } - .column { + tbody > tr > th, + tbody > tr > td { padding-left: 8px; padding-right: 8px; padding-top: 4px; From adc3991626857521b1f74e6606ad9ce12fa68672 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:31 +0100 Subject: [PATCH 11/30] Use a more appropriate cursor, and only if tooltip --- conformance/results/results.html | 2 +- conformance/src/results_template.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conformance/results/results.html b/conformance/results/results.html index 2bed4e459..1b0fd9016 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -79,7 +79,6 @@ .col2 { width: 14%; - cursor: pointer; text-align: center; } @@ -143,6 +142,7 @@ } .hover-text { + cursor: help; position: relative; } diff --git a/conformance/src/results_template.html b/conformance/src/results_template.html index df70a0ca7..fed258c2b 100644 --- a/conformance/src/results_template.html +++ b/conformance/src/results_template.html @@ -79,7 +79,6 @@ .col2 { width: 14%; - cursor: pointer; text-align: center; } @@ -143,6 +142,7 @@ } .hover-text { + cursor: help; position: relative; } From 9b9862f107cee0eee352998a62a50c9e099a03cf Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:31 +0100 Subject: [PATCH 12/30] Make use of a `` --- conformance/results/results.html | 2232 +++++++++++++------------ conformance/src/reporting.py | 14 +- conformance/src/results_template.html | 12 +- 3 files changed, 1143 insertions(+), 1115 deletions(-) diff --git a/conformance/results/results.html b/conformance/results/results.html index 1b0fd9016..37d666579 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -44,6 +44,16 @@ font-weight: normal; } + th:first-child, + td:first-child { + vertical-align: top; + } + + th:not(:first-child), + td:not(:first-child){ + text-align: center; + } + .content_container { margin: 20px; } @@ -74,12 +84,10 @@ .col1 { width: 28%; - vertical-align: top; } .col2 { width: 14%; - text-align: center; } thead > tr > th { @@ -169,7 +177,11 @@

Python Type System Conformance Test Results

- + +++ + @@ -184,1193 +196,1193 @@

Python Type System Conformance Test Results

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - + + + + + + +
mypy 2.1.0
Type annotations
     annotations_coroutinesPassPassPassPassPassPass
     annotations_forward_refs
Partial

Does not report error for a forward reference that is not enclosed in quotes.

Does not report error for use of quoted type with "|" operator (runtime error).

Incorrectly generates error for quoted type defined in class scope.

Partial

Fails to reject "x" | int annotations that fail at runtime.

Rejects some valid quoted annotations.

Partial

Types in quotes incorrectly refer to shadowing class member.

Does not reject some type forms that require quotes.

Pass
Partial

Resolves references in type annotations as referring to end-of-scope types (https://discuss.python.org/t/annotation-string-references-in-class-scope-in-conformance-tests/105439, https://github.com/python/typing/pull/2144)

Partial

Incorrectly generates error for quoted type defined in class scope.

     annotations_generators
Partial

Does not report incompatible Generator type in `yield from` statement.

PassPassPassPassPass
     annotations_methods
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

PassPass
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

PassPass
     annotations_typeexprPass
Partial

Fails to reject various weird annotations

PassPassPassPass
     annotations_coroutinesPassPassPassPassPassPass
     annotations_forward_refs
Partial

Does not report error for a forward reference that is not enclosed in quotes.

Does not report error for use of quoted type with "|" operator (runtime error).

Incorrectly generates error for quoted type defined in class scope.

Partial

Fails to reject "x" | int annotations that fail at runtime.

Rejects some valid quoted annotations.

Partial

Types in quotes incorrectly refer to shadowing class member.

Does not reject some type forms that require quotes.

Pass
Partial

Resolves references in type annotations as referring to end-of-scope types (https://discuss.python.org/t/annotation-string-references-in-class-scope-in-conformance-tests/105439, https://github.com/python/typing/pull/2144)

Partial

Incorrectly generates error for quoted type defined in class scope.

     annotations_generators
Partial

Does not report incompatible Generator type in `yield from` statement.

PassPassPassPassPass
     annotations_methods
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

PassPass
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

PassPass
     annotations_typeexprPass
Partial

Fails to reject various weird annotations

PassPassPassPass
Type forms
     typeforms_typeform
Partial

Does not support assigning Union and GenericAlias objects to their runtime types.

Partial

Fails to reject various weird annotations

PassUnsupportedPassPass
     typeforms_typeform
Partial

Does not support assigning Union and GenericAlias objects to their runtime types.

Partial

Fails to reject various weird annotations

PassUnsupportedPassPass
Special types in annotations
     specialtypes_anyPassPassPassPassPassPass
     specialtypes_neverPass
Partial

Does not enforce invariance in some contexts

Does not allow Any to be assigned to Never

PassPassPassPass
     specialtypes_nonePassPassPassPassPassPass
     specialtypes_promotionsPassPassPassPassPassPass
     specialtypes_type
Partial

Does not treat `type` same as `type[Any]` for assert_type.

Does not allow access to unknown attributes from object of type `type[Any]`.

PassPassPass
Partial

Allows arbitrary attributes to be accessed on `TA` where `TA = typing.Type[typing.Any]` or `TA = type[typing.Any]`.

Treats `type` equivalently to `type[object]` rather than `type[typing.Any]`.

Pass
     specialtypes_anyPassPassPassPassPassPass
     specialtypes_neverPass
Partial

Does not enforce invariance in some contexts

Does not allow Any to be assigned to Never

PassPassPassPass
     specialtypes_nonePassPassPassPassPassPass
     specialtypes_promotionsPassPassPassPassPassPass
     specialtypes_type
Partial

Does not treat `type` same as `type[Any]` for assert_type.

Does not allow access to unknown attributes from object of type `type[Any]`.

PassPassPass
Partial

Allows arbitrary attributes to be accessed on `TA` where `TA = typing.Type[typing.Any]` or `TA = type[typing.Any]`.

Treats `type` equivalently to `type[object]` rather than `type[typing.Any]`.

Pass
Generics
     generics_base_class
Partial

Does not detect inconsistent type variable ordering.

PassPassPassPassPass
     generics_basicPassPassPassPass
Partial

Incorrectly allows constrained type variables to be solved to a union of their constraints.

Pass
     generics_defaults
Partial

Does not detect a TypeVar with a default used after a TypeVarTuple.

Does not fully support defaults on TypeVarTuple and ParamSpec.

Partial

Numerous issues; does not support TypeVarTuple and ParamSpec defaults.

PassPass
Partial

Does not forbid a `TypeVar` immediately following a `TypeVarTuple` in a parameter list from having a default.

Does not support `TypeVarTuple`.

Pass
     generics_defaults_referential
Partial

Does not correctly handle defaults referencing other TypeVars.

Partial

Fails to apply default specializations in some cases.

PassPassPassPass
     generics_defaults_specialization
Partial

Does not correctly resolve defaults when classes are used directly.

PassPass
Partial

Allows incorrect assignment to type[].

Partial

Does not reject subscription of an already-specialized generic class.

Pass
     generics_paramspec_basicPassPassPassPassPassPass
     generics_paramspec_componentsPassPassPassPass
Partial

Incorrectly allows using `*args: P.args` and `**kwargs: P.kwargs` when `P` has not been put into scope by any other parameter annotation or enclosing scope.

Pass
     generics_paramspec_semanticsPassPassPass
Pass*

Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).

PassPass
     generics_paramspec_specializationPassPassPassPassPassPass
     generics_scopingPassPass
Partial

Does not implement several scoping checks/restrictions for generics

Pass
Partial

Does not reject `list[T]()` in the global scope, where `T` is an unbound type variable.

Does nto reject `alias: TypeAlias = list[T]` in the body scope of a class generic over a type variable `T`.

Pass
     generics_self_advanced
Partial

Does not infer the type of an unannotated `self` parameter to be type `Self`.

Does not retain `Self` when calling method that returns `Self`.

Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`.

Does not retain `Self` when accessing attribute through `type[Self]`.

Pass
Pass*

Treats attributes not initialized on the class as instance-only

PassPassPass
     generics_self_attributesPassPassPassPassPassPass
     generics_self_basicPassPassPassPassPassPass
     generics_self_protocolsPassPassPassPassPassPass
     generics_self_usagePassPassPassPass
Partial

Does not reject `Self` used in a return annotation when `self` is annotated using another type variable.

Pass
     generics_syntax_compatibilityPassPassPassPassPassPass
     generics_syntax_declarationsPassPassPassPassPassPass
     generics_syntax_infer_variance
Unsupported

Type parameter syntax not yet supported.

PassPassPassPassPass
     generics_syntax_scoping
Partial

Does not following runtime scoping rules for type parameters in all cases.

Partial

Misses some details of scoping rules.

PassPassPassPass
     generics_type_erasure
Partial

Infers Node[Never] instead of Node[Any] when argument is not provided.

False negative on instance attribute access on type(node).

PassPassPass
Partial

Does not reject access of generic instance variable from the class object.

Pass
     generics_typevartuple_args
Partial

Does not enforce that tuples captured by TypeVarTuple are of the same length.

PassPass
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

Partial

Supports PEP-646 unpacked tuples but not TypeVarTuple.

Pass
     generics_typevartuple_basic
Partial

Does not enforce that tuples captured by TypeVarTuple are same length.

PassPass
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

UnsupportedPass
     generics_typevartuple_callablePassPassPassPassUnsupportedPass
     generics_typevartuple_concatPassPassPassPassUnsupportedPass
     generics_typevartuple_overloadsPassPassPassPassPassPass
     generics_typevartuple_specialization
Partial

Incorrectly specializes generic alias that includes a TypeVar and TypeVarTuple if no type arguments are provided.

Rejects specialization of generic type alias defined as a tuple containing a TypeVar.

PassPassPassUnsupportedPass
     generics_typevartuple_unpackPassPassPassPassUnsupportedPass
     generics_upper_bound
Partial

Does not reject use of type variable within an upper bound.

PassPassPassPassPass
     generics_variance
Partial

Does not reject use of class-scoped TypeVar used in a base class when variance is incompatible.

PassPassPassPassPass
     generics_variance_inferencePassPassPassPassPassPass
     generics_base_class
Partial

Does not detect inconsistent type variable ordering.

PassPassPassPassPass
     generics_basicPassPassPassPass
Partial

Incorrectly allows constrained type variables to be solved to a union of their constraints.

Pass
     generics_defaults
Partial

Does not detect a TypeVar with a default used after a TypeVarTuple.

Does not fully support defaults on TypeVarTuple and ParamSpec.

Partial

Numerous issues; does not support TypeVarTuple and ParamSpec defaults.

PassPass
Partial

Does not forbid a `TypeVar` immediately following a `TypeVarTuple` in a parameter list from having a default.

Does not support `TypeVarTuple`.

Pass
     generics_defaults_referential
Partial

Does not correctly handle defaults referencing other TypeVars.

Partial

Fails to apply default specializations in some cases.

PassPassPassPass
     generics_defaults_specialization
Partial

Does not correctly resolve defaults when classes are used directly.

PassPass
Partial

Allows incorrect assignment to type[].

Partial

Does not reject subscription of an already-specialized generic class.

Pass
     generics_paramspec_basicPassPassPassPassPassPass
     generics_paramspec_componentsPassPassPassPass
Partial

Incorrectly allows using `*args: P.args` and `**kwargs: P.kwargs` when `P` has not been put into scope by any other parameter annotation or enclosing scope.

Pass
     generics_paramspec_semanticsPassPassPass
Pass*

Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).

PassPass
     generics_paramspec_specializationPassPassPassPassPassPass
     generics_scopingPassPass
Partial

Does not implement several scoping checks/restrictions for generics

Pass
Partial

Does not reject `list[T]()` in the global scope, where `T` is an unbound type variable.

Does nto reject `alias: TypeAlias = list[T]` in the body scope of a class generic over a type variable `T`.

Pass
     generics_self_advanced
Partial

Does not infer the type of an unannotated `self` parameter to be type `Self`.

Does not retain `Self` when calling method that returns `Self`.

Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`.

Does not retain `Self` when accessing attribute through `type[Self]`.

Pass
Pass*

Treats attributes not initialized on the class as instance-only

PassPassPass
     generics_self_attributesPassPassPassPassPassPass
     generics_self_basicPassPassPassPassPassPass
     generics_self_protocolsPassPassPassPassPassPass
     generics_self_usagePassPassPassPass
Partial

Does not reject `Self` used in a return annotation when `self` is annotated using another type variable.

Pass
     generics_syntax_compatibilityPassPassPassPassPassPass
     generics_syntax_declarationsPassPassPassPassPassPass
     generics_syntax_infer_variance
Unsupported

Type parameter syntax not yet supported.

PassPassPassPassPass
     generics_syntax_scoping
Partial

Does not following runtime scoping rules for type parameters in all cases.

Partial

Misses some details of scoping rules.

PassPassPassPass
     generics_type_erasure
Partial

Infers Node[Never] instead of Node[Any] when argument is not provided.

False negative on instance attribute access on type(node).

PassPassPass
Partial

Does not reject access of generic instance variable from the class object.

Pass
     generics_typevartuple_args
Partial

Does not enforce that tuples captured by TypeVarTuple are of the same length.

PassPass
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

Partial

Supports PEP-646 unpacked tuples but not TypeVarTuple.

Pass
     generics_typevartuple_basic
Partial

Does not enforce that tuples captured by TypeVarTuple are same length.

PassPass
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

UnsupportedPass
     generics_typevartuple_callablePassPassPassPassUnsupportedPass
     generics_typevartuple_concatPassPassPassPassUnsupportedPass
     generics_typevartuple_overloadsPassPassPassPassPassPass
     generics_typevartuple_specialization
Partial

Incorrectly specializes generic alias that includes a TypeVar and TypeVarTuple if no type arguments are provided.

Rejects specialization of generic type alias defined as a tuple containing a TypeVar.

PassPassPassUnsupportedPass
     generics_typevartuple_unpackPassPassPassPassUnsupportedPass
     generics_upper_bound
Partial

Does not reject use of type variable within an upper bound.

PassPassPassPassPass
     generics_variance
Partial

Does not reject use of class-scoped TypeVar used in a base class when variance is incompatible.

PassPassPassPassPass
     generics_variance_inferencePassPassPassPassPassPass
Type qualifiers
     qualifiers_annotated
Partial

Does not allow ClassVar to be nested within Annotated.

Does not allow Final to be nested within Annotated.

Does not allow Required and NotRequired to be nested within Annotated.

Does not reject type[T] compatibility for type alias defined with Annotated.

Does not reject call of type alias defined with Annotated.

Partial

Fails to reject various weird annotations.

False positive on lambda in Annotated.

PassPassPassPass
     qualifiers_final_annotation
Partial

Does not treat use of Final name as if it was replaced by the literal in NamedTuple definition.

Does not allow conditional assignment of Final instance variable in __init__ method.

Does not allow redefinition of private class variable that is marked Final in parent class.

Does not report modification of local Final variable via "for" statement.

PassPassPassPassPass
     qualifiers_final_decoratorPassPassPassPassPassPass
     qualifiers_annotated
Partial

Does not allow ClassVar to be nested within Annotated.

Does not allow Final to be nested within Annotated.

Does not allow Required and NotRequired to be nested within Annotated.

Does not reject type[T] compatibility for type alias defined with Annotated.

Does not reject call of type alias defined with Annotated.

Partial

Fails to reject various weird annotations.

False positive on lambda in Annotated.

PassPassPassPass
     qualifiers_final_annotation
Partial

Does not treat use of Final name as if it was replaced by the literal in NamedTuple definition.

Does not allow conditional assignment of Final instance variable in __init__ method.

Does not allow redefinition of private class variable that is marked Final in parent class.

Does not report modification of local Final variable via "for" statement.

PassPassPassPassPass
     qualifiers_final_decoratorPassPassPassPassPassPass
Class type compatibility
     classes_classvar
Partial

Internal error if TypeVarTuple is used in ClassVar.

Does not reject use of ParamSpec in ClassVar.

Rejects ClassVar nested in Annotated.

Does not reject use of ClassVar in TypeAlias definition.

PassPassPassPassPass
     classes_overridePassPassPassPassPassPass
     classes_classvar
Partial

Internal error if TypeVarTuple is used in ClassVar.

Does not reject use of ParamSpec in ClassVar.

Rejects ClassVar nested in Annotated.

Does not reject use of ClassVar in TypeAlias definition.

PassPassPassPassPass
     classes_overridePassPassPassPassPassPass
Type aliases
     aliases_explicit
Partial

Does not reject specialization of type alias that has already been implicitly specialized.

PassPassPass
Partial

Does not emit a diagnostic if a type alias to a union, in which every element of the union is implicitly specialized with `Unknown`, is invalidly specialized again.

Pass
     aliases_implicitPass
Partial

Fails to handle various weird annotations.

Various bugs with resolving generic aliases.

PassPass
Partial

Does not reject variables with `Any` or `Unknown` types when used as implicit type aliases.

Falls short on full syntactic validation of type aliases.

Pass
     aliases_newtype
Partial

`NewType`s are incorrectly considered to be classes.

PassPassPassPassPass
     aliases_recursivePassPassPassPassUnsupportedPass
     aliases_type_statementPass
Partial

Fails to reject various weird annotations.

PassPass
Partial

Does not reject circular definitions of type aliases.

Does not support `type` statements generic over `TypeVarTuple`s.

Pass
     aliases_typealiastype
Partial

Incorrectly rejects some recursive type aliases using TypeAliasType.

Incorrectly rejects the use of a class-scoped TypeVar in a TypeAliasType definition.

Partial

Rejects valid ParamSpec specialization.

PassPass
Partial

Does not reject specializing a type parameter in a generic type alias with a type inconsistent with the parameter's upper bound.

Does not reject declaring a type alias with a type variable that is not in scope.

Does not reject declaring a type alias with a non-literal tuple passed to the `type_params` parameter.

Does not reject cyclically defined type aliases.

Pass
     aliases_variancePassPassPassPassPassPass
     aliases_explicit
Partial

Does not reject specialization of type alias that has already been implicitly specialized.

PassPassPass
Partial

Does not emit a diagnostic if a type alias to a union, in which every element of the union is implicitly specialized with `Unknown`, is invalidly specialized again.

Pass
     aliases_implicitPass
Partial

Fails to handle various weird annotations.

Various bugs with resolving generic aliases.

PassPass
Partial

Does not reject variables with `Any` or `Unknown` types when used as implicit type aliases.

Falls short on full syntactic validation of type aliases.

Pass
     aliases_newtype
Partial

`NewType`s are incorrectly considered to be classes.

PassPassPassPassPass
     aliases_recursivePassPassPassPassUnsupportedPass
     aliases_type_statementPass
Partial

Fails to reject various weird annotations.

PassPass
Partial

Does not reject circular definitions of type aliases.

Does not support `type` statements generic over `TypeVarTuple`s.

Pass
     aliases_typealiastype
Partial

Incorrectly rejects some recursive type aliases using TypeAliasType.

Incorrectly rejects the use of a class-scoped TypeVar in a TypeAliasType definition.

Partial

Rejects valid ParamSpec specialization.

PassPass
Partial

Does not reject specializing a type parameter in a generic type alias with a type inconsistent with the parameter's upper bound.

Does not reject declaring a type alias with a type variable that is not in scope.

Does not reject declaring a type alias with a non-literal tuple passed to the `type_params` parameter.

Does not reject cyclically defined type aliases.

Pass
     aliases_variancePassPassPassPassPassPass
Literals
     literals_interactions
Pass*

Does not narrow `str` or `LiteralString` types to `Literal` string types via equality or containment checks.

PassPassPass
Pass*

Deliberately does not allow `str` to be narrowed to literal string types through equality or containment checks due to the possibility of `str` subclasses that could have unexpected equality semantics.

Pass
     literals_literalstring
Unsupported

Support for `LiteralString` is not implemented.

PassPassPassPassPass
     literals_parameterizations
Partial

Does not reject tuple within Literal.

Partial

Fails to reject various invalid literal parameterizations.

PassPassPassPass
     literals_semanticsPassPassPassPassPassPass
     literals_interactions
Pass*

Does not narrow `str` or `LiteralString` types to `Literal` string types via equality or containment checks.

PassPassPass
Pass*

Deliberately does not allow `str` to be narrowed to literal string types through equality or containment checks due to the possibility of `str` subclasses that could have unexpected equality semantics.

Pass
     literals_literalstring
Unsupported

Support for `LiteralString` is not implemented.

PassPassPassPassPass
     literals_parameterizations
Partial

Does not reject tuple within Literal.

Partial

Fails to reject various invalid literal parameterizations.

PassPassPassPass
     literals_semanticsPassPassPassPassPassPass
Protocols
     protocols_class_objectsPass
Partial

Abstract type[Proto] still allows protocol class objects in some paths.

PassPass
Unsupported

`type[Proto]` is not yet supported.

`ClassVar` protocol members are not yet supported.

`@property` protocol members are only partially supported.

A class object `C` is only considered to inhabit a protocol type with a method member `f` if `f` exists as an attribute on the metaclass of `C`.

Pass
     protocols_definition
Partial

Does not detect protocol mismatch if concrete method is missing annotations.

Does not detect protocol mismatch if concrete method's parameters are position-only.

PassPassPass
Partial

Does not reject implicit instance attributes in `Protocol` methods.

Does not support `ClassVar` protocol members.

Incorrectly considers `ClassVar` attributes on concrete classes as satisfying non-`ClassVar` attribute members on protocols.

Only has partial support for `@property` protocol members.

Pass
     protocols_explicit
Pass*

Does not report unimplemented attributes for class that explicitly derives from protocol until it is instantiated.

PassPassPass
Unsupported

Allows implicitly abstract protocol methods to be called via `super()` on a protocol subclass.

Allows instantiation of abstract subclasses of protocol classes.

Pass
     protocols_genericPass
Partial

Fails to reject duplicate generic/protocol bases.

Treats global object as a literal.

PassPass
Partial

Only partially supports `@property` protocol members.

Pass
     protocols_mergingPassPassPassPass
Partial

Does not reject attempted instantiation of abstract subclasses of protocols.

Pass
     protocols_modulesPassPassPassPass
Partial

Never considers a module as satisfying a protocol with a method member due to the fact that the method will never exist on the class `types.ModuleType`, only on a given specific module instance.

Pass
     protocols_recursivePassPassPassPass
Partial

Fails to solve a type variable involving a recursive generic protocol.

Pass
     protocols_runtime_checkable
Partial

Does not report unsafe overlap for runtime_checkable protocol.

PassPassPass
Partial

Does not reject `isinstance()` or `issubclass()` calls against runtime-checkable protocols where there is an unsafe overlap between the type of the first argument and the protocol.

Pass
     protocols_selfPassPassPassPassPassPass
     protocols_subtypingPassPassPassPassPassPass
     protocols_variancePassPassPassPassUnsupportedPass
     protocols_class_objectsPass
Partial

Abstract type[Proto] still allows protocol class objects in some paths.

PassPass
Unsupported

`type[Proto]` is not yet supported.

`ClassVar` protocol members are not yet supported.

`@property` protocol members are only partially supported.

A class object `C` is only considered to inhabit a protocol type with a method member `f` if `f` exists as an attribute on the metaclass of `C`.

Pass
     protocols_definition
Partial

Does not detect protocol mismatch if concrete method is missing annotations.

Does not detect protocol mismatch if concrete method's parameters are position-only.

PassPassPass
Partial

Does not reject implicit instance attributes in `Protocol` methods.

Does not support `ClassVar` protocol members.

Incorrectly considers `ClassVar` attributes on concrete classes as satisfying non-`ClassVar` attribute members on protocols.

Only has partial support for `@property` protocol members.

Pass
     protocols_explicit
Pass*

Does not report unimplemented attributes for class that explicitly derives from protocol until it is instantiated.

PassPassPass
Unsupported

Allows implicitly abstract protocol methods to be called via `super()` on a protocol subclass.

Allows instantiation of abstract subclasses of protocol classes.

Pass
     protocols_genericPass
Partial

Fails to reject duplicate generic/protocol bases.

Treats global object as a literal.

PassPass
Partial

Only partially supports `@property` protocol members.

Pass
     protocols_mergingPassPassPassPass
Partial

Does not reject attempted instantiation of abstract subclasses of protocols.

Pass
     protocols_modulesPassPassPassPass
Partial

Never considers a module as satisfying a protocol with a method member due to the fact that the method will never exist on the class `types.ModuleType`, only on a given specific module instance.

Pass
     protocols_recursivePassPassPassPass
Partial

Fails to solve a type variable involving a recursive generic protocol.

Pass
     protocols_runtime_checkable
Partial

Does not report unsafe overlap for runtime_checkable protocol.

PassPassPass
Partial

Does not reject `isinstance()` or `issubclass()` calls against runtime-checkable protocols where there is an unsafe overlap between the type of the first argument and the protocol.

Pass
     protocols_selfPassPassPassPassPassPass
     protocols_subtypingPassPassPassPassPassPass
     protocols_variancePassPassPassPassUnsupportedPass
Callables
     callables_annotation
Partial

Incorrectly treats "*args: T, **kwargs: T" as "..." when T is specialized to Any.

Does not treat "*args: Any, **kargs: Any" as "..." when separated by keyword parameter.

Pass
Partial

Parameter names are lost when resolving ParamSpec

Pass
Partial

Infers a callback protocol as being a gradual type if the callback has signature `__call__[T](self, *args: T, **kwargs: T)` and `T` has been explicitly specialized to `Any`.

Pass
     callables_kwargs
Partial

Allows callable without kwargs to be assigned to callable with unpacked kwargs

PassPassPassPassPass
     callables_protocolPassPassPassPassPassPass
     callables_subtypingPassPassPassPassPassPass
     callables_annotation
Partial

Incorrectly treats "*args: T, **kwargs: T" as "..." when T is specialized to Any.

Does not treat "*args: Any, **kargs: Any" as "..." when separated by keyword parameter.

Pass
Partial

Parameter names are lost when resolving ParamSpec

Pass
Partial

Infers a callback protocol as being a gradual type if the callback has signature `__call__[T](self, *args: T, **kwargs: T)` and `T` has been explicitly specialized to `Any`.

Pass
     callables_kwargs
Partial

Allows callable without kwargs to be assigned to callable with unpacked kwargs

PassPassPassPassPass
     callables_protocolPassPassPassPassPassPass
     callables_subtypingPassPassPassPassPassPass
Constructors
     constructors_call_init
Partial

Does not report errors during binding to self parameter of __init__ method.

Does not reject use of class-scoped type variables in annotation of self parameter in __init__ method.

PassPassPass
Partial

Does not reject invalid argument types to an inherited constructor in a specialized subclass of a generic superclass.

Does not reject class-scoped type variables used in the `self` annotation.

Pass
     constructors_call_metaclass
Unsupported

Does not honor metaclass __call__ method when evaluating constructor call.

Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

PassPassPassPassPass
     constructors_call_new
Partial

Does not support __new__ return type that is not a subclass of the class being constructed.

Does not skip evaluation of __init__ based on __new__ return type.

Does not report errors during binding to cls parameter of __new__ method.

PassPassPassPassPass
     constructors_call_type
Partial

Does not validate call to custom metaclass __call__ method through type[T].

PassPassPass
Partial

Has overly lenient handling of calls to `type[T]` if `T` is a type variable without an upper bound.

Pass
     constructors_callable
Partial

Does not generate a union type for __new__ and __init__ when converting class to callable.

Does not ignore __init__ based on __new__ return type when converting class to callable.

Does not support __new__ return type that is different from class being constructed.

Pass
Partial

Converting constructor to callable does not preserve class-scoped type params.

Pass
Partial

Does not include `__init__` when `__new__` returns `Self`.

Does not ignore `__init__` when `__new__` returns `Any`.

Unions overload return types.

Pass
     constructors_consistency
Pass*

Does not report inconsistency between __new__ and __init__ (optional).

PassPassPassPassPass
     constructors_call_init
Partial

Does not report errors during binding to self parameter of __init__ method.

Does not reject use of class-scoped type variables in annotation of self parameter in __init__ method.

PassPassPass
Partial

Does not reject invalid argument types to an inherited constructor in a specialized subclass of a generic superclass.

Does not reject class-scoped type variables used in the `self` annotation.

Pass
     constructors_call_metaclass
Unsupported

Does not honor metaclass __call__ method when evaluating constructor call.

Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

PassPassPassPassPass
     constructors_call_new
Partial

Does not support __new__ return type that is not a subclass of the class being constructed.

Does not skip evaluation of __init__ based on __new__ return type.

Does not report errors during binding to cls parameter of __new__ method.

PassPassPassPassPass
     constructors_call_type
Partial

Does not validate call to custom metaclass __call__ method through type[T].

PassPassPass
Partial

Has overly lenient handling of calls to `type[T]` if `T` is a type variable without an upper bound.

Pass
     constructors_callable
Partial

Does not generate a union type for __new__ and __init__ when converting class to callable.

Does not ignore __init__ based on __new__ return type when converting class to callable.

Does not support __new__ return type that is different from class being constructed.

Pass
Partial

Converting constructor to callable does not preserve class-scoped type params.

Pass
Partial

Does not include `__init__` when `__new__` returns `Self`.

Does not ignore `__init__` when `__new__` returns `Any`.

Unions overload return types.

Pass
     constructors_consistency
Pass*

Does not report inconsistency between __new__ and __init__ (optional).

PassPassPassPassPass
Overloads
     overloads_basicPass
Partial

Does less literal promotion than the test asks for.

PassPassPassPass
     overloads_consistencyPassPassPassPassPassPass
     overloads_definitions
Partial

Allows @override to be on all overloads and implementation, instead of just implementation.

PassPassPassPassPass
     overloads_definitions_stub
Partial

Allows @override to appear in a stub file not on the first overload.

Unsupported

Does not support checking stubs.

PassPassPassPass
     overloads_evaluation
Partial

Does not expand boolean arguments to Literal[True] and Literal[False].

Does not expand enum arguments to literal variants.

Does not expand tuple arguments to possible combinations.

Does not evaluate Any in some cases where overload is ambiguous.

Evaluates Any in some cases where overload is not ambiguous.

PassPass
Partial

Does not evaluate Any in some cases where overload is ambiguous.

PassPass
     overloads_basicPass
Partial

Does less literal promotion than the test asks for.

PassPassPassPass
     overloads_consistencyPassPassPassPassPassPass
     overloads_definitions
Partial

Allows @override to be on all overloads and implementation, instead of just implementation.

PassPassPassPassPass
     overloads_definitions_stub
Partial

Allows @override to appear in a stub file not on the first overload.

Unsupported

Does not support checking stubs.

PassPassPassPass
     overloads_evaluation
Partial

Does not expand boolean arguments to Literal[True] and Literal[False].

Does not expand enum arguments to literal variants.

Does not expand tuple arguments to possible combinations.

Does not evaluate Any in some cases where overload is ambiguous.

Evaluates Any in some cases where overload is not ambiguous.

PassPass
Partial

Does not evaluate Any in some cases where overload is ambiguous.

PassPass
Exceptions
     exceptions_context_managersPassPass
Partial

Some error suppressing context managers are not detected

PassUnsupportedPass
     exceptions_context_managersPassPass
Partial

Some error suppressing context managers are not detected

PassUnsupportedPass
Dataclasses
     dataclasses_descriptors
Partial

Assumes descriptor behavior only when field is assigned in class body.

Does not correctly evaluate type of descriptor access.

Partial

Conformance suite is questionable; see https://github.com/python/typing/issues/2259

Partial

* Assumes descriptor behavior only when field is assigned in class body

* Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types

Pass
Partial

Only infers a descriptor `__get__` method as being called when a descriptor attribute is accessed on an instance if the descriptor attribute is present in the class namespace.

Pass
     dataclasses_final
Partial

Wrongly requires a Final dataclass field to be initialized at class level.

Doesn't support Final nested inside ClassVar.

PassPassPassPassPass
     dataclasses_frozenPassPassPassPassPassPass
     dataclasses_hash
Unsupported

Does not synthesize `__hash__ = None` as a class attribute for unhashable dataclasses.

Does not report when an unhashable dataclass has `__hash__` called directly on an instance.

Does not report when dataclass is not compatible with Hashable protocol.

PassPassPass
Partial

Understands the `Hashable` protocol as equivalent to `object`.

Pass
     dataclasses_inheritancePassPassPassPassPassPass
     dataclasses_kwonlyPassPassPassPassPassPass
     dataclasses_match_argsPassPassPassPassPassPass
     dataclasses_orderPassPassPassPassPassPass
     dataclasses_postinitPassPassPassPassPassPass
     dataclasses_slots
Partial

Does not reject write to instance variable that is not defined in __slots__.

PassPassPass
Partial

Synthesizes a `__slots__` attribute but does not validate attribute assignments against `__slots__`.

Pass
     dataclasses_transform_classPassPassPassPassPassPass
     dataclasses_transform_converter
Unsupported

Converter parameter not yet supported.

PassPassPassPassPass
     dataclasses_transform_field
Partial

Does not properly handle field constructor that has default value for `kw_only` or `init` parameter.

PassPassPassPassPass
     dataclasses_transform_func
Partial

Does not handle `kw_only=False` override when `kw_only_default=True`.

Does not report error when `order=False` and comparison operators are used.

PassPassPassPassPass
     dataclasses_transform_metaPassPassPassPassPassPass
     dataclasses_usage
Pass*

Does not detect unannotated usage of `dataclasses.field()`.

PassPassPassPassPass
     dataclasses_descriptors
Partial

Assumes descriptor behavior only when field is assigned in class body.

Does not correctly evaluate type of descriptor access.

Partial

Conformance suite is questionable; see https://github.com/python/typing/issues/2259

Partial

* Assumes descriptor behavior only when field is assigned in class body

* Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types

Pass
Partial

Only infers a descriptor `__get__` method as being called when a descriptor attribute is accessed on an instance if the descriptor attribute is present in the class namespace.

Pass
     dataclasses_final
Partial

Wrongly requires a Final dataclass field to be initialized at class level.

Doesn't support Final nested inside ClassVar.

PassPassPassPassPass
     dataclasses_frozenPassPassPassPassPassPass
     dataclasses_hash
Unsupported

Does not synthesize `__hash__ = None` as a class attribute for unhashable dataclasses.

Does not report when an unhashable dataclass has `__hash__` called directly on an instance.

Does not report when dataclass is not compatible with Hashable protocol.

PassPassPass
Partial

Understands the `Hashable` protocol as equivalent to `object`.

Pass
     dataclasses_inheritancePassPassPassPassPassPass
     dataclasses_kwonlyPassPassPassPassPassPass
     dataclasses_match_argsPassPassPassPassPassPass
     dataclasses_orderPassPassPassPassPassPass
     dataclasses_postinitPassPassPassPassPassPass
     dataclasses_slots
Partial

Does not reject write to instance variable that is not defined in __slots__.

PassPassPass
Partial

Synthesizes a `__slots__` attribute but does not validate attribute assignments against `__slots__`.

Pass
     dataclasses_transform_classPassPassPassPassPassPass
     dataclasses_transform_converter
Unsupported

Converter parameter not yet supported.

PassPassPassPassPass
     dataclasses_transform_field
Partial

Does not properly handle field constructor that has default value for `kw_only` or `init` parameter.

PassPassPassPassPass
     dataclasses_transform_func
Partial

Does not handle `kw_only=False` override when `kw_only_default=True`.

Does not report error when `order=False` and comparison operators are used.

PassPassPassPassPass
     dataclasses_transform_metaPassPassPassPassPassPass
     dataclasses_usage
Pass*

Does not detect unannotated usage of `dataclasses.field()`.

PassPassPassPassPass
Typed dictionaries
     typeddicts_alt_syntax
Pass*

Does not support keyword-argument form of alternative syntax (deprecated in 3.11).

PassPassPassPassPass
     typeddicts_class_syntax
Partial

Does not support version-conditional items in TypedDict definitions.

PassPass
Partial

Does not support version-conditional items in TypedDict definitions.

PassPass
     typeddicts_extra_items
Unsupported

Not supported.

PassPassPassPassPass
     typeddicts_finalPassPassPassPassPassPass
     typeddicts_inheritancePassPassPassPassPassPass
     typeddicts_operationsPassPassPassPassPassPass
     typeddicts_readonlyPassPassPassPassPassPass
     typeddicts_readonly_consistencyPassPassPassPassPassPass
     typeddicts_readonly_inheritance
Partial

Incorrectly rejects non-ReadOnly override of ReadOnly item.

Incorrectly rejects override of ReadOnly item with another ReadOnly item with narrower type.

Incorrectly rejects override of NotRequired ReadOnly item with a Required ReadOnly item.

PassPassPassPassPass
     typeddicts_readonly_kwargsPassPassPassPassPassPass
     typeddicts_readonly_update
Partial

Incorrectly allows update of ReadOnly item.

Incorrectly rejects update involving an item with Never type.

PassPassPassPassPass
     typeddicts_requiredPassPassPassPassPassPass
     typeddicts_type_consistencyPass
Partial

Considers TypedDicts to be assignable to plain dict types.

PassPassPassPass
     typeddicts_usagePassPassPassPassPassPass
     typeddicts_alt_syntax
Pass*

Does not support keyword-argument form of alternative syntax (deprecated in 3.11).

PassPassPassPassPass
     typeddicts_class_syntax
Partial

Does not support version-conditional items in TypedDict definitions.

PassPass
Partial

Does not support version-conditional items in TypedDict definitions.

PassPass
     typeddicts_extra_items
Unsupported

Not supported.

PassPassPassPassPass
     typeddicts_finalPassPassPassPassPassPass
     typeddicts_inheritancePassPassPassPassPassPass
     typeddicts_operationsPassPassPassPassPassPass
     typeddicts_readonlyPassPassPassPassPassPass
     typeddicts_readonly_consistencyPassPassPassPassPassPass
     typeddicts_readonly_inheritance
Partial

Incorrectly rejects non-ReadOnly override of ReadOnly item.

Incorrectly rejects override of ReadOnly item with another ReadOnly item with narrower type.

Incorrectly rejects override of NotRequired ReadOnly item with a Required ReadOnly item.

PassPassPassPassPass
     typeddicts_readonly_kwargsPassPassPassPassPassPass
     typeddicts_readonly_update
Partial

Incorrectly allows update of ReadOnly item.

Incorrectly rejects update involving an item with Never type.

PassPassPassPassPass
     typeddicts_requiredPassPassPassPassPassPass
     typeddicts_type_consistencyPass
Partial

Considers TypedDicts to be assignable to plain dict types.

PassPassPassPass
     typeddicts_usagePassPassPassPassPassPass
Tuples
     tuples_type_compat
Partial

Does not support tuple narrowing based on `len()` type guard (optional).

Incorrectly marks a match case as unreachable.

PassPassPassPassPass
     tuples_type_formPassPassPassPassPassPass
     tuples_unpacked
Partial

"More than one unpack" error is missing in some cases.

PassPassPassPassPass
     tuples_type_compat
Partial

Does not support tuple narrowing based on `len()` type guard (optional).

Incorrectly marks a match case as unreachable.

PassPassPassPassPass
     tuples_type_formPassPassPassPassPassPass
     tuples_unpacked
Partial

"More than one unpack" error is missing in some cases.

PassPassPassPassPass
Named tuples
     namedtuples_define_class
Partial

Does not reject override of named tuple attribute in child class.

Does not support version-conditional fields.

Partial

Does not support precise type inference for slices over namedtuples.

PassPassPassPass
     namedtuples_define_functionalPassPassPassPassPassPass
     namedtuples_type_compatPassPassPassPassPassPass
     namedtuples_usage
Partial

Does not reject attempt to delete named tuple field by name.

PassPassPassPassPass
     namedtuples_define_class
Partial

Does not reject override of named tuple attribute in child class.

Does not support version-conditional fields.

Partial

Does not support precise type inference for slices over namedtuples.

PassPassPassPass
     namedtuples_define_functionalPassPassPassPassPassPass
     namedtuples_type_compatPassPassPassPassPassPass
     namedtuples_usage
Partial

Does not reject attempt to delete named tuple field by name.

PassPassPassPassPass
Enumerations
     enums_behaviorsPassPassPassPassPassPass
     enums_definitionPass
Partial

Does not allow enum members to be conditional on version/platform checks.

PassPassPassPass
     enums_expansion
Partial

Improperly applies narrowing to Flag subclass.

PassPassPass
Partial

Does not support `enum.Flag`.

Pass
     enums_member_names
Pass*

Does not support special-cased handling of member name literal types in some cases (optional).

PassPassPassPassPass
     enums_member_values
Partial

Does not enforce declared type of `_value_`.

Does not enforce assigned tuple types for enum members (optional).

PassPassPassPassPass
     enums_members
Partial

Does not treat attribute with annotation and no assignment as non-member.

Does not treat callables as non-members.

Does not honor `enum.member` as method decorator.

Does not properly handle aliased enum members.

Does not support `_ignore_` mechanism (optional).

Does not treat attributes with private names as non-members.

PassPass
Pass*

Does not support `_ignore_` mechanism (optional).

PassPass
     enums_behaviorsPassPassPassPassPassPass
     enums_definitionPass
Partial

Does not allow enum members to be conditional on version/platform checks.

PassPassPassPass
     enums_expansion
Partial

Improperly applies narrowing to Flag subclass.

PassPassPass
Partial

Does not support `enum.Flag`.

Pass
     enums_member_names
Pass*

Does not support special-cased handling of member name literal types in some cases (optional).

PassPassPassPassPass
     enums_member_values
Partial

Does not enforce declared type of `_value_`.

Does not enforce assigned tuple types for enum members (optional).

PassPassPassPassPass
     enums_members
Partial

Does not treat attribute with annotation and no assignment as non-member.

Does not treat callables as non-members.

Does not honor `enum.member` as method decorator.

Does not properly handle aliased enum members.

Does not support `_ignore_` mechanism (optional).

Does not treat attributes with private names as non-members.

PassPass
Pass*

Does not support `_ignore_` mechanism (optional).

PassPass
Type narrowing
     narrowing_typeguardPassPassPassPassPassPass
     narrowing_typeisPassPassPassPass
Partial

Intersects the pre-existing type with the top materialization of the bracketed type rather than the bracketed type itself.

Pass
     narrowing_typeguardPassPassPassPassPassPass
     narrowing_typeisPassPassPassPass
Partial

Intersects the pre-existing type with the top materialization of the bracketed type rather than the bracketed type itself.

Pass
Type checker directives
     directives_assert_typePassPassPassPassPassPass
     directives_castPassPassPassPassPassPass
     directives_deprecatedPassPassPassPass
Partial

Does not detect calls to deprecated overloads.

Does not detect implicit calls to deprecated dunder methods, for example via operators.

Does not detect accesses of, or attempts to set, deprecated properties.

Pass
     directives_disjoint_basePass
Partial

Does not reject invalid class definitions due to disjoint bases, but uses disjoint base information in type narrowing.

Pass
Unsupported

Does not support PEP 800 disjoint-base semantics.

PassPass
     directives_no_type_check
Partial

Does not honor `@no_type_check` class decorator (allowed).

Does not reject invalid call of `@no_type_check` function.

PassPass
Pass*

Does not honor `@no_type_check` class decorator (allowed).

PassPass
     directives_reveal_typePassPassPassPassPassPass
     directives_type_checkingPassPassPassPassPassPass
     directives_type_ignore
Partial

Does not honor "# type: ignore" comment if comment includes additional text.

PassPassPassPassPass
     directives_type_ignore_file1PassPassPassPassPassPass
     directives_type_ignore_file2PassPassPassPassPassPass
     directives_version_platform
Pass*

Does not understand three-element form of sys.version checks.

Does not understand os.name checks.

PassPassPassPassPass
     directives_assert_typePassPassPassPassPassPass
     directives_castPassPassPassPassPassPass
     directives_deprecatedPassPassPassPass
Partial

Does not detect calls to deprecated overloads.

Does not detect implicit calls to deprecated dunder methods, for example via operators.

Does not detect accesses of, or attempts to set, deprecated properties.

Pass
     directives_disjoint_basePass
Partial

Does not reject invalid class definitions due to disjoint bases, but uses disjoint base information in type narrowing.

Pass
Unsupported

Does not support PEP 800 disjoint-base semantics.

PassPass
     directives_no_type_check
Partial

Does not honor `@no_type_check` class decorator (allowed).

Does not reject invalid call of `@no_type_check` function.

PassPass
Pass*

Does not honor `@no_type_check` class decorator (allowed).

PassPass
     directives_reveal_typePassPassPassPassPassPass
     directives_type_checkingPassPassPassPassPassPass
     directives_type_ignore
Partial

Does not honor "# type: ignore" comment if comment includes additional text.

PassPassPassPassPass
     directives_type_ignore_file1PassPassPassPassPassPass
     directives_type_ignore_file2PassPassPassPassPassPass
     directives_version_platform
Pass*

Does not understand three-element form of sys.version checks.

Does not understand os.name checks.

PassPassPassPassPass
Historical and deprecated features
     historical_positional
Partial

Does not reject positional-only parameter after non-positional-only parameter.

Treats keyword-only parameter as positional-only.

Applies legacy positional-only rules when PEP 570 syntax is used.

PassPassPassPassPass
     historical_positional
Partial

Does not reject positional-only parameter after non-positional-only parameter.

Treats keyword-only parameter as positional-only.

Applies legacy positional-only rules when PEP 570 syntax is used.

PassPassPassPassPass
diff --git a/conformance/src/reporting.py b/conformance/src/reporting.py index 149527f20..066cbb9e4 100644 --- a/conformance/src/reporting.py +++ b/conformance/src/reporting.py @@ -30,7 +30,15 @@ def generate_summary_html(root_dir: Path) -> str: test_groups = get_test_groups(root_dir) test_cases = get_test_cases(test_groups, root_dir / "tests") - summary_html = ["", "", ""] + summary_html = [ + "", + '', + f'', + "", + "", + "", + "", + ] for type_checker in type_checkers: # Load the version file for the type checker. @@ -69,7 +77,7 @@ def generate_summary_html(root_dir: Path) -> str: for test_case in tests_in_group: test_case_name = test_case.stem - summary_html.append(f'     {test_case_name}') + summary_html.append(f'     {test_case_name}') for type_checker in type_checkers: try: @@ -112,7 +120,7 @@ def generate_summary_html(root_dir: Path) -> str: if raw_notes != "": conformance_cell = f'
{conformance_cell}
{notes}
' - summary_html.append(f'{conformance_cell}') + summary_html.append(f'{conformance_cell}') summary_html.append("") diff --git a/conformance/src/results_template.html b/conformance/src/results_template.html index fed258c2b..2b905356f 100644 --- a/conformance/src/results_template.html +++ b/conformance/src/results_template.html @@ -44,6 +44,16 @@ font-weight: normal; } + th:first-child, + td:first-child { + vertical-align: top; + } + + th:not(:first-child), + td:not(:first-child){ + text-align: center; + } + .content_container { margin: 20px; } @@ -74,12 +84,10 @@ .col1 { width: 28%; - vertical-align: top; } .col2 { width: 14%; - text-align: center; } thead > tr > th { From fca0e19256f54b906fad3c6bbd4ef56d6f8ca83b Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:31 +0100 Subject: [PATCH 13/30] Remove unused CSS rules --- conformance/results/results.html | 4 ---- conformance/src/results_template.html | 4 ---- 2 files changed, 8 deletions(-) diff --git a/conformance/results/results.html b/conformance/results/results.html index 37d666579..5ab8fe9e0 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -97,10 +97,6 @@ text-align: center; } - .tc-time { - font-size: 12px; - } - .test_group { font-size: 10pt; font-weight: bold; diff --git a/conformance/src/results_template.html b/conformance/src/results_template.html index 2b905356f..d19a0f7b5 100644 --- a/conformance/src/results_template.html +++ b/conformance/src/results_template.html @@ -97,10 +97,6 @@ text-align: center; } - .tc-time { - font-size: 12px; - } - .test_group { font-size: 10pt; font-weight: bold; From 3a0ebb69097b66d8a6e6d8190f94edc29daa0a86 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:31 +0100 Subject: [PATCH 14/30] Reduce excess elements for tooltip implementation --- conformance/results/results.html | 288 +++++++++++++------------- conformance/src/reporting.py | 7 +- conformance/src/results_template.html | 12 +- 3 files changed, 154 insertions(+), 153 deletions(-) diff --git a/conformance/results/results.html b/conformance/results/results.html index 5ab8fe9e0..b20b3424b 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -114,7 +114,7 @@ background-color: rgb(242, 171, 171); } - .tooltip-text { + .notes { visibility: hidden; position: absolute; z-index: 2; @@ -127,25 +127,25 @@ text-align: left; } - .hover-text:hover .tooltip-text { + .tooltip:hover .notes { visibility: visible; } - .hover-text .tooltip-text p { + .tooltip .notes p { margin: 4px; } - .hover-text .tooltip-text { + .tooltip .notes { top: 25px; left: -80%; } - .hover-text .tooltip-text::before { + .tooltip .notes::before { top: -5%; left: 94%; } - .hover-text { + .tooltip { cursor: help; position: relative; } @@ -201,15 +201,15 @@

Python Type System Conformance Test Results

Pass      annotations_forward_refs -
Partial

Does not report error for a forward reference that is not enclosed in quotes.

Does not report error for use of quoted type with "|" operator (runtime error).

Incorrectly generates error for quoted type defined in class scope.

-
Partial

Fails to reject "x" | int annotations that fail at runtime.

Rejects some valid quoted annotations.

-
Partial

Types in quotes incorrectly refer to shadowing class member.

Does not reject some type forms that require quotes.

+Partial

Does not report error for a forward reference that is not enclosed in quotes.

Does not report error for use of quoted type with "|" operator (runtime error).

Incorrectly generates error for quoted type defined in class scope.

+Partial

Fails to reject "x" | int annotations that fail at runtime.

Rejects some valid quoted annotations.

+Partial

Types in quotes incorrectly refer to shadowing class member.

Does not reject some type forms that require quotes.

Pass -
Partial

Resolves references in type annotations as referring to end-of-scope types (https://discuss.python.org/t/annotation-string-references-in-class-scope-in-conformance-tests/105439, https://github.com/python/typing/pull/2144)

-
Partial

Incorrectly generates error for quoted type defined in class scope.

+Partial

Resolves references in type annotations as referring to end-of-scope types (https://discuss.python.org/t/annotation-string-references-in-class-scope-in-conformance-tests/105439, https://github.com/python/typing/pull/2144)

+Partial

Incorrectly generates error for quoted type defined in class scope.

     annotations_generators -
Partial

Does not report incompatible Generator type in `yield from` statement.

+Partial

Does not report incompatible Generator type in `yield from` statement.

Pass Pass Pass @@ -217,16 +217,16 @@

Python Type System Conformance Test Results

Pass      annotations_methods -
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

+Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

Pass Pass -
Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

+Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

Pass Pass      annotations_typeexpr Pass -
Partial

Fails to reject various weird annotations

+Partial

Fails to reject various weird annotations

Pass Pass Pass @@ -236,8 +236,8 @@

Python Type System Conformance Test Results

Type forms      typeforms_typeform -
Partial

Does not support assigning Union and GenericAlias objects to their runtime types.

-
Partial

Fails to reject various weird annotations

+Partial

Does not support assigning Union and GenericAlias objects to their runtime types.

+Partial

Fails to reject various weird annotations

Pass Unsupported Pass @@ -256,7 +256,7 @@

Python Type System Conformance Test Results

     specialtypes_never Pass -
Partial

Does not enforce invariance in some contexts

Does not allow Any to be assigned to Never

+Partial

Does not enforce invariance in some contexts

Does not allow Any to be assigned to Never

Pass Pass Pass @@ -279,18 +279,18 @@

Python Type System Conformance Test Results

Pass      specialtypes_type -
Partial

Does not treat `type` same as `type[Any]` for assert_type.

Does not allow access to unknown attributes from object of type `type[Any]`.

+Partial

Does not treat `type` same as `type[Any]` for assert_type.

Does not allow access to unknown attributes from object of type `type[Any]`.

Pass Pass Pass -
Partial

Allows arbitrary attributes to be accessed on `TA` where `TA = typing.Type[typing.Any]` or `TA = type[typing.Any]`.

Treats `type` equivalently to `type[object]` rather than `type[typing.Any]`.

+Partial

Allows arbitrary attributes to be accessed on `TA` where `TA = typing.Type[typing.Any]` or `TA = type[typing.Any]`.

Treats `type` equivalently to `type[object]` rather than `type[typing.Any]`.

Pass Generics      generics_base_class -
Partial

Does not detect inconsistent type variable ordering.

+Partial

Does not detect inconsistent type variable ordering.

Pass Pass Pass @@ -302,31 +302,31 @@

Python Type System Conformance Test Results

Pass Pass Pass -
Partial

Incorrectly allows constrained type variables to be solved to a union of their constraints.

+Partial

Incorrectly allows constrained type variables to be solved to a union of their constraints.

Pass      generics_defaults -
Partial

Does not detect a TypeVar with a default used after a TypeVarTuple.

Does not fully support defaults on TypeVarTuple and ParamSpec.

-
Partial

Numerous issues; does not support TypeVarTuple and ParamSpec defaults.

+Partial

Does not detect a TypeVar with a default used after a TypeVarTuple.

Does not fully support defaults on TypeVarTuple and ParamSpec.

+Partial

Numerous issues; does not support TypeVarTuple and ParamSpec defaults.

Pass Pass -
Partial

Does not forbid a `TypeVar` immediately following a `TypeVarTuple` in a parameter list from having a default.

Does not support `TypeVarTuple`.

+Partial

Does not forbid a `TypeVar` immediately following a `TypeVarTuple` in a parameter list from having a default.

Does not support `TypeVarTuple`.

Pass      generics_defaults_referential -
Partial

Does not correctly handle defaults referencing other TypeVars.

-
Partial

Fails to apply default specializations in some cases.

+Partial

Does not correctly handle defaults referencing other TypeVars.

+Partial

Fails to apply default specializations in some cases.

Pass Pass Pass Pass      generics_defaults_specialization -
Partial

Does not correctly resolve defaults when classes are used directly.

+Partial

Does not correctly resolve defaults when classes are used directly.

Pass Pass -
Partial

Allows incorrect assignment to type[].

-
Partial

Does not reject subscription of an already-specialized generic class.

+Partial

Allows incorrect assignment to type[].

+Partial

Does not reject subscription of an already-specialized generic class.

Pass      generics_paramspec_basic @@ -342,14 +342,14 @@

Python Type System Conformance Test Results

Pass Pass Pass -
Partial

Incorrectly allows using `*args: P.args` and `**kwargs: P.kwargs` when `P` has not been put into scope by any other parameter annotation or enclosing scope.

+Partial

Incorrectly allows using `*args: P.args` and `**kwargs: P.kwargs` when `P` has not been put into scope by any other parameter annotation or enclosing scope.

Pass      generics_paramspec_semantics Pass Pass Pass -
Pass*

Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).

+Pass*

Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).

Pass Pass @@ -364,15 +364,15 @@

Python Type System Conformance Test Results

     generics_scoping Pass Pass -
Partial

Does not implement several scoping checks/restrictions for generics

+Partial

Does not implement several scoping checks/restrictions for generics

Pass -
Partial

Does not reject `list[T]()` in the global scope, where `T` is an unbound type variable.

Does nto reject `alias: TypeAlias = list[T]` in the body scope of a class generic over a type variable `T`.

+Partial

Does not reject `list[T]()` in the global scope, where `T` is an unbound type variable.

Does nto reject `alias: TypeAlias = list[T]` in the body scope of a class generic over a type variable `T`.

Pass      generics_self_advanced -
Partial

Does not infer the type of an unannotated `self` parameter to be type `Self`.

Does not retain `Self` when calling method that returns `Self`.

Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`.

Does not retain `Self` when accessing attribute through `type[Self]`.

+Partial

Does not infer the type of an unannotated `self` parameter to be type `Self`.

Does not retain `Self` when calling method that returns `Self`.

Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`.

Does not retain `Self` when accessing attribute through `type[Self]`.

Pass -
Pass*

Treats attributes not initialized on the class as instance-only

+Pass*

Treats attributes not initialized on the class as instance-only

Pass Pass Pass @@ -406,7 +406,7 @@

Python Type System Conformance Test Results

Pass Pass Pass -
Partial

Does not reject `Self` used in a return annotation when `self` is annotated using another type variable.

+Partial

Does not reject `Self` used in a return annotation when `self` is annotated using another type variable.

Pass      generics_syntax_compatibility @@ -426,7 +426,7 @@

Python Type System Conformance Test Results

Pass      generics_syntax_infer_variance -
Unsupported

Type parameter syntax not yet supported.

+Unsupported

Type parameter syntax not yet supported.

Pass Pass Pass @@ -434,34 +434,34 @@

Python Type System Conformance Test Results

Pass      generics_syntax_scoping -
Partial

Does not following runtime scoping rules for type parameters in all cases.

-
Partial

Misses some details of scoping rules.

+Partial

Does not following runtime scoping rules for type parameters in all cases.

+Partial

Misses some details of scoping rules.

Pass Pass Pass Pass      generics_type_erasure -
Partial

Infers Node[Never] instead of Node[Any] when argument is not provided.

False negative on instance attribute access on type(node).

+Partial

Infers Node[Never] instead of Node[Any] when argument is not provided.

False negative on instance attribute access on type(node).

Pass Pass Pass -
Partial

Does not reject access of generic instance variable from the class object.

+Partial

Does not reject access of generic instance variable from the class object.

Pass      generics_typevartuple_args -
Partial

Does not enforce that tuples captured by TypeVarTuple are of the same length.

+Partial

Does not enforce that tuples captured by TypeVarTuple are of the same length.

Pass Pass -
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

-
Partial

Supports PEP-646 unpacked tuples but not TypeVarTuple.

+Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

+Partial

Supports PEP-646 unpacked tuples but not TypeVarTuple.

Pass      generics_typevartuple_basic -
Partial

Does not enforce that tuples captured by TypeVarTuple are same length.

+Partial

Does not enforce that tuples captured by TypeVarTuple are same length.

Pass Pass -
Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

+Partial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

Unsupported Pass @@ -490,7 +490,7 @@

Python Type System Conformance Test Results

Pass      generics_typevartuple_specialization -
Partial

Incorrectly specializes generic alias that includes a TypeVar and TypeVarTuple if no type arguments are provided.

Rejects specialization of generic type alias defined as a tuple containing a TypeVar.

+Partial

Incorrectly specializes generic alias that includes a TypeVar and TypeVarTuple if no type arguments are provided.

Rejects specialization of generic type alias defined as a tuple containing a TypeVar.

Pass Pass Pass @@ -506,7 +506,7 @@

Python Type System Conformance Test Results

Pass      generics_upper_bound -
Partial

Does not reject use of type variable within an upper bound.

+Partial

Does not reject use of type variable within an upper bound.

Pass Pass Pass @@ -514,7 +514,7 @@

Python Type System Conformance Test Results

Pass      generics_variance -
Partial

Does not reject use of class-scoped TypeVar used in a base class when variance is incompatible.

+Partial

Does not reject use of class-scoped TypeVar used in a base class when variance is incompatible.

Pass Pass Pass @@ -533,15 +533,15 @@

Python Type System Conformance Test Results

Type qualifiers      qualifiers_annotated -
Partial

Does not allow ClassVar to be nested within Annotated.

Does not allow Final to be nested within Annotated.

Does not allow Required and NotRequired to be nested within Annotated.

Does not reject type[T] compatibility for type alias defined with Annotated.

Does not reject call of type alias defined with Annotated.

-
Partial

Fails to reject various weird annotations.

False positive on lambda in Annotated.

+Partial

Does not allow ClassVar to be nested within Annotated.

Does not allow Final to be nested within Annotated.

Does not allow Required and NotRequired to be nested within Annotated.

Does not reject type[T] compatibility for type alias defined with Annotated.

Does not reject call of type alias defined with Annotated.

+Partial

Fails to reject various weird annotations.

False positive on lambda in Annotated.

Pass Pass Pass Pass      qualifiers_final_annotation -
Partial

Does not treat use of Final name as if it was replaced by the literal in NamedTuple definition.

Does not allow conditional assignment of Final instance variable in __init__ method.

Does not allow redefinition of private class variable that is marked Final in parent class.

Does not report modification of local Final variable via "for" statement.

+Partial

Does not treat use of Final name as if it was replaced by the literal in NamedTuple definition.

Does not allow conditional assignment of Final instance variable in __init__ method.

Does not allow redefinition of private class variable that is marked Final in parent class.

Does not report modification of local Final variable via "for" statement.

Pass Pass Pass @@ -560,7 +560,7 @@

Python Type System Conformance Test Results

Class type compatibility      classes_classvar -
Partial

Internal error if TypeVarTuple is used in ClassVar.

Does not reject use of ParamSpec in ClassVar.

Rejects ClassVar nested in Annotated.

Does not reject use of ClassVar in TypeAlias definition.

+Partial

Internal error if TypeVarTuple is used in ClassVar.

Does not reject use of ParamSpec in ClassVar.

Rejects ClassVar nested in Annotated.

Does not reject use of ClassVar in TypeAlias definition.

Pass Pass Pass @@ -579,23 +579,23 @@

Python Type System Conformance Test Results

Type aliases      aliases_explicit -
Partial

Does not reject specialization of type alias that has already been implicitly specialized.

+Partial

Does not reject specialization of type alias that has already been implicitly specialized.

Pass Pass Pass -
Partial

Does not emit a diagnostic if a type alias to a union, in which every element of the union is implicitly specialized with `Unknown`, is invalidly specialized again.

+Partial

Does not emit a diagnostic if a type alias to a union, in which every element of the union is implicitly specialized with `Unknown`, is invalidly specialized again.

Pass      aliases_implicit Pass -
Partial

Fails to handle various weird annotations.

Various bugs with resolving generic aliases.

+Partial

Fails to handle various weird annotations.

Various bugs with resolving generic aliases.

Pass Pass -
Partial

Does not reject variables with `Any` or `Unknown` types when used as implicit type aliases.

Falls short on full syntactic validation of type aliases.

+Partial

Does not reject variables with `Any` or `Unknown` types when used as implicit type aliases.

Falls short on full syntactic validation of type aliases.

Pass      aliases_newtype -
Partial

`NewType`s are incorrectly considered to be classes.

+Partial

`NewType`s are incorrectly considered to be classes.

Pass Pass Pass @@ -612,18 +612,18 @@

Python Type System Conformance Test Results

     aliases_type_statement Pass -
Partial

Fails to reject various weird annotations.

+Partial

Fails to reject various weird annotations.

Pass Pass -
Partial

Does not reject circular definitions of type aliases.

Does not support `type` statements generic over `TypeVarTuple`s.

+Partial

Does not reject circular definitions of type aliases.

Does not support `type` statements generic over `TypeVarTuple`s.

Pass      aliases_typealiastype -
Partial

Incorrectly rejects some recursive type aliases using TypeAliasType.

Incorrectly rejects the use of a class-scoped TypeVar in a TypeAliasType definition.

-
Partial

Rejects valid ParamSpec specialization.

+Partial

Incorrectly rejects some recursive type aliases using TypeAliasType.

Incorrectly rejects the use of a class-scoped TypeVar in a TypeAliasType definition.

+Partial

Rejects valid ParamSpec specialization.

Pass Pass -
Partial

Does not reject specializing a type parameter in a generic type alias with a type inconsistent with the parameter's upper bound.

Does not reject declaring a type alias with a type variable that is not in scope.

Does not reject declaring a type alias with a non-literal tuple passed to the `type_params` parameter.

Does not reject cyclically defined type aliases.

+Partial

Does not reject specializing a type parameter in a generic type alias with a type inconsistent with the parameter's upper bound.

Does not reject declaring a type alias with a type variable that is not in scope.

Does not reject declaring a type alias with a non-literal tuple passed to the `type_params` parameter.

Does not reject cyclically defined type aliases.

Pass      aliases_variance @@ -638,15 +638,15 @@

Python Type System Conformance Test Results

Literals      literals_interactions -
Pass*

Does not narrow `str` or `LiteralString` types to `Literal` string types via equality or containment checks.

+Pass*

Does not narrow `str` or `LiteralString` types to `Literal` string types via equality or containment checks.

Pass Pass Pass -
Pass*

Deliberately does not allow `str` to be narrowed to literal string types through equality or containment checks due to the possibility of `str` subclasses that could have unexpected equality semantics.

+Pass*

Deliberately does not allow `str` to be narrowed to literal string types through equality or containment checks due to the possibility of `str` subclasses that could have unexpected equality semantics.

Pass      literals_literalstring -
Unsupported

Support for `LiteralString` is not implemented.

+Unsupported

Support for `LiteralString` is not implemented.

Pass Pass Pass @@ -654,8 +654,8 @@

Python Type System Conformance Test Results

Pass      literals_parameterizations -
Partial

Does not reject tuple within Literal.

-
Partial

Fails to reject various invalid literal parameterizations.

+Partial

Does not reject tuple within Literal.

+Partial

Fails to reject various invalid literal parameterizations.

Pass Pass Pass @@ -674,34 +674,34 @@

Python Type System Conformance Test Results

     protocols_class_objects Pass -
Partial

Abstract type[Proto] still allows protocol class objects in some paths.

+Partial

Abstract type[Proto] still allows protocol class objects in some paths.

Pass Pass -
Unsupported

`type[Proto]` is not yet supported.

`ClassVar` protocol members are not yet supported.

`@property` protocol members are only partially supported.

A class object `C` is only considered to inhabit a protocol type with a method member `f` if `f` exists as an attribute on the metaclass of `C`.

+Unsupported

`type[Proto]` is not yet supported.

`ClassVar` protocol members are not yet supported.

`@property` protocol members are only partially supported.

A class object `C` is only considered to inhabit a protocol type with a method member `f` if `f` exists as an attribute on the metaclass of `C`.

Pass      protocols_definition -
Partial

Does not detect protocol mismatch if concrete method is missing annotations.

Does not detect protocol mismatch if concrete method's parameters are position-only.

+Partial

Does not detect protocol mismatch if concrete method is missing annotations.

Does not detect protocol mismatch if concrete method's parameters are position-only.

Pass Pass Pass -
Partial

Does not reject implicit instance attributes in `Protocol` methods.

Does not support `ClassVar` protocol members.

Incorrectly considers `ClassVar` attributes on concrete classes as satisfying non-`ClassVar` attribute members on protocols.

Only has partial support for `@property` protocol members.

+Partial

Does not reject implicit instance attributes in `Protocol` methods.

Does not support `ClassVar` protocol members.

Incorrectly considers `ClassVar` attributes on concrete classes as satisfying non-`ClassVar` attribute members on protocols.

Only has partial support for `@property` protocol members.

Pass      protocols_explicit -
Pass*

Does not report unimplemented attributes for class that explicitly derives from protocol until it is instantiated.

+Pass*

Does not report unimplemented attributes for class that explicitly derives from protocol until it is instantiated.

Pass Pass Pass -
Unsupported

Allows implicitly abstract protocol methods to be called via `super()` on a protocol subclass.

Allows instantiation of abstract subclasses of protocol classes.

+Unsupported

Allows implicitly abstract protocol methods to be called via `super()` on a protocol subclass.

Allows instantiation of abstract subclasses of protocol classes.

Pass      protocols_generic Pass -
Partial

Fails to reject duplicate generic/protocol bases.

Treats global object as a literal.

+Partial

Fails to reject duplicate generic/protocol bases.

Treats global object as a literal.

Pass Pass -
Partial

Only partially supports `@property` protocol members.

+Partial

Only partially supports `@property` protocol members.

Pass      protocols_merging @@ -709,7 +709,7 @@

Python Type System Conformance Test Results

Pass Pass Pass -
Partial

Does not reject attempted instantiation of abstract subclasses of protocols.

+Partial

Does not reject attempted instantiation of abstract subclasses of protocols.

Pass      protocols_modules @@ -717,7 +717,7 @@

Python Type System Conformance Test Results

Pass Pass Pass -
Partial

Never considers a module as satisfying a protocol with a method member due to the fact that the method will never exist on the class `types.ModuleType`, only on a given specific module instance.

+Partial

Never considers a module as satisfying a protocol with a method member due to the fact that the method will never exist on the class `types.ModuleType`, only on a given specific module instance.

Pass      protocols_recursive @@ -725,15 +725,15 @@

Python Type System Conformance Test Results

Pass Pass Pass -
Partial

Fails to solve a type variable involving a recursive generic protocol.

+Partial

Fails to solve a type variable involving a recursive generic protocol.

Pass      protocols_runtime_checkable -
Partial

Does not report unsafe overlap for runtime_checkable protocol.

+Partial

Does not report unsafe overlap for runtime_checkable protocol.

Pass Pass Pass -
Partial

Does not reject `isinstance()` or `issubclass()` calls against runtime-checkable protocols where there is an unsafe overlap between the type of the first argument and the protocol.

+Partial

Does not reject `isinstance()` or `issubclass()` calls against runtime-checkable protocols where there is an unsafe overlap between the type of the first argument and the protocol.

Pass      protocols_self @@ -764,15 +764,15 @@

Python Type System Conformance Test Results

Callables      callables_annotation -
Partial

Incorrectly treats "*args: T, **kwargs: T" as "..." when T is specialized to Any.

Does not treat "*args: Any, **kargs: Any" as "..." when separated by keyword parameter.

+Partial

Incorrectly treats "*args: T, **kwargs: T" as "..." when T is specialized to Any.

Does not treat "*args: Any, **kargs: Any" as "..." when separated by keyword parameter.

Pass -
Partial

Parameter names are lost when resolving ParamSpec

+Partial

Parameter names are lost when resolving ParamSpec

Pass -
Partial

Infers a callback protocol as being a gradual type if the callback has signature `__call__[T](self, *args: T, **kwargs: T)` and `T` has been explicitly specialized to `Any`.

+Partial

Infers a callback protocol as being a gradual type if the callback has signature `__call__[T](self, *args: T, **kwargs: T)` and `T` has been explicitly specialized to `Any`.

Pass      callables_kwargs -
Partial

Allows callable without kwargs to be assigned to callable with unpacked kwargs

+Partial

Allows callable without kwargs to be assigned to callable with unpacked kwargs

Pass Pass Pass @@ -799,15 +799,15 @@

Python Type System Conformance Test Results

Constructors      constructors_call_init -
Partial

Does not report errors during binding to self parameter of __init__ method.

Does not reject use of class-scoped type variables in annotation of self parameter in __init__ method.

+Partial

Does not report errors during binding to self parameter of __init__ method.

Does not reject use of class-scoped type variables in annotation of self parameter in __init__ method.

Pass Pass Pass -
Partial

Does not reject invalid argument types to an inherited constructor in a specialized subclass of a generic superclass.

Does not reject class-scoped type variables used in the `self` annotation.

+Partial

Does not reject invalid argument types to an inherited constructor in a specialized subclass of a generic superclass.

Does not reject class-scoped type variables used in the `self` annotation.

Pass      constructors_call_metaclass -
Unsupported

Does not honor metaclass __call__ method when evaluating constructor call.

Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

+Unsupported

Does not honor metaclass __call__ method when evaluating constructor call.

Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

Pass Pass Pass @@ -815,7 +815,7 @@

Python Type System Conformance Test Results

Pass      constructors_call_new -
Partial

Does not support __new__ return type that is not a subclass of the class being constructed.

Does not skip evaluation of __init__ based on __new__ return type.

Does not report errors during binding to cls parameter of __new__ method.

+Partial

Does not support __new__ return type that is not a subclass of the class being constructed.

Does not skip evaluation of __init__ based on __new__ return type.

Does not report errors during binding to cls parameter of __new__ method.

Pass Pass Pass @@ -823,23 +823,23 @@

Python Type System Conformance Test Results

Pass      constructors_call_type -
Partial

Does not validate call to custom metaclass __call__ method through type[T].

+Partial

Does not validate call to custom metaclass __call__ method through type[T].

Pass Pass Pass -
Partial

Has overly lenient handling of calls to `type[T]` if `T` is a type variable without an upper bound.

+Partial

Has overly lenient handling of calls to `type[T]` if `T` is a type variable without an upper bound.

Pass      constructors_callable -
Partial

Does not generate a union type for __new__ and __init__ when converting class to callable.

Does not ignore __init__ based on __new__ return type when converting class to callable.

Does not support __new__ return type that is different from class being constructed.

+Partial

Does not generate a union type for __new__ and __init__ when converting class to callable.

Does not ignore __init__ based on __new__ return type when converting class to callable.

Does not support __new__ return type that is different from class being constructed.

Pass -
Partial

Converting constructor to callable does not preserve class-scoped type params.

+Partial

Converting constructor to callable does not preserve class-scoped type params.

Pass -
Partial

Does not include `__init__` when `__new__` returns `Self`.

Does not ignore `__init__` when `__new__` returns `Any`.

Unions overload return types.

+Partial

Does not include `__init__` when `__new__` returns `Self`.

Does not ignore `__init__` when `__new__` returns `Any`.

Unions overload return types.

Pass      constructors_consistency -
Pass*

Does not report inconsistency between __new__ and __init__ (optional).

+Pass*

Does not report inconsistency between __new__ and __init__ (optional).

Pass Pass Pass @@ -851,7 +851,7 @@

Python Type System Conformance Test Results

     overloads_basic Pass -
Partial

Does less literal promotion than the test asks for.

+Partial

Does less literal promotion than the test asks for.

Pass Pass Pass @@ -866,7 +866,7 @@

Python Type System Conformance Test Results

Pass      overloads_definitions -
Partial

Allows @override to be on all overloads and implementation, instead of just implementation.

+Partial

Allows @override to be on all overloads and implementation, instead of just implementation.

Pass Pass Pass @@ -874,18 +874,18 @@

Python Type System Conformance Test Results

Pass      overloads_definitions_stub -
Partial

Allows @override to appear in a stub file not on the first overload.

-
Unsupported

Does not support checking stubs.

+Partial

Allows @override to appear in a stub file not on the first overload.

+Unsupported

Does not support checking stubs.

Pass Pass Pass Pass      overloads_evaluation -
Partial

Does not expand boolean arguments to Literal[True] and Literal[False].

Does not expand enum arguments to literal variants.

Does not expand tuple arguments to possible combinations.

Does not evaluate Any in some cases where overload is ambiguous.

Evaluates Any in some cases where overload is not ambiguous.

+Partial

Does not expand boolean arguments to Literal[True] and Literal[False].

Does not expand enum arguments to literal variants.

Does not expand tuple arguments to possible combinations.

Does not evaluate Any in some cases where overload is ambiguous.

Evaluates Any in some cases where overload is not ambiguous.

Pass Pass -
Partial

Does not evaluate Any in some cases where overload is ambiguous.

+Partial

Does not evaluate Any in some cases where overload is ambiguous.

Pass Pass @@ -895,7 +895,7 @@

Python Type System Conformance Test Results

     exceptions_context_managers Pass Pass -
Partial

Some error suppressing context managers are not detected

+Partial

Some error suppressing context managers are not detected

Pass Unsupported Pass @@ -904,15 +904,15 @@

Python Type System Conformance Test Results

Dataclasses      dataclasses_descriptors -
Partial

Assumes descriptor behavior only when field is assigned in class body.

Does not correctly evaluate type of descriptor access.

-
Partial

Conformance suite is questionable; see https://github.com/python/typing/issues/2259

-
Partial

* Assumes descriptor behavior only when field is assigned in class body

* Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types

+Partial

Assumes descriptor behavior only when field is assigned in class body.

Does not correctly evaluate type of descriptor access.

+Partial

Conformance suite is questionable; see https://github.com/python/typing/issues/2259

+Partial

* Assumes descriptor behavior only when field is assigned in class body

* Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types

Pass -
Partial

Only infers a descriptor `__get__` method as being called when a descriptor attribute is accessed on an instance if the descriptor attribute is present in the class namespace.

+Partial

Only infers a descriptor `__get__` method as being called when a descriptor attribute is accessed on an instance if the descriptor attribute is present in the class namespace.

Pass      dataclasses_final -
Partial

Wrongly requires a Final dataclass field to be initialized at class level.

Doesn't support Final nested inside ClassVar.

+Partial

Wrongly requires a Final dataclass field to be initialized at class level.

Doesn't support Final nested inside ClassVar.

Pass Pass Pass @@ -928,11 +928,11 @@

Python Type System Conformance Test Results

Pass      dataclasses_hash -
Unsupported

Does not synthesize `__hash__ = None` as a class attribute for unhashable dataclasses.

Does not report when an unhashable dataclass has `__hash__` called directly on an instance.

Does not report when dataclass is not compatible with Hashable protocol.

+Unsupported

Does not synthesize `__hash__ = None` as a class attribute for unhashable dataclasses.

Does not report when an unhashable dataclass has `__hash__` called directly on an instance.

Does not report when dataclass is not compatible with Hashable protocol.

Pass Pass Pass -
Partial

Understands the `Hashable` protocol as equivalent to `object`.

+Partial

Understands the `Hashable` protocol as equivalent to `object`.

Pass      dataclasses_inheritance @@ -976,11 +976,11 @@

Python Type System Conformance Test Results

Pass      dataclasses_slots -
Partial

Does not reject write to instance variable that is not defined in __slots__.

+Partial

Does not reject write to instance variable that is not defined in __slots__.

Pass Pass Pass -
Partial

Synthesizes a `__slots__` attribute but does not validate attribute assignments against `__slots__`.

+Partial

Synthesizes a `__slots__` attribute but does not validate attribute assignments against `__slots__`.

Pass      dataclasses_transform_class @@ -992,7 +992,7 @@

Python Type System Conformance Test Results

Pass      dataclasses_transform_converter -
Unsupported

Converter parameter not yet supported.

+Unsupported

Converter parameter not yet supported.

Pass Pass Pass @@ -1000,7 +1000,7 @@

Python Type System Conformance Test Results

Pass      dataclasses_transform_field -
Partial

Does not properly handle field constructor that has default value for `kw_only` or `init` parameter.

+Partial

Does not properly handle field constructor that has default value for `kw_only` or `init` parameter.

Pass Pass Pass @@ -1008,7 +1008,7 @@

Python Type System Conformance Test Results

Pass      dataclasses_transform_func -
Partial

Does not handle `kw_only=False` override when `kw_only_default=True`.

Does not report error when `order=False` and comparison operators are used.

+Partial

Does not handle `kw_only=False` override when `kw_only_default=True`.

Does not report error when `order=False` and comparison operators are used.

Pass Pass Pass @@ -1024,7 +1024,7 @@

Python Type System Conformance Test Results

Pass      dataclasses_usage -
Pass*

Does not detect unannotated usage of `dataclasses.field()`.

+Pass*

Does not detect unannotated usage of `dataclasses.field()`.

Pass Pass Pass @@ -1035,7 +1035,7 @@

Python Type System Conformance Test Results

Typed dictionaries      typeddicts_alt_syntax -
Pass*

Does not support keyword-argument form of alternative syntax (deprecated in 3.11).

+Pass*

Does not support keyword-argument form of alternative syntax (deprecated in 3.11).

Pass Pass Pass @@ -1043,15 +1043,15 @@

Python Type System Conformance Test Results

Pass      typeddicts_class_syntax -
Partial

Does not support version-conditional items in TypedDict definitions.

+Partial

Does not support version-conditional items in TypedDict definitions.

Pass Pass -
Partial

Does not support version-conditional items in TypedDict definitions.

+Partial

Does not support version-conditional items in TypedDict definitions.

Pass Pass      typeddicts_extra_items -
Unsupported

Not supported.

+Unsupported

Not supported.

Pass Pass Pass @@ -1099,7 +1099,7 @@

Python Type System Conformance Test Results

Pass      typeddicts_readonly_inheritance -
Partial

Incorrectly rejects non-ReadOnly override of ReadOnly item.

Incorrectly rejects override of ReadOnly item with another ReadOnly item with narrower type.

Incorrectly rejects override of NotRequired ReadOnly item with a Required ReadOnly item.

+Partial

Incorrectly rejects non-ReadOnly override of ReadOnly item.

Incorrectly rejects override of ReadOnly item with another ReadOnly item with narrower type.

Incorrectly rejects override of NotRequired ReadOnly item with a Required ReadOnly item.

Pass Pass Pass @@ -1115,7 +1115,7 @@

Python Type System Conformance Test Results

Pass      typeddicts_readonly_update -
Partial

Incorrectly allows update of ReadOnly item.

Incorrectly rejects update involving an item with Never type.

+Partial

Incorrectly allows update of ReadOnly item.

Incorrectly rejects update involving an item with Never type.

Pass Pass Pass @@ -1132,7 +1132,7 @@

Python Type System Conformance Test Results

     typeddicts_type_consistency Pass -
Partial

Considers TypedDicts to be assignable to plain dict types.

+Partial

Considers TypedDicts to be assignable to plain dict types.

Pass Pass Pass @@ -1150,7 +1150,7 @@

Python Type System Conformance Test Results

Tuples      tuples_type_compat -
Partial

Does not support tuple narrowing based on `len()` type guard (optional).

Incorrectly marks a match case as unreachable.

+Partial

Does not support tuple narrowing based on `len()` type guard (optional).

Incorrectly marks a match case as unreachable.

Pass Pass Pass @@ -1166,7 +1166,7 @@

Python Type System Conformance Test Results

Pass      tuples_unpacked -
Partial

"More than one unpack" error is missing in some cases.

+Partial

"More than one unpack" error is missing in some cases.

Pass Pass Pass @@ -1177,8 +1177,8 @@

Python Type System Conformance Test Results

Named tuples      namedtuples_define_class -
Partial

Does not reject override of named tuple attribute in child class.

Does not support version-conditional fields.

-
Partial

Does not support precise type inference for slices over namedtuples.

+Partial

Does not reject override of named tuple attribute in child class.

Does not support version-conditional fields.

+Partial

Does not support precise type inference for slices over namedtuples.

Pass Pass Pass @@ -1201,7 +1201,7 @@

Python Type System Conformance Test Results

Pass      namedtuples_usage -
Partial

Does not reject attempt to delete named tuple field by name.

+Partial

Does not reject attempt to delete named tuple field by name.

Pass Pass Pass @@ -1221,22 +1221,22 @@

Python Type System Conformance Test Results

     enums_definition Pass -
Partial

Does not allow enum members to be conditional on version/platform checks.

+Partial

Does not allow enum members to be conditional on version/platform checks.

Pass Pass Pass Pass      enums_expansion -
Partial

Improperly applies narrowing to Flag subclass.

+Partial

Improperly applies narrowing to Flag subclass.

Pass Pass Pass -
Partial

Does not support `enum.Flag`.

+Partial

Does not support `enum.Flag`.

Pass      enums_member_names -
Pass*

Does not support special-cased handling of member name literal types in some cases (optional).

+Pass*

Does not support special-cased handling of member name literal types in some cases (optional).

Pass Pass Pass @@ -1244,7 +1244,7 @@

Python Type System Conformance Test Results

Pass      enums_member_values -
Partial

Does not enforce declared type of `_value_`.

Does not enforce assigned tuple types for enum members (optional).

+Partial

Does not enforce declared type of `_value_`.

Does not enforce assigned tuple types for enum members (optional).

Pass Pass Pass @@ -1252,10 +1252,10 @@

Python Type System Conformance Test Results

Pass      enums_members -
Partial

Does not treat attribute with annotation and no assignment as non-member.

Does not treat callables as non-members.

Does not honor `enum.member` as method decorator.

Does not properly handle aliased enum members.

Does not support `_ignore_` mechanism (optional).

Does not treat attributes with private names as non-members.

+Partial

Does not treat attribute with annotation and no assignment as non-member.

Does not treat callables as non-members.

Does not honor `enum.member` as method decorator.

Does not properly handle aliased enum members.

Does not support `_ignore_` mechanism (optional).

Does not treat attributes with private names as non-members.

Pass Pass -
Pass*

Does not support `_ignore_` mechanism (optional).

+Pass*

Does not support `_ignore_` mechanism (optional).

Pass Pass @@ -1275,7 +1275,7 @@

Python Type System Conformance Test Results

Pass Pass Pass -
Partial

Intersects the pre-existing type with the top materialization of the bracketed type rather than the bracketed type itself.

+Partial

Intersects the pre-existing type with the top materialization of the bracketed type rather than the bracketed type itself.

Pass @@ -1302,22 +1302,22 @@

Python Type System Conformance Test Results

Pass Pass Pass -
Partial

Does not detect calls to deprecated overloads.

Does not detect implicit calls to deprecated dunder methods, for example via operators.

Does not detect accesses of, or attempts to set, deprecated properties.

+Partial

Does not detect calls to deprecated overloads.

Does not detect implicit calls to deprecated dunder methods, for example via operators.

Does not detect accesses of, or attempts to set, deprecated properties.

Pass      directives_disjoint_base Pass -
Partial

Does not reject invalid class definitions due to disjoint bases, but uses disjoint base information in type narrowing.

+Partial

Does not reject invalid class definitions due to disjoint bases, but uses disjoint base information in type narrowing.

Pass -
Unsupported

Does not support PEP 800 disjoint-base semantics.

+Unsupported

Does not support PEP 800 disjoint-base semantics.

Pass Pass      directives_no_type_check -
Partial

Does not honor `@no_type_check` class decorator (allowed).

Does not reject invalid call of `@no_type_check` function.

+Partial

Does not honor `@no_type_check` class decorator (allowed).

Does not reject invalid call of `@no_type_check` function.

Pass Pass -
Pass*

Does not honor `@no_type_check` class decorator (allowed).

+Pass*

Does not honor `@no_type_check` class decorator (allowed).

Pass Pass @@ -1338,7 +1338,7 @@

Python Type System Conformance Test Results

Pass      directives_type_ignore -
Partial

Does not honor "# type: ignore" comment if comment includes additional text.

+Partial

Does not honor "# type: ignore" comment if comment includes additional text.

Pass Pass Pass @@ -1362,7 +1362,7 @@

Python Type System Conformance Test Results

Pass      directives_version_platform -
Pass*

Does not understand three-element form of sys.version checks.

Does not understand os.name checks.

+Pass*

Does not understand three-element form of sys.version checks.

Does not understand os.name checks.

Pass Pass Pass @@ -1373,7 +1373,7 @@

Python Type System Conformance Test Results

Historical and deprecated features      historical_positional -
Partial

Does not reject positional-only parameter after non-positional-only parameter.

Treats keyword-only parameter as positional-only.

Applies legacy positional-only rules when PEP 570 syntax is used.

+Partial

Does not reject positional-only parameter after non-positional-only parameter.

Treats keyword-only parameter as positional-only.

Applies legacy positional-only rules when PEP 570 syntax is used.

Pass Pass Pass diff --git a/conformance/src/reporting.py b/conformance/src/reporting.py index 066cbb9e4..089a63390 100644 --- a/conformance/src/reporting.py +++ b/conformance/src/reporting.py @@ -104,7 +104,7 @@ def generate_summary_html(root_dir: Path) -> str: [f"

{note}

" for note in raw_notes.split("\n")] ) - conformance_class = ( + conformance_classes = ( "conformant" if conformance == "Pass" else "partially-conformant" @@ -118,9 +118,10 @@ def generate_summary_html(root_dir: Path) -> str: conformance_cell = f"{conformance}" if raw_notes != "": - conformance_cell = f'
{conformance_cell}
{notes}
' + conformance_classes = f"{conformance_classes} tooltip" + conformance_cell = f'{conformance_cell}
{notes}
' - summary_html.append(f'{conformance_cell}') + summary_html.append(f'{conformance_cell}') summary_html.append("") diff --git a/conformance/src/results_template.html b/conformance/src/results_template.html index d19a0f7b5..c62b4e0a3 100644 --- a/conformance/src/results_template.html +++ b/conformance/src/results_template.html @@ -114,7 +114,7 @@ background-color: rgb(242, 171, 171); } - .tooltip-text { + .notes { visibility: hidden; position: absolute; z-index: 2; @@ -127,25 +127,25 @@ text-align: left; } - .hover-text:hover .tooltip-text { + .tooltip:hover .notes { visibility: visible; } - .hover-text .tooltip-text p { + .tooltip .notes p { margin: 4px; } - .hover-text .tooltip-text { + .tooltip .notes { top: 25px; left: -80%; } - .hover-text .tooltip-text::before { + .tooltip .notes::before { top: -5%; left: 94%; } - .hover-text { + .tooltip { cursor: help; position: relative; } From 2ae037180c8076f8ea4a4a83772b8ed2551d69fc Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:31 +0100 Subject: [PATCH 15/30] Use padding instead of non-breaking spaces --- conformance/results/results.html | 286 +++++++++++++------------- conformance/src/reporting.py | 2 +- conformance/src/results_template.html | 4 + 3 files changed, 150 insertions(+), 142 deletions(-) diff --git a/conformance/results/results.html b/conformance/results/results.html index b20b3424b..ffd6eb21a 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -82,6 +82,10 @@ border-bottom-width: 0; } + tbody > tr > th:not([colspan]) { + padding-left: 2em; + } + .col1 { width: 28%; } @@ -192,7 +196,7 @@

Python Type System Conformance Test Results

Type annotations -     annotations_coroutines +annotations_coroutines Pass Pass Pass @@ -200,7 +204,7 @@

Python Type System Conformance Test Results

Pass Pass -     annotations_forward_refs +annotations_forward_refs Partial

Does not report error for a forward reference that is not enclosed in quotes.

Does not report error for use of quoted type with "|" operator (runtime error).

Incorrectly generates error for quoted type defined in class scope.

Partial

Fails to reject "x" | int annotations that fail at runtime.

Rejects some valid quoted annotations.

Partial

Types in quotes incorrectly refer to shadowing class member.

Does not reject some type forms that require quotes.

@@ -208,7 +212,7 @@

Python Type System Conformance Test Results

Partial

Resolves references in type annotations as referring to end-of-scope types (https://discuss.python.org/t/annotation-string-references-in-class-scope-in-conformance-tests/105439, https://github.com/python/typing/pull/2144)

Partial

Incorrectly generates error for quoted type defined in class scope.

-     annotations_generators +annotations_generators Partial

Does not report incompatible Generator type in `yield from` statement.

Pass Pass @@ -216,7 +220,7 @@

Python Type System Conformance Test Results

Pass Pass -     annotations_methods +annotations_methods Pass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

Pass Pass @@ -224,7 +228,7 @@

Python Type System Conformance Test Results

Pass Pass -     annotations_typeexpr +annotations_typeexpr Pass Partial

Fails to reject various weird annotations

Pass @@ -235,7 +239,7 @@

Python Type System Conformance Test Results

Type forms -     typeforms_typeform +typeforms_typeform Partial

Does not support assigning Union and GenericAlias objects to their runtime types.

Partial

Fails to reject various weird annotations

Pass @@ -246,7 +250,7 @@

Python Type System Conformance Test Results

Special types in annotations -     specialtypes_any +specialtypes_any Pass Pass Pass @@ -254,7 +258,7 @@

Python Type System Conformance Test Results

Pass Pass -     specialtypes_never +specialtypes_never Pass Partial

Does not enforce invariance in some contexts

Does not allow Any to be assigned to Never

Pass @@ -262,7 +266,7 @@

Python Type System Conformance Test Results

Pass Pass -     specialtypes_none +specialtypes_none Pass Pass Pass @@ -270,7 +274,7 @@

Python Type System Conformance Test Results

Pass Pass -     specialtypes_promotions +specialtypes_promotions Pass Pass Pass @@ -278,7 +282,7 @@

Python Type System Conformance Test Results

Pass Pass -     specialtypes_type +specialtypes_type Partial

Does not treat `type` same as `type[Any]` for assert_type.

Does not allow access to unknown attributes from object of type `type[Any]`.

Pass Pass @@ -289,7 +293,7 @@

Python Type System Conformance Test Results

Generics -     generics_base_class +generics_base_class Partial

Does not detect inconsistent type variable ordering.

Pass Pass @@ -297,7 +301,7 @@

Python Type System Conformance Test Results

Pass Pass -     generics_basic +generics_basic Pass Pass Pass @@ -305,7 +309,7 @@

Python Type System Conformance Test Results

Partial

Incorrectly allows constrained type variables to be solved to a union of their constraints.

Pass -     generics_defaults +generics_defaults Partial

Does not detect a TypeVar with a default used after a TypeVarTuple.

Does not fully support defaults on TypeVarTuple and ParamSpec.

Partial

Numerous issues; does not support TypeVarTuple and ParamSpec defaults.

Pass @@ -313,7 +317,7 @@

Python Type System Conformance Test Results

Partial

Does not forbid a `TypeVar` immediately following a `TypeVarTuple` in a parameter list from having a default.

Does not support `TypeVarTuple`.

Pass -     generics_defaults_referential +generics_defaults_referential Partial

Does not correctly handle defaults referencing other TypeVars.

Partial

Fails to apply default specializations in some cases.

Pass @@ -321,7 +325,7 @@

Python Type System Conformance Test Results

Pass Pass -     generics_defaults_specialization +generics_defaults_specialization Partial

Does not correctly resolve defaults when classes are used directly.

Pass Pass @@ -329,7 +333,7 @@

Python Type System Conformance Test Results

Partial

Does not reject subscription of an already-specialized generic class.

Pass -     generics_paramspec_basic +generics_paramspec_basic Pass Pass Pass @@ -337,7 +341,7 @@

Python Type System Conformance Test Results

Pass Pass -     generics_paramspec_components +generics_paramspec_components Pass Pass Pass @@ -345,7 +349,7 @@

Python Type System Conformance Test Results

Partial

Incorrectly allows using `*args: P.args` and `**kwargs: P.kwargs` when `P` has not been put into scope by any other parameter annotation or enclosing scope.

Pass -     generics_paramspec_semantics +generics_paramspec_semantics Pass Pass Pass @@ -353,7 +357,7 @@

Python Type System Conformance Test Results

Pass Pass -     generics_paramspec_specialization +generics_paramspec_specialization Pass Pass Pass @@ -361,7 +365,7 @@

Python Type System Conformance Test Results

Pass Pass -     generics_scoping +generics_scoping Pass Pass Partial

Does not implement several scoping checks/restrictions for generics

@@ -369,7 +373,7 @@

Python Type System Conformance Test Results

Partial

Does not reject `list[T]()` in the global scope, where `T` is an unbound type variable.

Does nto reject `alias: TypeAlias = list[T]` in the body scope of a class generic over a type variable `T`.

Pass -     generics_self_advanced +generics_self_advanced Partial

Does not infer the type of an unannotated `self` parameter to be type `Self`.

Does not retain `Self` when calling method that returns `Self`.

Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`.

Does not retain `Self` when accessing attribute through `type[Self]`.

Pass Pass*

Treats attributes not initialized on the class as instance-only

@@ -377,7 +381,7 @@

Python Type System Conformance Test Results

Pass Pass -     generics_self_attributes +generics_self_attributes Pass Pass Pass @@ -385,7 +389,7 @@

Python Type System Conformance Test Results

Pass Pass -     generics_self_basic +generics_self_basic Pass Pass Pass @@ -393,7 +397,7 @@

Python Type System Conformance Test Results

Pass Pass -     generics_self_protocols +generics_self_protocols Pass Pass Pass @@ -401,7 +405,7 @@

Python Type System Conformance Test Results

Pass Pass -     generics_self_usage +generics_self_usage Pass Pass Pass @@ -409,7 +413,7 @@

Python Type System Conformance Test Results

Partial

Does not reject `Self` used in a return annotation when `self` is annotated using another type variable.

Pass -     generics_syntax_compatibility +generics_syntax_compatibility Pass Pass Pass @@ -417,7 +421,7 @@

Python Type System Conformance Test Results

Pass Pass -     generics_syntax_declarations +generics_syntax_declarations Pass Pass Pass @@ -425,7 +429,7 @@

Python Type System Conformance Test Results

Pass Pass -     generics_syntax_infer_variance +generics_syntax_infer_variance Unsupported

Type parameter syntax not yet supported.

Pass Pass @@ -433,7 +437,7 @@

Python Type System Conformance Test Results

Pass Pass -     generics_syntax_scoping +generics_syntax_scoping Partial

Does not following runtime scoping rules for type parameters in all cases.

Partial

Misses some details of scoping rules.

Pass @@ -441,7 +445,7 @@

Python Type System Conformance Test Results

Pass Pass -     generics_type_erasure +generics_type_erasure Partial

Infers Node[Never] instead of Node[Any] when argument is not provided.

False negative on instance attribute access on type(node).

Pass Pass @@ -449,7 +453,7 @@

Python Type System Conformance Test Results

Partial

Does not reject access of generic instance variable from the class object.

Pass -     generics_typevartuple_args +generics_typevartuple_args Partial

Does not enforce that tuples captured by TypeVarTuple are of the same length.

Pass Pass @@ -457,7 +461,7 @@

Python Type System Conformance Test Results

Partial

Supports PEP-646 unpacked tuples but not TypeVarTuple.

Pass -     generics_typevartuple_basic +generics_typevartuple_basic Partial

Does not enforce that tuples captured by TypeVarTuple are same length.

Pass Pass @@ -465,7 +469,7 @@

Python Type System Conformance Test Results

Unsupported Pass -     generics_typevartuple_callable +generics_typevartuple_callable Pass Pass Pass @@ -473,7 +477,7 @@

Python Type System Conformance Test Results

Unsupported Pass -     generics_typevartuple_concat +generics_typevartuple_concat Pass Pass Pass @@ -481,7 +485,7 @@

Python Type System Conformance Test Results

Unsupported Pass -     generics_typevartuple_overloads +generics_typevartuple_overloads Pass Pass Pass @@ -489,7 +493,7 @@

Python Type System Conformance Test Results

Pass Pass -     generics_typevartuple_specialization +generics_typevartuple_specialization Partial

Incorrectly specializes generic alias that includes a TypeVar and TypeVarTuple if no type arguments are provided.

Rejects specialization of generic type alias defined as a tuple containing a TypeVar.

Pass Pass @@ -497,7 +501,7 @@

Python Type System Conformance Test Results

Unsupported Pass -     generics_typevartuple_unpack +generics_typevartuple_unpack Pass Pass Pass @@ -505,7 +509,7 @@

Python Type System Conformance Test Results

Unsupported Pass -     generics_upper_bound +generics_upper_bound Partial

Does not reject use of type variable within an upper bound.

Pass Pass @@ -513,7 +517,7 @@

Python Type System Conformance Test Results

Pass Pass -     generics_variance +generics_variance Partial

Does not reject use of class-scoped TypeVar used in a base class when variance is incompatible.

Pass Pass @@ -521,7 +525,7 @@

Python Type System Conformance Test Results

Pass Pass -     generics_variance_inference +generics_variance_inference Pass Pass Pass @@ -532,7 +536,7 @@

Python Type System Conformance Test Results

Type qualifiers -     qualifiers_annotated +qualifiers_annotated Partial

Does not allow ClassVar to be nested within Annotated.

Does not allow Final to be nested within Annotated.

Does not allow Required and NotRequired to be nested within Annotated.

Does not reject type[T] compatibility for type alias defined with Annotated.

Does not reject call of type alias defined with Annotated.

Partial

Fails to reject various weird annotations.

False positive on lambda in Annotated.

Pass @@ -540,7 +544,7 @@

Python Type System Conformance Test Results

Pass Pass -     qualifiers_final_annotation +qualifiers_final_annotation Partial

Does not treat use of Final name as if it was replaced by the literal in NamedTuple definition.

Does not allow conditional assignment of Final instance variable in __init__ method.

Does not allow redefinition of private class variable that is marked Final in parent class.

Does not report modification of local Final variable via "for" statement.

Pass Pass @@ -548,7 +552,7 @@

Python Type System Conformance Test Results

Pass Pass -     qualifiers_final_decorator +qualifiers_final_decorator Pass Pass Pass @@ -559,7 +563,7 @@

Python Type System Conformance Test Results

Class type compatibility -     classes_classvar +classes_classvar Partial

Internal error if TypeVarTuple is used in ClassVar.

Does not reject use of ParamSpec in ClassVar.

Rejects ClassVar nested in Annotated.

Does not reject use of ClassVar in TypeAlias definition.

Pass Pass @@ -567,7 +571,7 @@

Python Type System Conformance Test Results

Pass Pass -     classes_override +classes_override Pass Pass Pass @@ -578,7 +582,7 @@

Python Type System Conformance Test Results

Type aliases -     aliases_explicit +aliases_explicit Partial

Does not reject specialization of type alias that has already been implicitly specialized.

Pass Pass @@ -586,7 +590,7 @@

Python Type System Conformance Test Results

Partial

Does not emit a diagnostic if a type alias to a union, in which every element of the union is implicitly specialized with `Unknown`, is invalidly specialized again.

Pass -     aliases_implicit +aliases_implicit Pass Partial

Fails to handle various weird annotations.

Various bugs with resolving generic aliases.

Pass @@ -594,7 +598,7 @@

Python Type System Conformance Test Results

Partial

Does not reject variables with `Any` or `Unknown` types when used as implicit type aliases.

Falls short on full syntactic validation of type aliases.

Pass -     aliases_newtype +aliases_newtype Partial

`NewType`s are incorrectly considered to be classes.

Pass Pass @@ -602,7 +606,7 @@

Python Type System Conformance Test Results

Pass Pass -     aliases_recursive +aliases_recursive Pass Pass Pass @@ -610,7 +614,7 @@

Python Type System Conformance Test Results

Unsupported Pass -     aliases_type_statement +aliases_type_statement Pass Partial

Fails to reject various weird annotations.

Pass @@ -618,7 +622,7 @@

Python Type System Conformance Test Results

Partial

Does not reject circular definitions of type aliases.

Does not support `type` statements generic over `TypeVarTuple`s.

Pass -     aliases_typealiastype +aliases_typealiastype Partial

Incorrectly rejects some recursive type aliases using TypeAliasType.

Incorrectly rejects the use of a class-scoped TypeVar in a TypeAliasType definition.

Partial

Rejects valid ParamSpec specialization.

Pass @@ -626,7 +630,7 @@

Python Type System Conformance Test Results

Partial

Does not reject specializing a type parameter in a generic type alias with a type inconsistent with the parameter's upper bound.

Does not reject declaring a type alias with a type variable that is not in scope.

Does not reject declaring a type alias with a non-literal tuple passed to the `type_params` parameter.

Does not reject cyclically defined type aliases.

Pass -     aliases_variance +aliases_variance Pass Pass Pass @@ -637,7 +641,7 @@

Python Type System Conformance Test Results

Literals -     literals_interactions +literals_interactions Pass*

Does not narrow `str` or `LiteralString` types to `Literal` string types via equality or containment checks.

Pass Pass @@ -645,7 +649,7 @@

Python Type System Conformance Test Results

Pass*

Deliberately does not allow `str` to be narrowed to literal string types through equality or containment checks due to the possibility of `str` subclasses that could have unexpected equality semantics.

Pass -     literals_literalstring +literals_literalstring Unsupported

Support for `LiteralString` is not implemented.

Pass Pass @@ -653,7 +657,7 @@

Python Type System Conformance Test Results

Pass Pass -     literals_parameterizations +literals_parameterizations Partial

Does not reject tuple within Literal.

Partial

Fails to reject various invalid literal parameterizations.

Pass @@ -661,7 +665,7 @@

Python Type System Conformance Test Results

Pass Pass -     literals_semantics +literals_semantics Pass Pass Pass @@ -672,7 +676,7 @@

Python Type System Conformance Test Results

Protocols -     protocols_class_objects +protocols_class_objects Pass Partial

Abstract type[Proto] still allows protocol class objects in some paths.

Pass @@ -680,7 +684,7 @@

Python Type System Conformance Test Results

Unsupported

`type[Proto]` is not yet supported.

`ClassVar` protocol members are not yet supported.

`@property` protocol members are only partially supported.

A class object `C` is only considered to inhabit a protocol type with a method member `f` if `f` exists as an attribute on the metaclass of `C`.

Pass -     protocols_definition +protocols_definition Partial

Does not detect protocol mismatch if concrete method is missing annotations.

Does not detect protocol mismatch if concrete method's parameters are position-only.

Pass Pass @@ -688,7 +692,7 @@

Python Type System Conformance Test Results

Partial

Does not reject implicit instance attributes in `Protocol` methods.

Does not support `ClassVar` protocol members.

Incorrectly considers `ClassVar` attributes on concrete classes as satisfying non-`ClassVar` attribute members on protocols.

Only has partial support for `@property` protocol members.

Pass -     protocols_explicit +protocols_explicit Pass*

Does not report unimplemented attributes for class that explicitly derives from protocol until it is instantiated.

Pass Pass @@ -696,7 +700,7 @@

Python Type System Conformance Test Results

Unsupported

Allows implicitly abstract protocol methods to be called via `super()` on a protocol subclass.

Allows instantiation of abstract subclasses of protocol classes.

Pass -     protocols_generic +protocols_generic Pass Partial

Fails to reject duplicate generic/protocol bases.

Treats global object as a literal.

Pass @@ -704,7 +708,7 @@

Python Type System Conformance Test Results

Partial

Only partially supports `@property` protocol members.

Pass -     protocols_merging +protocols_merging Pass Pass Pass @@ -712,7 +716,7 @@

Python Type System Conformance Test Results

Partial

Does not reject attempted instantiation of abstract subclasses of protocols.

Pass -     protocols_modules +protocols_modules Pass Pass Pass @@ -720,7 +724,7 @@

Python Type System Conformance Test Results

Partial

Never considers a module as satisfying a protocol with a method member due to the fact that the method will never exist on the class `types.ModuleType`, only on a given specific module instance.

Pass -     protocols_recursive +protocols_recursive Pass Pass Pass @@ -728,7 +732,7 @@

Python Type System Conformance Test Results

Partial

Fails to solve a type variable involving a recursive generic protocol.

Pass -     protocols_runtime_checkable +protocols_runtime_checkable Partial

Does not report unsafe overlap for runtime_checkable protocol.

Pass Pass @@ -736,7 +740,7 @@

Python Type System Conformance Test Results

Partial

Does not reject `isinstance()` or `issubclass()` calls against runtime-checkable protocols where there is an unsafe overlap between the type of the first argument and the protocol.

Pass -     protocols_self +protocols_self Pass Pass Pass @@ -744,7 +748,7 @@

Python Type System Conformance Test Results

Pass Pass -     protocols_subtyping +protocols_subtyping Pass Pass Pass @@ -752,7 +756,7 @@

Python Type System Conformance Test Results

Pass Pass -     protocols_variance +protocols_variance Pass Pass Pass @@ -763,7 +767,7 @@

Python Type System Conformance Test Results

Callables -     callables_annotation +callables_annotation Partial

Incorrectly treats "*args: T, **kwargs: T" as "..." when T is specialized to Any.

Does not treat "*args: Any, **kargs: Any" as "..." when separated by keyword parameter.

Pass Partial

Parameter names are lost when resolving ParamSpec

@@ -771,7 +775,7 @@

Python Type System Conformance Test Results

Partial

Infers a callback protocol as being a gradual type if the callback has signature `__call__[T](self, *args: T, **kwargs: T)` and `T` has been explicitly specialized to `Any`.

Pass -     callables_kwargs +callables_kwargs Partial

Allows callable without kwargs to be assigned to callable with unpacked kwargs

Pass Pass @@ -779,7 +783,7 @@

Python Type System Conformance Test Results

Pass Pass -     callables_protocol +callables_protocol Pass Pass Pass @@ -787,7 +791,7 @@

Python Type System Conformance Test Results

Pass Pass -     callables_subtyping +callables_subtyping Pass Pass Pass @@ -798,7 +802,7 @@

Python Type System Conformance Test Results

Constructors -     constructors_call_init +constructors_call_init Partial

Does not report errors during binding to self parameter of __init__ method.

Does not reject use of class-scoped type variables in annotation of self parameter in __init__ method.

Pass Pass @@ -806,7 +810,7 @@

Python Type System Conformance Test Results

Partial

Does not reject invalid argument types to an inherited constructor in a specialized subclass of a generic superclass.

Does not reject class-scoped type variables used in the `self` annotation.

Pass -     constructors_call_metaclass +constructors_call_metaclass Unsupported

Does not honor metaclass __call__ method when evaluating constructor call.

Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

Pass Pass @@ -814,7 +818,7 @@

Python Type System Conformance Test Results

Pass Pass -     constructors_call_new +constructors_call_new Partial

Does not support __new__ return type that is not a subclass of the class being constructed.

Does not skip evaluation of __init__ based on __new__ return type.

Does not report errors during binding to cls parameter of __new__ method.

Pass Pass @@ -822,7 +826,7 @@

Python Type System Conformance Test Results

Pass Pass -     constructors_call_type +constructors_call_type Partial

Does not validate call to custom metaclass __call__ method through type[T].

Pass Pass @@ -830,7 +834,7 @@

Python Type System Conformance Test Results

Partial

Has overly lenient handling of calls to `type[T]` if `T` is a type variable without an upper bound.

Pass -     constructors_callable +constructors_callable Partial

Does not generate a union type for __new__ and __init__ when converting class to callable.

Does not ignore __init__ based on __new__ return type when converting class to callable.

Does not support __new__ return type that is different from class being constructed.

Pass Partial

Converting constructor to callable does not preserve class-scoped type params.

@@ -838,7 +842,7 @@

Python Type System Conformance Test Results

Partial

Does not include `__init__` when `__new__` returns `Self`.

Does not ignore `__init__` when `__new__` returns `Any`.

Unions overload return types.

Pass -     constructors_consistency +constructors_consistency Pass*

Does not report inconsistency between __new__ and __init__ (optional).

Pass Pass @@ -849,7 +853,7 @@

Python Type System Conformance Test Results

Overloads -     overloads_basic +overloads_basic Pass Partial

Does less literal promotion than the test asks for.

Pass @@ -857,7 +861,7 @@

Python Type System Conformance Test Results

Pass Pass -     overloads_consistency +overloads_consistency Pass Pass Pass @@ -865,7 +869,7 @@

Python Type System Conformance Test Results

Pass Pass -     overloads_definitions +overloads_definitions Partial

Allows @override to be on all overloads and implementation, instead of just implementation.

Pass Pass @@ -873,7 +877,7 @@

Python Type System Conformance Test Results

Pass Pass -     overloads_definitions_stub +overloads_definitions_stub Partial

Allows @override to appear in a stub file not on the first overload.

Unsupported

Does not support checking stubs.

Pass @@ -881,7 +885,7 @@

Python Type System Conformance Test Results

Pass Pass -     overloads_evaluation +overloads_evaluation Partial

Does not expand boolean arguments to Literal[True] and Literal[False].

Does not expand enum arguments to literal variants.

Does not expand tuple arguments to possible combinations.

Does not evaluate Any in some cases where overload is ambiguous.

Evaluates Any in some cases where overload is not ambiguous.

Pass Pass @@ -892,7 +896,7 @@

Python Type System Conformance Test Results

Exceptions -     exceptions_context_managers +exceptions_context_managers Pass Pass Partial

Some error suppressing context managers are not detected

@@ -903,7 +907,7 @@

Python Type System Conformance Test Results

Dataclasses -     dataclasses_descriptors +dataclasses_descriptors Partial

Assumes descriptor behavior only when field is assigned in class body.

Does not correctly evaluate type of descriptor access.

Partial

Conformance suite is questionable; see https://github.com/python/typing/issues/2259

Partial

* Assumes descriptor behavior only when field is assigned in class body

* Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types

@@ -911,7 +915,7 @@

Python Type System Conformance Test Results

Partial

Only infers a descriptor `__get__` method as being called when a descriptor attribute is accessed on an instance if the descriptor attribute is present in the class namespace.

Pass -     dataclasses_final +dataclasses_final Partial

Wrongly requires a Final dataclass field to be initialized at class level.

Doesn't support Final nested inside ClassVar.

Pass Pass @@ -919,7 +923,7 @@

Python Type System Conformance Test Results

Pass Pass -     dataclasses_frozen +dataclasses_frozen Pass Pass Pass @@ -927,7 +931,7 @@

Python Type System Conformance Test Results

Pass Pass -     dataclasses_hash +dataclasses_hash Unsupported

Does not synthesize `__hash__ = None` as a class attribute for unhashable dataclasses.

Does not report when an unhashable dataclass has `__hash__` called directly on an instance.

Does not report when dataclass is not compatible with Hashable protocol.

Pass Pass @@ -935,7 +939,7 @@

Python Type System Conformance Test Results

Partial

Understands the `Hashable` protocol as equivalent to `object`.

Pass -     dataclasses_inheritance +dataclasses_inheritance Pass Pass Pass @@ -943,7 +947,7 @@

Python Type System Conformance Test Results

Pass Pass -     dataclasses_kwonly +dataclasses_kwonly Pass Pass Pass @@ -951,7 +955,7 @@

Python Type System Conformance Test Results

Pass Pass -     dataclasses_match_args +dataclasses_match_args Pass Pass Pass @@ -959,7 +963,7 @@

Python Type System Conformance Test Results

Pass Pass -     dataclasses_order +dataclasses_order Pass Pass Pass @@ -967,7 +971,7 @@

Python Type System Conformance Test Results

Pass Pass -     dataclasses_postinit +dataclasses_postinit Pass Pass Pass @@ -975,7 +979,7 @@

Python Type System Conformance Test Results

Pass Pass -     dataclasses_slots +dataclasses_slots Partial

Does not reject write to instance variable that is not defined in __slots__.

Pass Pass @@ -983,7 +987,7 @@

Python Type System Conformance Test Results

Partial

Synthesizes a `__slots__` attribute but does not validate attribute assignments against `__slots__`.

Pass -     dataclasses_transform_class +dataclasses_transform_class Pass Pass Pass @@ -991,7 +995,7 @@

Python Type System Conformance Test Results

Pass Pass -     dataclasses_transform_converter +dataclasses_transform_converter Unsupported

Converter parameter not yet supported.

Pass Pass @@ -999,7 +1003,7 @@

Python Type System Conformance Test Results

Pass Pass -     dataclasses_transform_field +dataclasses_transform_field Partial

Does not properly handle field constructor that has default value for `kw_only` or `init` parameter.

Pass Pass @@ -1007,7 +1011,7 @@

Python Type System Conformance Test Results

Pass Pass -     dataclasses_transform_func +dataclasses_transform_func Partial

Does not handle `kw_only=False` override when `kw_only_default=True`.

Does not report error when `order=False` and comparison operators are used.

Pass Pass @@ -1015,7 +1019,7 @@

Python Type System Conformance Test Results

Pass Pass -     dataclasses_transform_meta +dataclasses_transform_meta Pass Pass Pass @@ -1023,7 +1027,7 @@

Python Type System Conformance Test Results

Pass Pass -     dataclasses_usage +dataclasses_usage Pass*

Does not detect unannotated usage of `dataclasses.field()`.

Pass Pass @@ -1034,7 +1038,7 @@

Python Type System Conformance Test Results

Typed dictionaries -     typeddicts_alt_syntax +typeddicts_alt_syntax Pass*

Does not support keyword-argument form of alternative syntax (deprecated in 3.11).

Pass Pass @@ -1042,7 +1046,7 @@

Python Type System Conformance Test Results

Pass Pass -     typeddicts_class_syntax +typeddicts_class_syntax Partial

Does not support version-conditional items in TypedDict definitions.

Pass Pass @@ -1050,7 +1054,7 @@

Python Type System Conformance Test Results

Pass Pass -     typeddicts_extra_items +typeddicts_extra_items Unsupported

Not supported.

Pass Pass @@ -1058,7 +1062,7 @@

Python Type System Conformance Test Results

Pass Pass -     typeddicts_final +typeddicts_final Pass Pass Pass @@ -1066,7 +1070,7 @@

Python Type System Conformance Test Results

Pass Pass -     typeddicts_inheritance +typeddicts_inheritance Pass Pass Pass @@ -1074,7 +1078,7 @@

Python Type System Conformance Test Results

Pass Pass -     typeddicts_operations +typeddicts_operations Pass Pass Pass @@ -1082,7 +1086,7 @@

Python Type System Conformance Test Results

Pass Pass -     typeddicts_readonly +typeddicts_readonly Pass Pass Pass @@ -1090,7 +1094,7 @@

Python Type System Conformance Test Results

Pass Pass -     typeddicts_readonly_consistency +typeddicts_readonly_consistency Pass Pass Pass @@ -1098,7 +1102,7 @@

Python Type System Conformance Test Results

Pass Pass -     typeddicts_readonly_inheritance +typeddicts_readonly_inheritance Partial

Incorrectly rejects non-ReadOnly override of ReadOnly item.

Incorrectly rejects override of ReadOnly item with another ReadOnly item with narrower type.

Incorrectly rejects override of NotRequired ReadOnly item with a Required ReadOnly item.

Pass Pass @@ -1106,7 +1110,7 @@

Python Type System Conformance Test Results

Pass Pass -     typeddicts_readonly_kwargs +typeddicts_readonly_kwargs Pass Pass Pass @@ -1114,7 +1118,7 @@

Python Type System Conformance Test Results

Pass Pass -     typeddicts_readonly_update +typeddicts_readonly_update Partial

Incorrectly allows update of ReadOnly item.

Incorrectly rejects update involving an item with Never type.

Pass Pass @@ -1122,7 +1126,7 @@

Python Type System Conformance Test Results

Pass Pass -     typeddicts_required +typeddicts_required Pass Pass Pass @@ -1130,7 +1134,7 @@

Python Type System Conformance Test Results

Pass Pass -     typeddicts_type_consistency +typeddicts_type_consistency Pass Partial

Considers TypedDicts to be assignable to plain dict types.

Pass @@ -1138,7 +1142,7 @@

Python Type System Conformance Test Results

Pass Pass -     typeddicts_usage +typeddicts_usage Pass Pass Pass @@ -1149,7 +1153,7 @@

Python Type System Conformance Test Results

Tuples -     tuples_type_compat +tuples_type_compat Partial

Does not support tuple narrowing based on `len()` type guard (optional).

Incorrectly marks a match case as unreachable.

Pass Pass @@ -1157,7 +1161,7 @@

Python Type System Conformance Test Results

Pass Pass -     tuples_type_form +tuples_type_form Pass Pass Pass @@ -1165,7 +1169,7 @@

Python Type System Conformance Test Results

Pass Pass -     tuples_unpacked +tuples_unpacked Partial

"More than one unpack" error is missing in some cases.

Pass Pass @@ -1176,7 +1180,7 @@

Python Type System Conformance Test Results

Named tuples -     namedtuples_define_class +namedtuples_define_class Partial

Does not reject override of named tuple attribute in child class.

Does not support version-conditional fields.

Partial

Does not support precise type inference for slices over namedtuples.

Pass @@ -1184,7 +1188,7 @@

Python Type System Conformance Test Results

Pass Pass -     namedtuples_define_functional +namedtuples_define_functional Pass Pass Pass @@ -1192,7 +1196,7 @@

Python Type System Conformance Test Results

Pass Pass -     namedtuples_type_compat +namedtuples_type_compat Pass Pass Pass @@ -1200,7 +1204,7 @@

Python Type System Conformance Test Results

Pass Pass -     namedtuples_usage +namedtuples_usage Partial

Does not reject attempt to delete named tuple field by name.

Pass Pass @@ -1211,7 +1215,7 @@

Python Type System Conformance Test Results

Enumerations -     enums_behaviors +enums_behaviors Pass Pass Pass @@ -1219,7 +1223,7 @@

Python Type System Conformance Test Results

Pass Pass -     enums_definition +enums_definition Pass Partial

Does not allow enum members to be conditional on version/platform checks.

Pass @@ -1227,7 +1231,7 @@

Python Type System Conformance Test Results

Pass Pass -     enums_expansion +enums_expansion Partial

Improperly applies narrowing to Flag subclass.

Pass Pass @@ -1235,7 +1239,7 @@

Python Type System Conformance Test Results

Partial

Does not support `enum.Flag`.

Pass -     enums_member_names +enums_member_names Pass*

Does not support special-cased handling of member name literal types in some cases (optional).

Pass Pass @@ -1243,7 +1247,7 @@

Python Type System Conformance Test Results

Pass Pass -     enums_member_values +enums_member_values Partial

Does not enforce declared type of `_value_`.

Does not enforce assigned tuple types for enum members (optional).

Pass Pass @@ -1251,7 +1255,7 @@

Python Type System Conformance Test Results

Pass Pass -     enums_members +enums_members Partial

Does not treat attribute with annotation and no assignment as non-member.

Does not treat callables as non-members.

Does not honor `enum.member` as method decorator.

Does not properly handle aliased enum members.

Does not support `_ignore_` mechanism (optional).

Does not treat attributes with private names as non-members.

Pass Pass @@ -1262,7 +1266,7 @@

Python Type System Conformance Test Results

Type narrowing -     narrowing_typeguard +narrowing_typeguard Pass Pass Pass @@ -1270,7 +1274,7 @@

Python Type System Conformance Test Results

Pass Pass -     narrowing_typeis +narrowing_typeis Pass Pass Pass @@ -1281,7 +1285,7 @@

Python Type System Conformance Test Results

Type checker directives -     directives_assert_type +directives_assert_type Pass Pass Pass @@ -1289,7 +1293,7 @@

Python Type System Conformance Test Results

Pass Pass -     directives_cast +directives_cast Pass Pass Pass @@ -1297,7 +1301,7 @@

Python Type System Conformance Test Results

Pass Pass -     directives_deprecated +directives_deprecated Pass Pass Pass @@ -1305,7 +1309,7 @@

Python Type System Conformance Test Results

Partial

Does not detect calls to deprecated overloads.

Does not detect implicit calls to deprecated dunder methods, for example via operators.

Does not detect accesses of, or attempts to set, deprecated properties.

Pass -     directives_disjoint_base +directives_disjoint_base Pass Partial

Does not reject invalid class definitions due to disjoint bases, but uses disjoint base information in type narrowing.

Pass @@ -1313,7 +1317,7 @@

Python Type System Conformance Test Results

Pass Pass -     directives_no_type_check +directives_no_type_check Partial

Does not honor `@no_type_check` class decorator (allowed).

Does not reject invalid call of `@no_type_check` function.

Pass Pass @@ -1321,7 +1325,7 @@

Python Type System Conformance Test Results

Pass Pass -     directives_reveal_type +directives_reveal_type Pass Pass Pass @@ -1329,7 +1333,7 @@

Python Type System Conformance Test Results

Pass Pass -     directives_type_checking +directives_type_checking Pass Pass Pass @@ -1337,7 +1341,7 @@

Python Type System Conformance Test Results

Pass Pass -     directives_type_ignore +directives_type_ignore Partial

Does not honor "# type: ignore" comment if comment includes additional text.

Pass Pass @@ -1345,7 +1349,7 @@

Python Type System Conformance Test Results

Pass Pass -     directives_type_ignore_file1 +directives_type_ignore_file1 Pass Pass Pass @@ -1353,7 +1357,7 @@

Python Type System Conformance Test Results

Pass Pass -     directives_type_ignore_file2 +directives_type_ignore_file2 Pass Pass Pass @@ -1361,7 +1365,7 @@

Python Type System Conformance Test Results

Pass Pass -     directives_version_platform +directives_version_platform Pass*

Does not understand three-element form of sys.version checks.

Does not understand os.name checks.

Pass Pass @@ -1372,7 +1376,7 @@

Python Type System Conformance Test Results

Historical and deprecated features -     historical_positional +historical_positional Partial

Does not reject positional-only parameter after non-positional-only parameter.

Treats keyword-only parameter as positional-only.

Applies legacy positional-only rules when PEP 570 syntax is used.

Pass Pass diff --git a/conformance/src/reporting.py b/conformance/src/reporting.py index 089a63390..1ba95e9c8 100644 --- a/conformance/src/reporting.py +++ b/conformance/src/reporting.py @@ -77,7 +77,7 @@ def generate_summary_html(root_dir: Path) -> str: for test_case in tests_in_group: test_case_name = test_case.stem - summary_html.append(f'     {test_case_name}') + summary_html.append(f"{test_case_name}") for type_checker in type_checkers: try: diff --git a/conformance/src/results_template.html b/conformance/src/results_template.html index c62b4e0a3..72b27e16e 100644 --- a/conformance/src/results_template.html +++ b/conformance/src/results_template.html @@ -82,6 +82,10 @@ border-bottom-width: 0; } + tbody > tr > th:not([colspan]) { + padding-left: 2em; + } + .col1 { width: 28%; } From 08bcbffc3026d056a6bbde1a49e83bdd491dbbba Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:31 +0100 Subject: [PATCH 16/30] Remove another unnecessary class --- conformance/results/results.html | 52 +++++++++++++-------------- conformance/src/reporting.py | 4 +-- conformance/src/results_template.html | 10 +++--- 3 files changed, 32 insertions(+), 34 deletions(-) diff --git a/conformance/results/results.html b/conformance/results/results.html index ffd6eb21a..75785d3b3 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -86,6 +86,11 @@ padding-left: 2em; } + tbody > tr > th[colspan] > a { + font-size: 10pt; + font-weight: bold; + } + .col1 { width: 28%; } @@ -101,11 +106,6 @@ text-align: center; } - .test_group { - font-size: 10pt; - font-weight: bold; - } - .conformant { background-color: rgb(203, 255, 203); } @@ -194,7 +194,7 @@

Python Type System Conformance Test Results

-Type annotations +Type annotations annotations_coroutines Pass @@ -237,7 +237,7 @@

Python Type System Conformance Test Results

Pass -Type forms +Type forms typeforms_typeform Partial

Does not support assigning Union and GenericAlias objects to their runtime types.

@@ -248,7 +248,7 @@

Python Type System Conformance Test Results

Pass -Special types in annotations +Special types in annotations specialtypes_any Pass @@ -291,7 +291,7 @@

Python Type System Conformance Test Results

Pass -Generics +Generics generics_base_class Partial

Does not detect inconsistent type variable ordering.

@@ -534,7 +534,7 @@

Python Type System Conformance Test Results

Pass -Type qualifiers +Type qualifiers qualifiers_annotated Partial

Does not allow ClassVar to be nested within Annotated.

Does not allow Final to be nested within Annotated.

Does not allow Required and NotRequired to be nested within Annotated.

Does not reject type[T] compatibility for type alias defined with Annotated.

Does not reject call of type alias defined with Annotated.

@@ -561,7 +561,7 @@

Python Type System Conformance Test Results

Pass -Class type compatibility +Class type compatibility classes_classvar Partial

Internal error if TypeVarTuple is used in ClassVar.

Does not reject use of ParamSpec in ClassVar.

Rejects ClassVar nested in Annotated.

Does not reject use of ClassVar in TypeAlias definition.

@@ -580,7 +580,7 @@

Python Type System Conformance Test Results

Pass -Type aliases +Type aliases aliases_explicit Partial

Does not reject specialization of type alias that has already been implicitly specialized.

@@ -639,7 +639,7 @@

Python Type System Conformance Test Results

Pass -Literals +Literals literals_interactions Pass*

Does not narrow `str` or `LiteralString` types to `Literal` string types via equality or containment checks.

@@ -674,7 +674,7 @@

Python Type System Conformance Test Results

Pass -Protocols +Protocols protocols_class_objects Pass @@ -765,7 +765,7 @@

Python Type System Conformance Test Results

Pass -Callables +Callables callables_annotation Partial

Incorrectly treats "*args: T, **kwargs: T" as "..." when T is specialized to Any.

Does not treat "*args: Any, **kargs: Any" as "..." when separated by keyword parameter.

@@ -800,7 +800,7 @@

Python Type System Conformance Test Results

Pass -Constructors +Constructors constructors_call_init Partial

Does not report errors during binding to self parameter of __init__ method.

Does not reject use of class-scoped type variables in annotation of self parameter in __init__ method.

@@ -851,7 +851,7 @@

Python Type System Conformance Test Results

Pass -Overloads +Overloads overloads_basic Pass @@ -894,7 +894,7 @@

Python Type System Conformance Test Results

Pass -Exceptions +Exceptions exceptions_context_managers Pass @@ -905,7 +905,7 @@

Python Type System Conformance Test Results

Pass -Dataclasses +Dataclasses dataclasses_descriptors Partial

Assumes descriptor behavior only when field is assigned in class body.

Does not correctly evaluate type of descriptor access.

@@ -1036,7 +1036,7 @@

Python Type System Conformance Test Results

Pass -Typed dictionaries +Typed dictionaries typeddicts_alt_syntax Pass*

Does not support keyword-argument form of alternative syntax (deprecated in 3.11).

@@ -1151,7 +1151,7 @@

Python Type System Conformance Test Results

Pass -Tuples +Tuples tuples_type_compat Partial

Does not support tuple narrowing based on `len()` type guard (optional).

Incorrectly marks a match case as unreachable.

@@ -1178,7 +1178,7 @@

Python Type System Conformance Test Results

Pass -Named tuples +Named tuples namedtuples_define_class Partial

Does not reject override of named tuple attribute in child class.

Does not support version-conditional fields.

@@ -1213,7 +1213,7 @@

Python Type System Conformance Test Results

Pass -Enumerations +Enumerations enums_behaviors Pass @@ -1264,7 +1264,7 @@

Python Type System Conformance Test Results

Pass -Type narrowing +Type narrowing narrowing_typeguard Pass @@ -1283,7 +1283,7 @@

Python Type System Conformance Test Results

Pass -Type checker directives +Type checker directives directives_assert_type Pass @@ -1374,7 +1374,7 @@

Python Type System Conformance Test Results

Pass -Historical and deprecated features +Historical and deprecated features historical_positional Partial

Does not reject positional-only parameter after non-positional-only parameter.

Treats keyword-only parameter as positional-only.

Applies legacy positional-only rules when PEP 570 syntax is used.

diff --git a/conformance/src/reporting.py b/conformance/src/reporting.py index 1ba95e9c8..7d40d97a9 100644 --- a/conformance/src/reporting.py +++ b/conformance/src/reporting.py @@ -69,9 +69,7 @@ def generate_summary_html(root_dir: Path) -> str: # Are there any test cases in this group? if len(tests_in_group) > 0: summary_html.append(f'') - summary_html.append( - f'{test_group.name}' - ) + summary_html.append(f'{test_group.name}') summary_html.append("") for test_case in tests_in_group: diff --git a/conformance/src/results_template.html b/conformance/src/results_template.html index 72b27e16e..0bcd14262 100644 --- a/conformance/src/results_template.html +++ b/conformance/src/results_template.html @@ -86,6 +86,11 @@ padding-left: 2em; } + tbody > tr > th[colspan] > a { + font-size: 10pt; + font-weight: bold; + } + .col1 { width: 28%; } @@ -101,11 +106,6 @@ text-align: center; } - .test_group { - font-size: 10pt; - font-weight: bold; - } - .conformant { background-color: rgb(203, 255, 203); } From e9363e661076f6b7f5392f8ea435d94b090d5661 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:31 +0100 Subject: [PATCH 17/30] Move base template to a folder --- conformance/src/reporting.py | 2 +- conformance/src/{results_template.html => templates/base.html} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename conformance/src/{results_template.html => templates/base.html} (100%) diff --git a/conformance/src/reporting.py b/conformance/src/reporting.py index 7d40d97a9..6fafa8626 100644 --- a/conformance/src/reporting.py +++ b/conformance/src/reporting.py @@ -12,7 +12,7 @@ def generate_summary(root_dir: Path): print("Generating summary report") - template_file = root_dir / "src" / "results_template.html" + template_file = root_dir / "src" / "templates" / "base.html" with open(template_file, "r") as f: template = f.read() diff --git a/conformance/src/results_template.html b/conformance/src/templates/base.html similarity index 100% rename from conformance/src/results_template.html rename to conformance/src/templates/base.html From 05e4b149db335eac0625c4f0b7868d2dfdea8974 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:32 +0100 Subject: [PATCH 18/30] Add jinja2 as a dependency --- conformance/pyproject.toml | 3 ++- conformance/uv.lock | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/conformance/pyproject.toml b/conformance/pyproject.toml index 5d6133596..428adad56 100644 --- a/conformance/pyproject.toml +++ b/conformance/pyproject.toml @@ -3,13 +3,14 @@ name = "typing-conformance" version = "0.1.0" requires-python = "==3.12.*" dependencies = [ + "jinja2", "mypy", "pycroscope", "pyrefly", "pyright", "tomlkit", - "zuban", "ty", + "zuban", ] [tool.uv] diff --git a/conformance/uv.lock b/conformance/uv.lock index 770b62411..e66d5428d 100644 --- a/conformance/uv.lock +++ b/conformance/uv.lock @@ -35,6 +35,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/db/55a262f3606bebcae07cc14095338471ad7c0bbcaa37707e6f0ee49725b7/importlib_resources-7.1.0-py3-none-any.whl", hash = "sha256:1bd7b48b4088eddb2cd16382150bb515af0bd2c70128194392725f82ad2c96a1", size = 37232, upload-time = "2026-04-12T16:36:08.219Z" }, ] +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + [[package]] name = "librt" version = "0.11.0" @@ -56,6 +68,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/59/67/a6739ac96e28b7855808bdb0370e250606104a859750d209e5a0716fe7ab/librt-0.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:2f10cf143e4a9bb0f4f5af568a00df94a2d69ef41c2579584454bb0fe5cc642c", size = 103470, upload-time = "2026-05-10T18:16:10.369Z" }, ] +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, +] + [[package]] name = "mypy" version = "2.1.0" @@ -203,6 +234,7 @@ name = "typing-conformance" version = "0.1.0" source = { virtual = "." } dependencies = [ + { name = "jinja2" }, { name = "mypy" }, { name = "pycroscope" }, { name = "pyrefly" }, @@ -214,6 +246,7 @@ dependencies = [ [package.metadata] requires-dist = [ + { name = "jinja2" }, { name = "mypy" }, { name = "pycroscope" }, { name = "pyrefly" }, From faf18d1103377fcc2fffa641c1a791f5e2586cd7 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:32 +0100 Subject: [PATCH 19/30] Switch to using jinja2 templates --- conformance/results/results.html | 3684 +++++++++++++++++---------- conformance/src/main.py | 22 +- conformance/src/reporting.py | 230 +- conformance/src/templates/base.html | 250 +- conformance/src/templates/style.css | 162 ++ 5 files changed, 2686 insertions(+), 1662 deletions(-) create mode 100644 conformance/src/templates/style.css diff --git a/conformance/results/results.html b/conformance/results/results.html index 75785d3b3..bb3c7989d 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -1,1393 +1,2347 @@ - + + + + + Type System Test Results + - - - -
-
-

Python Type System Conformance Test Results

-

- While spec conformance is important for the ecosystem, we don't recommend using it - as the primary basis for choosing a type checker. It is not representative - of many of the things users typically care about. -

-
-
- - --- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
mypy 2.1.0pycroscope 0.4.0pyrefly 1.1.0pyright 1.1.410ty 0.0.50zuban 0.8.2
-Type annotations -
annotations_coroutinesPassPassPassPassPassPass
annotations_forward_refsPartial

Does not report error for a forward reference that is not enclosed in quotes.

Does not report error for use of quoted type with "|" operator (runtime error).

Incorrectly generates error for quoted type defined in class scope.

Partial

Fails to reject "x" | int annotations that fail at runtime.

Rejects some valid quoted annotations.

Partial

Types in quotes incorrectly refer to shadowing class member.

Does not reject some type forms that require quotes.

PassPartial

Resolves references in type annotations as referring to end-of-scope types (https://discuss.python.org/t/annotation-string-references-in-class-scope-in-conformance-tests/105439, https://github.com/python/typing/pull/2144)

Partial

Incorrectly generates error for quoted type defined in class scope.

annotations_generatorsPartial

Does not report incompatible Generator type in `yield from` statement.

PassPassPassPassPass
annotations_methodsPass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

PassPassPass*

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

PassPass
annotations_typeexprPassPartial

Fails to reject various weird annotations

PassPassPassPass
-Type forms -
typeforms_typeformPartial

Does not support assigning Union and GenericAlias objects to their runtime types.

Partial

Fails to reject various weird annotations

PassUnsupportedPassPass
-Special types in annotations -
specialtypes_anyPassPassPassPassPassPass
specialtypes_neverPassPartial

Does not enforce invariance in some contexts

Does not allow Any to be assigned to Never

PassPassPassPass
specialtypes_nonePassPassPassPassPassPass
specialtypes_promotionsPassPassPassPassPassPass
specialtypes_typePartial

Does not treat `type` same as `type[Any]` for assert_type.

Does not allow access to unknown attributes from object of type `type[Any]`.

PassPassPassPartial

Allows arbitrary attributes to be accessed on `TA` where `TA = typing.Type[typing.Any]` or `TA = type[typing.Any]`.

Treats `type` equivalently to `type[object]` rather than `type[typing.Any]`.

Pass
-Generics -
generics_base_classPartial

Does not detect inconsistent type variable ordering.

PassPassPassPassPass
generics_basicPassPassPassPassPartial

Incorrectly allows constrained type variables to be solved to a union of their constraints.

Pass
generics_defaultsPartial

Does not detect a TypeVar with a default used after a TypeVarTuple.

Does not fully support defaults on TypeVarTuple and ParamSpec.

Partial

Numerous issues; does not support TypeVarTuple and ParamSpec defaults.

PassPassPartial

Does not forbid a `TypeVar` immediately following a `TypeVarTuple` in a parameter list from having a default.

Does not support `TypeVarTuple`.

Pass
generics_defaults_referentialPartial

Does not correctly handle defaults referencing other TypeVars.

Partial

Fails to apply default specializations in some cases.

PassPassPassPass
generics_defaults_specializationPartial

Does not correctly resolve defaults when classes are used directly.

PassPassPartial

Allows incorrect assignment to type[].

Partial

Does not reject subscription of an already-specialized generic class.

Pass
generics_paramspec_basicPassPassPassPassPassPass
generics_paramspec_componentsPassPassPassPassPartial

Incorrectly allows using `*args: P.args` and `**kwargs: P.kwargs` when `P` has not been put into scope by any other parameter annotation or enclosing scope.

Pass
generics_paramspec_semanticsPassPassPassPass*

Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).

PassPass
generics_paramspec_specializationPassPassPassPassPassPass
generics_scopingPassPassPartial

Does not implement several scoping checks/restrictions for generics

PassPartial

Does not reject `list[T]()` in the global scope, where `T` is an unbound type variable.

Does nto reject `alias: TypeAlias = list[T]` in the body scope of a class generic over a type variable `T`.

Pass
generics_self_advancedPartial

Does not infer the type of an unannotated `self` parameter to be type `Self`.

Does not retain `Self` when calling method that returns `Self`.

Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`.

Does not retain `Self` when accessing attribute through `type[Self]`.

PassPass*

Treats attributes not initialized on the class as instance-only

PassPassPass
generics_self_attributesPassPassPassPassPassPass
generics_self_basicPassPassPassPassPassPass
generics_self_protocolsPassPassPassPassPassPass
generics_self_usagePassPassPassPassPartial

Does not reject `Self` used in a return annotation when `self` is annotated using another type variable.

Pass
generics_syntax_compatibilityPassPassPassPassPassPass
generics_syntax_declarationsPassPassPassPassPassPass
generics_syntax_infer_varianceUnsupported

Type parameter syntax not yet supported.

PassPassPassPassPass
generics_syntax_scopingPartial

Does not following runtime scoping rules for type parameters in all cases.

Partial

Misses some details of scoping rules.

PassPassPassPass
generics_type_erasurePartial

Infers Node[Never] instead of Node[Any] when argument is not provided.

False negative on instance attribute access on type(node).

PassPassPassPartial

Does not reject access of generic instance variable from the class object.

Pass
generics_typevartuple_argsPartial

Does not enforce that tuples captured by TypeVarTuple are of the same length.

PassPassPartial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

Partial

Supports PEP-646 unpacked tuples but not TypeVarTuple.

Pass
generics_typevartuple_basicPartial

Does not enforce that tuples captured by TypeVarTuple are same length.

PassPassPartial

Does not correctly solve TypeVarTuple with heterogeneous bounds.

UnsupportedPass
generics_typevartuple_callablePassPassPassPassUnsupportedPass
generics_typevartuple_concatPassPassPassPassUnsupportedPass
generics_typevartuple_overloadsPassPassPassPassPassPass
generics_typevartuple_specializationPartial

Incorrectly specializes generic alias that includes a TypeVar and TypeVarTuple if no type arguments are provided.

Rejects specialization of generic type alias defined as a tuple containing a TypeVar.

PassPassPassUnsupportedPass
generics_typevartuple_unpackPassPassPassPassUnsupportedPass
generics_upper_boundPartial

Does not reject use of type variable within an upper bound.

PassPassPassPassPass
generics_variancePartial

Does not reject use of class-scoped TypeVar used in a base class when variance is incompatible.

PassPassPassPassPass
generics_variance_inferencePassPassPassPassPassPass
-Type qualifiers -
qualifiers_annotatedPartial

Does not allow ClassVar to be nested within Annotated.

Does not allow Final to be nested within Annotated.

Does not allow Required and NotRequired to be nested within Annotated.

Does not reject type[T] compatibility for type alias defined with Annotated.

Does not reject call of type alias defined with Annotated.

Partial

Fails to reject various weird annotations.

False positive on lambda in Annotated.

PassPassPassPass
qualifiers_final_annotationPartial

Does not treat use of Final name as if it was replaced by the literal in NamedTuple definition.

Does not allow conditional assignment of Final instance variable in __init__ method.

Does not allow redefinition of private class variable that is marked Final in parent class.

Does not report modification of local Final variable via "for" statement.

PassPassPassPassPass
qualifiers_final_decoratorPassPassPassPassPassPass
-Class type compatibility -
classes_classvarPartial

Internal error if TypeVarTuple is used in ClassVar.

Does not reject use of ParamSpec in ClassVar.

Rejects ClassVar nested in Annotated.

Does not reject use of ClassVar in TypeAlias definition.

PassPassPassPassPass
classes_overridePassPassPassPassPassPass
-Type aliases -
aliases_explicitPartial

Does not reject specialization of type alias that has already been implicitly specialized.

PassPassPassPartial

Does not emit a diagnostic if a type alias to a union, in which every element of the union is implicitly specialized with `Unknown`, is invalidly specialized again.

Pass
aliases_implicitPassPartial

Fails to handle various weird annotations.

Various bugs with resolving generic aliases.

PassPassPartial

Does not reject variables with `Any` or `Unknown` types when used as implicit type aliases.

Falls short on full syntactic validation of type aliases.

Pass
aliases_newtypePartial

`NewType`s are incorrectly considered to be classes.

PassPassPassPassPass
aliases_recursivePassPassPassPassUnsupportedPass
aliases_type_statementPassPartial

Fails to reject various weird annotations.

PassPassPartial

Does not reject circular definitions of type aliases.

Does not support `type` statements generic over `TypeVarTuple`s.

Pass
aliases_typealiastypePartial

Incorrectly rejects some recursive type aliases using TypeAliasType.

Incorrectly rejects the use of a class-scoped TypeVar in a TypeAliasType definition.

Partial

Rejects valid ParamSpec specialization.

PassPassPartial

Does not reject specializing a type parameter in a generic type alias with a type inconsistent with the parameter's upper bound.

Does not reject declaring a type alias with a type variable that is not in scope.

Does not reject declaring a type alias with a non-literal tuple passed to the `type_params` parameter.

Does not reject cyclically defined type aliases.

Pass
aliases_variancePassPassPassPassPassPass
-Literals -
literals_interactionsPass*

Does not narrow `str` or `LiteralString` types to `Literal` string types via equality or containment checks.

PassPassPassPass*

Deliberately does not allow `str` to be narrowed to literal string types through equality or containment checks due to the possibility of `str` subclasses that could have unexpected equality semantics.

Pass
literals_literalstringUnsupported

Support for `LiteralString` is not implemented.

PassPassPassPassPass
literals_parameterizationsPartial

Does not reject tuple within Literal.

Partial

Fails to reject various invalid literal parameterizations.

PassPassPassPass
literals_semanticsPassPassPassPassPassPass
-Protocols -
protocols_class_objectsPassPartial

Abstract type[Proto] still allows protocol class objects in some paths.

PassPassUnsupported

`type[Proto]` is not yet supported.

`ClassVar` protocol members are not yet supported.

`@property` protocol members are only partially supported.

A class object `C` is only considered to inhabit a protocol type with a method member `f` if `f` exists as an attribute on the metaclass of `C`.

Pass
protocols_definitionPartial

Does not detect protocol mismatch if concrete method is missing annotations.

Does not detect protocol mismatch if concrete method's parameters are position-only.

PassPassPassPartial

Does not reject implicit instance attributes in `Protocol` methods.

Does not support `ClassVar` protocol members.

Incorrectly considers `ClassVar` attributes on concrete classes as satisfying non-`ClassVar` attribute members on protocols.

Only has partial support for `@property` protocol members.

Pass
protocols_explicitPass*

Does not report unimplemented attributes for class that explicitly derives from protocol until it is instantiated.

PassPassPassUnsupported

Allows implicitly abstract protocol methods to be called via `super()` on a protocol subclass.

Allows instantiation of abstract subclasses of protocol classes.

Pass
protocols_genericPassPartial

Fails to reject duplicate generic/protocol bases.

Treats global object as a literal.

PassPassPartial

Only partially supports `@property` protocol members.

Pass
protocols_mergingPassPassPassPassPartial

Does not reject attempted instantiation of abstract subclasses of protocols.

Pass
protocols_modulesPassPassPassPassPartial

Never considers a module as satisfying a protocol with a method member due to the fact that the method will never exist on the class `types.ModuleType`, only on a given specific module instance.

Pass
protocols_recursivePassPassPassPassPartial

Fails to solve a type variable involving a recursive generic protocol.

Pass
protocols_runtime_checkablePartial

Does not report unsafe overlap for runtime_checkable protocol.

PassPassPassPartial

Does not reject `isinstance()` or `issubclass()` calls against runtime-checkable protocols where there is an unsafe overlap between the type of the first argument and the protocol.

Pass
protocols_selfPassPassPassPassPassPass
protocols_subtypingPassPassPassPassPassPass
protocols_variancePassPassPassPassUnsupportedPass
-Callables -
callables_annotationPartial

Incorrectly treats "*args: T, **kwargs: T" as "..." when T is specialized to Any.

Does not treat "*args: Any, **kargs: Any" as "..." when separated by keyword parameter.

PassPartial

Parameter names are lost when resolving ParamSpec

PassPartial

Infers a callback protocol as being a gradual type if the callback has signature `__call__[T](self, *args: T, **kwargs: T)` and `T` has been explicitly specialized to `Any`.

Pass
callables_kwargsPartial

Allows callable without kwargs to be assigned to callable with unpacked kwargs

PassPassPassPassPass
callables_protocolPassPassPassPassPassPass
callables_subtypingPassPassPassPassPassPass
-Constructors -
constructors_call_initPartial

Does not report errors during binding to self parameter of __init__ method.

Does not reject use of class-scoped type variables in annotation of self parameter in __init__ method.

PassPassPassPartial

Does not reject invalid argument types to an inherited constructor in a specialized subclass of a generic superclass.

Does not reject class-scoped type variables used in the `self` annotation.

Pass
constructors_call_metaclassUnsupported

Does not honor metaclass __call__ method when evaluating constructor call.

Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

PassPassPassPassPass
constructors_call_newPartial

Does not support __new__ return type that is not a subclass of the class being constructed.

Does not skip evaluation of __init__ based on __new__ return type.

Does not report errors during binding to cls parameter of __new__ method.

PassPassPassPassPass
constructors_call_typePartial

Does not validate call to custom metaclass __call__ method through type[T].

PassPassPassPartial

Has overly lenient handling of calls to `type[T]` if `T` is a type variable without an upper bound.

Pass
constructors_callablePartial

Does not generate a union type for __new__ and __init__ when converting class to callable.

Does not ignore __init__ based on __new__ return type when converting class to callable.

Does not support __new__ return type that is different from class being constructed.

PassPartial

Converting constructor to callable does not preserve class-scoped type params.

PassPartial

Does not include `__init__` when `__new__` returns `Self`.

Does not ignore `__init__` when `__new__` returns `Any`.

Unions overload return types.

Pass
constructors_consistencyPass*

Does not report inconsistency between __new__ and __init__ (optional).

PassPassPassPassPass
-Overloads -
overloads_basicPassPartial

Does less literal promotion than the test asks for.

PassPassPassPass
overloads_consistencyPassPassPassPassPassPass
overloads_definitionsPartial

Allows @override to be on all overloads and implementation, instead of just implementation.

PassPassPassPassPass
overloads_definitions_stubPartial

Allows @override to appear in a stub file not on the first overload.

Unsupported

Does not support checking stubs.

PassPassPassPass
overloads_evaluationPartial

Does not expand boolean arguments to Literal[True] and Literal[False].

Does not expand enum arguments to literal variants.

Does not expand tuple arguments to possible combinations.

Does not evaluate Any in some cases where overload is ambiguous.

Evaluates Any in some cases where overload is not ambiguous.

PassPassPartial

Does not evaluate Any in some cases where overload is ambiguous.

PassPass
-Exceptions -
exceptions_context_managersPassPassPartial

Some error suppressing context managers are not detected

PassUnsupportedPass
-Dataclasses -
dataclasses_descriptorsPartial

Assumes descriptor behavior only when field is assigned in class body.

Does not correctly evaluate type of descriptor access.

Partial

Conformance suite is questionable; see https://github.com/python/typing/issues/2259

Partial

* Assumes descriptor behavior only when field is assigned in class body

* Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types

PassPartial

Only infers a descriptor `__get__` method as being called when a descriptor attribute is accessed on an instance if the descriptor attribute is present in the class namespace.

Pass
dataclasses_finalPartial

Wrongly requires a Final dataclass field to be initialized at class level.

Doesn't support Final nested inside ClassVar.

PassPassPassPassPass
dataclasses_frozenPassPassPassPassPassPass
dataclasses_hashUnsupported

Does not synthesize `__hash__ = None` as a class attribute for unhashable dataclasses.

Does not report when an unhashable dataclass has `__hash__` called directly on an instance.

Does not report when dataclass is not compatible with Hashable protocol.

PassPassPassPartial

Understands the `Hashable` protocol as equivalent to `object`.

Pass
dataclasses_inheritancePassPassPassPassPassPass
dataclasses_kwonlyPassPassPassPassPassPass
dataclasses_match_argsPassPassPassPassPassPass
dataclasses_orderPassPassPassPassPassPass
dataclasses_postinitPassPassPassPassPassPass
dataclasses_slotsPartial

Does not reject write to instance variable that is not defined in __slots__.

PassPassPassPartial

Synthesizes a `__slots__` attribute but does not validate attribute assignments against `__slots__`.

Pass
dataclasses_transform_classPassPassPassPassPassPass
dataclasses_transform_converterUnsupported

Converter parameter not yet supported.

PassPassPassPassPass
dataclasses_transform_fieldPartial

Does not properly handle field constructor that has default value for `kw_only` or `init` parameter.

PassPassPassPassPass
dataclasses_transform_funcPartial

Does not handle `kw_only=False` override when `kw_only_default=True`.

Does not report error when `order=False` and comparison operators are used.

PassPassPassPassPass
dataclasses_transform_metaPassPassPassPassPassPass
dataclasses_usagePass*

Does not detect unannotated usage of `dataclasses.field()`.

PassPassPassPassPass
-Typed dictionaries -
typeddicts_alt_syntaxPass*

Does not support keyword-argument form of alternative syntax (deprecated in 3.11).

PassPassPassPassPass
typeddicts_class_syntaxPartial

Does not support version-conditional items in TypedDict definitions.

PassPassPartial

Does not support version-conditional items in TypedDict definitions.

PassPass
typeddicts_extra_itemsUnsupported

Not supported.

PassPassPassPassPass
typeddicts_finalPassPassPassPassPassPass
typeddicts_inheritancePassPassPassPassPassPass
typeddicts_operationsPassPassPassPassPassPass
typeddicts_readonlyPassPassPassPassPassPass
typeddicts_readonly_consistencyPassPassPassPassPassPass
typeddicts_readonly_inheritancePartial

Incorrectly rejects non-ReadOnly override of ReadOnly item.

Incorrectly rejects override of ReadOnly item with another ReadOnly item with narrower type.

Incorrectly rejects override of NotRequired ReadOnly item with a Required ReadOnly item.

PassPassPassPassPass
typeddicts_readonly_kwargsPassPassPassPassPassPass
typeddicts_readonly_updatePartial

Incorrectly allows update of ReadOnly item.

Incorrectly rejects update involving an item with Never type.

PassPassPassPassPass
typeddicts_requiredPassPassPassPassPassPass
typeddicts_type_consistencyPassPartial

Considers TypedDicts to be assignable to plain dict types.

PassPassPassPass
typeddicts_usagePassPassPassPassPassPass
-Tuples -
tuples_type_compatPartial

Does not support tuple narrowing based on `len()` type guard (optional).

Incorrectly marks a match case as unreachable.

PassPassPassPassPass
tuples_type_formPassPassPassPassPassPass
tuples_unpackedPartial

"More than one unpack" error is missing in some cases.

PassPassPassPassPass
-Named tuples -
namedtuples_define_classPartial

Does not reject override of named tuple attribute in child class.

Does not support version-conditional fields.

Partial

Does not support precise type inference for slices over namedtuples.

PassPassPassPass
namedtuples_define_functionalPassPassPassPassPassPass
namedtuples_type_compatPassPassPassPassPassPass
namedtuples_usagePartial

Does not reject attempt to delete named tuple field by name.

PassPassPassPassPass
-Enumerations -
enums_behaviorsPassPassPassPassPassPass
enums_definitionPassPartial

Does not allow enum members to be conditional on version/platform checks.

PassPassPassPass
enums_expansionPartial

Improperly applies narrowing to Flag subclass.

PassPassPassPartial

Does not support `enum.Flag`.

Pass
enums_member_namesPass*

Does not support special-cased handling of member name literal types in some cases (optional).

PassPassPassPassPass
enums_member_valuesPartial

Does not enforce declared type of `_value_`.

Does not enforce assigned tuple types for enum members (optional).

PassPassPassPassPass
enums_membersPartial

Does not treat attribute with annotation and no assignment as non-member.

Does not treat callables as non-members.

Does not honor `enum.member` as method decorator.

Does not properly handle aliased enum members.

Does not support `_ignore_` mechanism (optional).

Does not treat attributes with private names as non-members.

PassPassPass*

Does not support `_ignore_` mechanism (optional).

PassPass
-Type narrowing -
narrowing_typeguardPassPassPassPassPassPass
narrowing_typeisPassPassPassPassPartial

Intersects the pre-existing type with the top materialization of the bracketed type rather than the bracketed type itself.

Pass
-Type checker directives -
directives_assert_typePassPassPassPassPassPass
directives_castPassPassPassPassPassPass
directives_deprecatedPassPassPassPassPartial

Does not detect calls to deprecated overloads.

Does not detect implicit calls to deprecated dunder methods, for example via operators.

Does not detect accesses of, or attempts to set, deprecated properties.

Pass
directives_disjoint_basePassPartial

Does not reject invalid class definitions due to disjoint bases, but uses disjoint base information in type narrowing.

PassUnsupported

Does not support PEP 800 disjoint-base semantics.

PassPass
directives_no_type_checkPartial

Does not honor `@no_type_check` class decorator (allowed).

Does not reject invalid call of `@no_type_check` function.

PassPassPass*

Does not honor `@no_type_check` class decorator (allowed).

PassPass
directives_reveal_typePassPassPassPassPassPass
directives_type_checkingPassPassPassPassPassPass
directives_type_ignorePartial

Does not honor "# type: ignore" comment if comment includes additional text.

PassPassPassPassPass
directives_type_ignore_file1PassPassPassPassPassPass
directives_type_ignore_file2PassPassPassPassPassPass
directives_version_platformPass*

Does not understand three-element form of sys.version checks.

Does not understand os.name checks.

PassPassPassPassPass
-Historical and deprecated features -
historical_positionalPartial

Does not reject positional-only parameter after non-positional-only parameter.

Treats keyword-only parameter as positional-only.

Applies legacy positional-only rules when PEP 570 syntax is used.

PassPassPassPassPass
+ .disclaimer { + margin: 6px auto 0; + max-width: 720px; + font-size: 12px; + line-height: 1.4; + font-weight: normal; + color: #d6d6d6; + } + + + +
+
+

Python Type System Conformance Test Results

+

+ While spec conformance is important for the ecosystem, we don't recommend using + it as the primary basis for choosing a type checker. It is not representative + of many of the things users typically care about. +

+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
mypy 2.1.0pycroscope 0.4.0pyrefly 1.1.0pyright 1.1.410ty 0.0.50zuban 0.8.2
+ Type annotations +
annotations_coroutinesPassPassPassPassPassPass
annotations_forward_refs + Partial +
+

Does not report error for a forward reference that is not enclosed in quotes.

+

Does not report error for use of quoted type with "|" operator (runtime error).

+

Incorrectly generates error for quoted type defined in class scope.

+
+ 		+ Partial +
+

Fails to reject "x" | int annotations that fail at runtime.

+

Rejects some valid quoted annotations.

+
+ 		+ Partial +
+

Types in quotes incorrectly refer to shadowing class member.

+

Does not reject some type forms that require quotes.

+
+ 		Pass + Partial +
+

Resolves references in type annotations as referring to end-of-scope types (https://discuss.python.org/t/annotation-string-references-in-class-scope-in-conformance-tests/105439, https://github.com/python/typing/pull/2144)

+
+ 		+ Partial +
+

Incorrectly generates error for quoted type defined in class scope.

+
+
annotations_generators + Partial +
+

Does not report incompatible Generator type in `yield from` statement.

+
+ 		PassPassPassPassPass
annotations_methods + Pass +
+

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

+
+ 		PassPass + Pass +
+

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

+
+ 		PassPass
annotations_typeexprPass + Partial +
+

Fails to reject various weird annotations

+
+ 		PassPassPassPass
+ Type forms +
typeforms_typeform + Partial +
+

Does not support assigning Union and GenericAlias objects to their runtime types.

+
+ 		+ Partial +
+

Fails to reject various weird annotations

+
+ 		PassUnsupportedPassPass
+ Special types in annotations +
specialtypes_anyPassPassPassPassPassPass
specialtypes_neverPass + Partial +
+

Does not enforce invariance in some contexts

+

Does not allow Any to be assigned to Never

+
+ 		PassPassPassPass
specialtypes_nonePassPassPassPassPassPass
specialtypes_promotionsPassPassPassPassPassPass
specialtypes_type + Partial +
+

Does not treat `type` same as `type[Any]` for assert_type.

+

Does not allow access to unknown attributes from object of type `type[Any]`.

+
+ 		PassPassPass + Partial +
+

Allows arbitrary attributes to be accessed on `TA` where `TA = typing.Type[typing.Any]` or `TA = type[typing.Any]`.

+

Treats `type` equivalently to `type[object]` rather than `type[typing.Any]`.

+
+ 		Pass
+ Generics +
generics_base_class + Partial +
+

Does not detect inconsistent type variable ordering.

+
+ 		PassPassPassPassPass
generics_basicPassPassPassPass + Partial +
+

Incorrectly allows constrained type variables to be solved to a union of their constraints.

+
+ 		Pass
generics_defaults + Partial +
+

Does not detect a TypeVar with a default used after a TypeVarTuple.

+

Does not fully support defaults on TypeVarTuple and ParamSpec.

+
+ 		+ Partial +
+

Numerous issues; does not support TypeVarTuple and ParamSpec defaults.

+
+ 		PassPass + Partial +
+

Does not forbid a `TypeVar` immediately following a `TypeVarTuple` in a parameter list from having a default.

+

Does not support `TypeVarTuple`.

+
+ 		Pass
generics_defaults_referential + Partial +
+

Does not correctly handle defaults referencing other TypeVars.

+
+ 		+ Partial +
+

Fails to apply default specializations in some cases.

+
+ 		PassPassPassPass
generics_defaults_specialization + Partial +
+

Does not correctly resolve defaults when classes are used directly.

+
+ 		PassPass + Partial +
+

Allows incorrect assignment to type[].

+
+ 		+ Partial +
+

Does not reject subscription of an already-specialized generic class.

+
+ 		Pass
generics_paramspec_basicPassPassPassPassPassPass
generics_paramspec_componentsPassPassPassPass + Partial +
+

Incorrectly allows using `*args: P.args` and `**kwargs: P.kwargs` when `P` has not been put into scope by any other parameter annotation or enclosing scope.

+
+ 		Pass
generics_paramspec_semanticsPassPassPass + Pass +
+

Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).

+
+ 		PassPass
generics_paramspec_specializationPassPassPassPassPassPass
generics_scopingPassPass + Partial +
+

Does not implement several scoping checks/restrictions for generics

+
+ 		Pass + Partial +
+

Does not reject `list[T]()` in the global scope, where `T` is an unbound type variable.

+

Does nto reject `alias: TypeAlias = list[T]` in the body scope of a class generic over a type variable `T`.

+
+ 		Pass
generics_self_advanced + Partial +
+

Does not infer the type of an unannotated `self` parameter to be type `Self`.

+

Does not retain `Self` when calling method that returns `Self`.

+

Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`.

+

Does not retain `Self` when accessing attribute through `type[Self]`.

+
+ 		Pass + Pass +
+

Treats attributes not initialized on the class as instance-only

+
+ 		PassPassPass
generics_self_attributesPassPassPassPassPassPass
generics_self_basicPassPassPassPassPassPass
generics_self_protocolsPassPassPassPassPassPass
generics_self_usagePassPassPassPass + Partial +
+

Does not reject `Self` used in a return annotation when `self` is annotated using another type variable.

+
+ 		Pass
generics_syntax_compatibilityPassPassPassPassPassPass
generics_syntax_declarationsPassPassPassPassPassPass
generics_syntax_infer_variance + Unsupported +
+

Type parameter syntax not yet supported.

+
+ 		PassPassPassPassPass
generics_syntax_scoping + Partial +
+

Does not following runtime scoping rules for type parameters in all cases.

+
+ 		+ Partial +
+

Misses some details of scoping rules.

+
+ 		PassPassPassPass
generics_type_erasure + Partial +
+

Infers Node[Never] instead of Node[Any] when argument is not provided.

+

False negative on instance attribute access on type(node).

+
+ 		PassPassPass + Partial +
+

Does not reject access of generic instance variable from the class object.

+
+ 		Pass
generics_typevartuple_args + Partial +
+

Does not enforce that tuples captured by TypeVarTuple are of the same length.

+
+ 		PassPass + Partial +
+

Does not correctly solve TypeVarTuple with heterogeneous bounds.

+
+ 		+ Partial +
+

Supports PEP-646 unpacked tuples but not TypeVarTuple.

+
+ 		Pass
generics_typevartuple_basic + Partial +
+

Does not enforce that tuples captured by TypeVarTuple are same length.

+
+ 		PassPass + Partial +
+

Does not correctly solve TypeVarTuple with heterogeneous bounds.

+
+ 		UnsupportedPass
generics_typevartuple_callablePassPassPassPassUnsupportedPass
generics_typevartuple_concatPassPassPassPassUnsupportedPass
generics_typevartuple_overloadsPassPassPassPassPassPass
generics_typevartuple_specialization + Partial +
+

Incorrectly specializes generic alias that includes a TypeVar and TypeVarTuple if no type arguments are provided.

+

Rejects specialization of generic type alias defined as a tuple containing a TypeVar.

+
+ 		PassPassPassUnsupportedPass
generics_typevartuple_unpackPassPassPassPassUnsupportedPass
generics_upper_bound + Partial +
+

Does not reject use of type variable within an upper bound.

+
+ 		PassPassPassPassPass
generics_variance + Partial +
+

Does not reject use of class-scoped TypeVar used in a base class when variance is incompatible.

+
+ 		PassPassPassPassPass
generics_variance_inferencePassPassPassPassPassPass
+ Type qualifiers +
qualifiers_annotated + Partial +
+

Does not allow ClassVar to be nested within Annotated.

+

Does not allow Final to be nested within Annotated.

+

Does not allow Required and NotRequired to be nested within Annotated.

+

Does not reject type[T] compatibility for type alias defined with Annotated.

+

Does not reject call of type alias defined with Annotated.

+
+ 		+ Partial +
+

Fails to reject various weird annotations.

+

False positive on lambda in Annotated.

+
+ 		PassPassPassPass
qualifiers_final_annotation + Partial +
+

Does not treat use of Final name as if it was replaced by the literal in NamedTuple definition.

+

Does not allow conditional assignment of Final instance variable in __init__ method.

+

Does not allow redefinition of private class variable that is marked Final in parent class.

+

Does not report modification of local Final variable via "for" statement.

+
+ 		PassPassPassPassPass
qualifiers_final_decoratorPassPassPassPassPassPass
+ Class type compatibility +
classes_classvar + Partial +
+

Internal error if TypeVarTuple is used in ClassVar.

+

Does not reject use of ParamSpec in ClassVar.

+

Rejects ClassVar nested in Annotated.

+

Does not reject use of ClassVar in TypeAlias definition.

+
+ 		PassPassPassPassPass
classes_overridePassPassPassPassPassPass
+ Type aliases +
aliases_explicit + Partial +
+

Does not reject specialization of type alias that has already been implicitly specialized.

+
+ 		PassPassPass + Partial +
+

Does not emit a diagnostic if a type alias to a union, in which every element of the union is implicitly specialized with `Unknown`, is invalidly specialized again.

+
+ 		Pass
aliases_implicitPass + Partial +
+

Fails to handle various weird annotations.

+

Various bugs with resolving generic aliases.

+
+ 		PassPass + Partial +
+

Does not reject variables with `Any` or `Unknown` types when used as implicit type aliases.

+

Falls short on full syntactic validation of type aliases.

+
+ 		Pass
aliases_newtype + Partial +
+

`NewType`s are incorrectly considered to be classes.

+
+ 		PassPassPassPassPass
aliases_recursivePassPassPassPassUnsupportedPass
aliases_type_statementPass + Partial +
+

Fails to reject various weird annotations.

+
+ 		PassPass + Partial +
+

Does not reject circular definitions of type aliases.

+

Does not support `type` statements generic over `TypeVarTuple`s.

+
+ 		Pass
aliases_typealiastype + Partial +
+

Incorrectly rejects some recursive type aliases using TypeAliasType.

+

Incorrectly rejects the use of a class-scoped TypeVar in a TypeAliasType definition.

+
+ 		+ Partial +
+

Rejects valid ParamSpec specialization.

+
+ 		PassPass + Partial +
+

Does not reject specializing a type parameter in a generic type alias with a type inconsistent with the parameter's upper bound.

+

Does not reject declaring a type alias with a type variable that is not in scope.

+

Does not reject declaring a type alias with a non-literal tuple passed to the `type_params` parameter.

+

Does not reject cyclically defined type aliases.

+
+ 		Pass
aliases_variancePassPassPassPassPassPass
+ Literals +
literals_interactions + Pass +
+

Does not narrow `str` or `LiteralString` types to `Literal` string types via equality or containment checks.

+
+ 		PassPassPass + Pass +
+

Deliberately does not allow `str` to be narrowed to literal string types through equality or containment checks due to the possibility of `str` subclasses that could have unexpected equality semantics.

+
+ 		Pass
literals_literalstring + Unsupported +
+

Support for `LiteralString` is not implemented.

+
+ 		PassPassPassPassPass
literals_parameterizations + Partial +
+

Does not reject tuple within Literal.

+
+ 		+ Partial +
+

Fails to reject various invalid literal parameterizations.

+
+ 		PassPassPassPass
literals_semanticsPassPassPassPassPassPass
+ Protocols +
protocols_class_objectsPass + Partial +
+

Abstract type[Proto] still allows protocol class objects in some paths.

+
+ 		PassPass + Unsupported +
+

`type[Proto]` is not yet supported.

+

`ClassVar` protocol members are not yet supported.

+

`@property` protocol members are only partially supported.

+

A class object `C` is only considered to inhabit a protocol type with a method member `f` if `f` exists as an attribute on the metaclass of `C`.

+
+ 		Pass
protocols_definition + Partial +
+

Does not detect protocol mismatch if concrete method is missing annotations.

+

Does not detect protocol mismatch if concrete method's parameters are position-only.

+
+ 		PassPassPass + Partial +
+

Does not reject implicit instance attributes in `Protocol` methods.

+

Does not support `ClassVar` protocol members.

+

Incorrectly considers `ClassVar` attributes on concrete classes as satisfying non-`ClassVar` attribute members on protocols.

+

Only has partial support for `@property` protocol members.

+
+ 		Pass
protocols_explicit + Pass +
+

Does not report unimplemented attributes for class that explicitly derives from protocol until it is instantiated.

+
+ 		PassPassPass + Unsupported +
+

Allows implicitly abstract protocol methods to be called via `super()` on a protocol subclass.

+

Allows instantiation of abstract subclasses of protocol classes.

+
+ 		Pass
protocols_genericPass + Partial +
+

Fails to reject duplicate generic/protocol bases.

+

Treats global object as a literal.

+
+ 		PassPass + Partial +
+

Only partially supports `@property` protocol members.

+
+ 		Pass
protocols_mergingPassPassPassPass + Partial +
+

Does not reject attempted instantiation of abstract subclasses of protocols.

+
+ 		Pass
protocols_modulesPassPassPassPass + Partial +
+

Never considers a module as satisfying a protocol with a method member due to the fact that the method will never exist on the class `types.ModuleType`, only on a given specific module instance.

+
+ 		Pass
protocols_recursivePassPassPassPass + Partial +
+

Fails to solve a type variable involving a recursive generic protocol.

+
+ 		Pass
protocols_runtime_checkable + Partial +
+

Does not report unsafe overlap for runtime_checkable protocol.

+
+ 		PassPassPass + Partial +
+

Does not reject `isinstance()` or `issubclass()` calls against runtime-checkable protocols where there is an unsafe overlap between the type of the first argument and the protocol.

+
+ 		Pass
protocols_selfPassPassPassPassPassPass
protocols_subtypingPassPassPassPassPassPass
protocols_variancePassPassPassPassUnsupportedPass
+ Callables +
callables_annotation + Partial +
+

Incorrectly treats "*args: T, **kwargs: T" as "..." when T is specialized to Any.

+

Does not treat "*args: Any, **kargs: Any" as "..." when separated by keyword parameter.

+
+ 		Pass + Partial +
+

Parameter names are lost when resolving ParamSpec

+
+ 		Pass + Partial +
+

Infers a callback protocol as being a gradual type if the callback has signature `__call__[T](self, *args: T, **kwargs: T)` and `T` has been explicitly specialized to `Any`.

+
+ 		Pass
callables_kwargs + Partial +
+

Allows callable without kwargs to be assigned to callable with unpacked kwargs

+
+ 		PassPassPassPassPass
callables_protocolPassPassPassPassPassPass
callables_subtypingPassPassPassPassPassPass
+ Constructors +
constructors_call_init + Partial +
+

Does not report errors during binding to self parameter of __init__ method.

+

Does not reject use of class-scoped type variables in annotation of self parameter in __init__ method.

+
+ 		PassPassPass + Partial +
+

Does not reject invalid argument types to an inherited constructor in a specialized subclass of a generic superclass.

+

Does not reject class-scoped type variables used in the `self` annotation.

+
+ 		Pass
constructors_call_metaclass + Unsupported +
+

Does not honor metaclass __call__ method when evaluating constructor call.

+

Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

+
+ 		PassPassPassPassPass
constructors_call_new + Partial +
+

Does not support __new__ return type that is not a subclass of the class being constructed.

+

Does not skip evaluation of __init__ based on __new__ return type.

+

Does not report errors during binding to cls parameter of __new__ method.

+
+ 		PassPassPassPassPass
constructors_call_type + Partial +
+

Does not validate call to custom metaclass __call__ method through type[T].

+
+ 		PassPassPass + Partial +
+

Has overly lenient handling of calls to `type[T]` if `T` is a type variable without an upper bound.

+
+ 		Pass
constructors_callable + Partial +
+

Does not generate a union type for __new__ and __init__ when converting class to callable.

+

Does not ignore __init__ based on __new__ return type when converting class to callable.

+

Does not support __new__ return type that is different from class being constructed.

+
+ 		Pass + Partial +
+

Converting constructor to callable does not preserve class-scoped type params.

+
+ 		Pass + Partial +
+

Does not include `__init__` when `__new__` returns `Self`.

+

Does not ignore `__init__` when `__new__` returns `Any`.

+

Unions overload return types.

+
+ 		Pass
constructors_consistency + Pass +
+

Does not report inconsistency between __new__ and __init__ (optional).

+
+ 		PassPassPassPassPass
+ Overloads +
overloads_basicPass + Partial +
+

Does less literal promotion than the test asks for.

+
+ 		PassPassPassPass
overloads_consistencyPassPassPassPassPassPass
overloads_definitions + Partial +
+

Allows @override to be on all overloads and implementation, instead of just implementation.

+
+ 		PassPassPassPassPass
overloads_definitions_stub + Partial +
+

Allows @override to appear in a stub file not on the first overload.

+
+ 		+ Unsupported +
+

Does not support checking stubs.

+
+ 		PassPassPassPass
overloads_evaluation + Partial +
+

Does not expand boolean arguments to Literal[True] and Literal[False].

+

Does not expand enum arguments to literal variants.

+

Does not expand tuple arguments to possible combinations.

+

Does not evaluate Any in some cases where overload is ambiguous.

+

Evaluates Any in some cases where overload is not ambiguous.

+
+ 		PassPass + Partial +
+

Does not evaluate Any in some cases where overload is ambiguous.

+
+ 		PassPass
+ Exceptions +
exceptions_context_managersPassPass + Partial +
+

Some error suppressing context managers are not detected

+
+ 		PassUnsupportedPass
+ Dataclasses +
dataclasses_descriptors + Partial +
+

Assumes descriptor behavior only when field is assigned in class body.

+

Does not correctly evaluate type of descriptor access.

+
+ 		+ Partial +
+

Conformance suite is questionable; see https://github.com/python/typing/issues/2259

+
+ 		+ Partial +
+

* Assumes descriptor behavior only when field is assigned in class body

+

* Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types

+
+ 		Pass + Partial +
+

Only infers a descriptor `__get__` method as being called when a descriptor attribute is accessed on an instance if the descriptor attribute is present in the class namespace.

+
+ 		Pass
dataclasses_final + Partial +
+

Wrongly requires a Final dataclass field to be initialized at class level.

+

Doesn't support Final nested inside ClassVar.

+
+ 		PassPassPassPassPass
dataclasses_frozenPassPassPassPassPassPass
dataclasses_hash + Unsupported +
+

Does not synthesize `__hash__ = None` as a class attribute for unhashable dataclasses.

+

Does not report when an unhashable dataclass has `__hash__` called directly on an instance.

+

Does not report when dataclass is not compatible with Hashable protocol.

+
+ 		PassPassPass + Partial +
+

Understands the `Hashable` protocol as equivalent to `object`.

+
+ 		Pass
dataclasses_inheritancePassPassPassPassPassPass
dataclasses_kwonlyPassPassPassPassPassPass
dataclasses_match_argsPassPassPassPassPassPass
dataclasses_orderPassPassPassPassPassPass
dataclasses_postinitPassPassPassPassPassPass
dataclasses_slots + Partial +
+

Does not reject write to instance variable that is not defined in __slots__.

+
+ 		PassPassPass + Partial +
+

Synthesizes a `__slots__` attribute but does not validate attribute assignments against `__slots__`.

+
+ 		Pass
dataclasses_transform_classPassPassPassPassPassPass
dataclasses_transform_converter + Unsupported +
+

Converter parameter not yet supported.

+
+ 		PassPassPassPassPass
dataclasses_transform_field + Partial +
+

Does not properly handle field constructor that has default value for `kw_only` or `init` parameter.

+
+ 		PassPassPassPassPass
dataclasses_transform_func + Partial +
+

Does not handle `kw_only=False` override when `kw_only_default=True`.

+

Does not report error when `order=False` and comparison operators are used.

+
+ 		PassPassPassPassPass
dataclasses_transform_metaPassPassPassPassPassPass
dataclasses_usage + Pass +
+

Does not detect unannotated usage of `dataclasses.field()`.

+
+ 		PassPassPassPassPass
+ Typed dictionaries +
typeddicts_alt_syntax + Pass +
+

Does not support keyword-argument form of alternative syntax (deprecated in 3.11).

+
+ 		PassPassPassPassPass
typeddicts_class_syntax + Partial +
+

Does not support version-conditional items in TypedDict definitions.

+
+ 		PassPass + Partial +
+

Does not support version-conditional items in TypedDict definitions.

+
+ 		PassPass
typeddicts_extra_items + Unsupported +
+

Not supported.

+
+ 		PassPassPassPassPass
typeddicts_finalPassPassPassPassPassPass
typeddicts_inheritancePassPassPassPassPassPass
typeddicts_operationsPassPassPassPassPassPass
typeddicts_readonlyPassPassPassPassPassPass
typeddicts_readonly_consistencyPassPassPassPassPassPass
typeddicts_readonly_inheritance + Partial +
+

Incorrectly rejects non-ReadOnly override of ReadOnly item.

+

Incorrectly rejects override of ReadOnly item with another ReadOnly item with narrower type.

+

Incorrectly rejects override of NotRequired ReadOnly item with a Required ReadOnly item.

+
+ 		PassPassPassPassPass
typeddicts_readonly_kwargsPassPassPassPassPassPass
typeddicts_readonly_update + Partial +
+

Incorrectly allows update of ReadOnly item.

+

Incorrectly rejects update involving an item with Never type.

+
+ 		PassPassPassPassPass
typeddicts_requiredPassPassPassPassPassPass
typeddicts_type_consistencyPass + Partial +
+

Considers TypedDicts to be assignable to plain dict types.

+
+ 		PassPassPassPass
typeddicts_usagePassPassPassPassPassPass
+ Tuples +
tuples_type_compat + Partial +
+

Does not support tuple narrowing based on `len()` type guard (optional).

+

Incorrectly marks a match case as unreachable.

+
+ 		PassPassPassPassPass
tuples_type_formPassPassPassPassPassPass
tuples_unpacked + Partial +
+

"More than one unpack" error is missing in some cases.

+
+ 		PassPassPassPassPass
+ Named tuples +
namedtuples_define_class + Partial +
+

Does not reject override of named tuple attribute in child class.

+

Does not support version-conditional fields.

+
+ 		+ Partial +
+

Does not support precise type inference for slices over namedtuples.

+
+ 		PassPassPassPass
namedtuples_define_functionalPassPassPassPassPassPass
namedtuples_type_compatPassPassPassPassPassPass
namedtuples_usage + Partial +
+

Does not reject attempt to delete named tuple field by name.

+
+ 		PassPassPassPassPass
+ Enumerations +
enums_behaviorsPassPassPassPassPassPass
enums_definitionPass + Partial +
+

Does not allow enum members to be conditional on version/platform checks.

+
+ 		PassPassPassPass
enums_expansion + Partial +
+

Improperly applies narrowing to Flag subclass.

+
+ 		PassPassPass + Partial +
+

Does not support `enum.Flag`.

+
+ 		Pass
enums_member_names + Pass +
+

Does not support special-cased handling of member name literal types in some cases (optional).

+
+ 		PassPassPassPassPass
enums_member_values + Partial +
+

Does not enforce declared type of `_value_`.

+

Does not enforce assigned tuple types for enum members (optional).

+
+ 		PassPassPassPassPass
enums_members + Partial +
+

Does not treat attribute with annotation and no assignment as non-member.

+

Does not treat callables as non-members.

+

Does not honor `enum.member` as method decorator.

+

Does not properly handle aliased enum members.

+

Does not support `_ignore_` mechanism (optional).

+

Does not treat attributes with private names as non-members.

+
+ 		PassPass + Pass +
+

Does not support `_ignore_` mechanism (optional).

+
+ 		PassPass
+ Type narrowing +
narrowing_typeguardPassPassPassPassPassPass
narrowing_typeisPassPassPassPass + Partial +
+

Intersects the pre-existing type with the top materialization of the bracketed type rather than the bracketed type itself.

+
+ 		Pass
+ Type checker directives +
directives_assert_typePassPassPassPassPassPass
directives_castPassPassPassPassPassPass
directives_deprecatedPassPassPassPass + Partial +
+

Does not detect calls to deprecated overloads.

+

Does not detect implicit calls to deprecated dunder methods, for example via operators.

+

Does not detect accesses of, or attempts to set, deprecated properties.

+
+ 		Pass
directives_disjoint_basePass + Partial +
+

Does not reject invalid class definitions due to disjoint bases, but uses disjoint base information in type narrowing.

+
+ 		Pass + Unsupported +
+

Does not support PEP 800 disjoint-base semantics.

+
+ 		PassPass
directives_no_type_check + Partial +
+

Does not honor `@no_type_check` class decorator (allowed).

+

Does not reject invalid call of `@no_type_check` function.

+
+ 		PassPass + Pass +
+

Does not honor `@no_type_check` class decorator (allowed).

+
+ 		PassPass
directives_reveal_typePassPassPassPassPassPass
directives_type_checkingPassPassPassPassPassPass
directives_type_ignore + Partial +
+

Does not honor "# type: ignore" comment if comment includes additional text.

+
+ 		PassPassPassPassPass
directives_type_ignore_file1PassPassPassPassPassPass
directives_type_ignore_file2PassPassPassPassPassPass
directives_version_platform + Pass +
+

Does not understand three-element form of sys.version checks.

+

Does not understand os.name checks.

+
+ 		PassPassPassPassPass
+ Historical and deprecated features +
historical_positional + Partial +
+

Does not reject positional-only parameter after non-positional-only parameter.

+

Treats keyword-only parameter as positional-only.

+

Applies legacy positional-only rules when PEP 570 syntax is used.

+
+ 		PassPassPassPassPass
+
-
- - - + + \ No newline at end of file diff --git a/conformance/src/main.py b/conformance/src/main.py index 117eee2c4..63f82ea72 100644 --- a/conformance/src/main.py +++ b/conformance/src/main.py @@ -2,7 +2,7 @@ Type system conformance test for static type checkers. """ -import os +import contextlib import re import sys import tomllib @@ -275,16 +275,16 @@ def main(): test_cases = get_test_cases(test_groups, tests_dir) # Switch to the tests directory. - os.chdir(tests_dir) - - # Run each test case with each type checker. - for type_checker in TYPE_CHECKERS: - if options.only_run and options.only_run != type_checker.name: - continue - if not type_checker.install(): - print(f"Skipping tests for {type_checker.name}") - else: - run_tests(root_dir, type_checker, test_cases, verbose=options.verbose) + with contextlib.chdir(tests_dir): + + # Run each test case with each type checker. + for type_checker in TYPE_CHECKERS: + if options.only_run and options.only_run != type_checker.name: + continue + if not type_checker.install(): + print(f"Skipping tests for {type_checker.name}") + else: + run_tests(root_dir, type_checker, test_cases, verbose=options.verbose) # Generate a summary report. generate_summary(root_dir) diff --git a/conformance/src/reporting.py b/conformance/src/reporting.py index 6fafa8626..167abc4c7 100644 --- a/conformance/src/reporting.py +++ b/conformance/src/reporting.py @@ -4,125 +4,149 @@ import operator import tomllib +from collections.abc import Sequence +from dataclasses import dataclass, field from pathlib import Path +import jinja2 + from test_groups import get_test_cases, get_test_groups -from type_checker import TYPE_CHECKERS +from type_checker import TYPE_CHECKERS, TypeChecker + + +@dataclass(frozen=True, kw_only=True, slots=True) +class TestResult: + type_checker: str + conformance: str + notes: list[str] = field(default_factory=list) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class TestCase: + name: str + results: list[TestResult] = field(default_factory=list) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class TestGroup: + slug: str + name: str + href: str + paths: list[Path] = field(default_factory=list) + cases: list[TestCase] = field(default_factory=list) def generate_summary(root_dir: Path): print("Generating summary report") - template_file = root_dir / "src" / "templates" / "base.html" - with open(template_file, "r") as f: - template = f.read() - summary = template.replace("{{summary}}", generate_summary_html(root_dir)) + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(root_dir.joinpath("src/templates")), + autoescape=jinja2.select_autoescape(), + ) + env.filters["conformance_class"] = _conformance_class + + template = env.get_template("base.html") - results_file = root_dir / "results" / "results.html" + type_checkers = sorted(TYPE_CHECKERS, key=operator.attrgetter("name")) - with open(results_file, "w") as f: - f.write(summary) + results = template.render( + groups=_get_groups(root_dir, type_checkers), + versions=_get_versions(root_dir, type_checkers), + ) + root_dir.joinpath("results", "results.html").write_text(results) -def generate_summary_html(root_dir: Path) -> str: - type_checkers = sorted(TYPE_CHECKERS, key=operator.attrgetter("name")) - column_count = len(type_checkers) + 1 + +def _conformance_class(value: str) -> str: + if value == "Pass": + return "conformant" + if value == "Partial": + return "partially-conformant" + return "not-conformant" + + +def _get_groups( + root_dir: Path, + type_checkers: Sequence[TypeChecker], +) -> list[TestGroup]: test_groups = get_test_groups(root_dir) test_cases = get_test_cases(test_groups, root_dir / "tests") - summary_html = [ - "", - '', - f'', - "", - "", - "", - "", - ] + groups = [] + + for test_group_slug, test_group in test_groups.items(): + paths = sorted( + [ + case + for case in test_cases + if case.name.startswith(f"{test_group_slug}_") + ], + key=operator.attrgetter("name"), + ) + + # Skip if there are no test cases in the group. + if not paths: + continue + + group = TestGroup( + slug=test_group_slug, + name=test_group.name, + href=test_group.href, + paths=paths, + ) + groups.append(group) + + for path in group.paths: + case = TestCase(name=path.stem) + group.cases.append(case) + + for type_checker in type_checkers: + result_path = ( + root_dir / "results" / type_checker.name / f"{case.name}.toml" + ) + try: + with result_path.open("rb") as f: + data = tomllib.load(f) + except FileNotFoundError: + data = {} + + conformance = data.get("conformant") + if not conformance: + # Try to look up the automated test results and use that if the test passes. + automated = data.get("conformance_automated") + conformance = "Pass" if automated == "Pass" else "Unknown" + + result = TestResult( + type_checker=type_checker.name, + conformance=conformance, + notes=data.get("notes", "").strip().splitlines(), + ) + case.results.append(result) + + return groups + + +def _get_versions( + root_dir: Path, + type_checkers: Sequence[TypeChecker], +) -> list[str]: + versions = [] for type_checker in type_checkers: - # Load the version file for the type checker. - version_file = root_dir / "results" / type_checker.name / "version.toml" + name = type_checker.name try: - with open(version_file, "rb") as f: - existing_info = tomllib.load(f) - except FileNotFoundError: - existing_info = {} - except tomllib.TOMLDecodeError: - print(f"Error decoding {version_file}") - existing_info = {} - - version = existing_info["version"] or "Unknown version" - - summary_html.append(f"{version}") - - summary_html.extend(["", "", ""]) - - for test_group_name, test_group in test_groups.items(): - tests_in_group = [ - case for case in test_cases if case.name.startswith(f"{test_group_name}_") - ] - - tests_in_group.sort(key=lambda x: x.name) - - # Are there any test cases in this group? - if len(tests_in_group) > 0: - summary_html.append(f'') - summary_html.append(f'{test_group.name}') - summary_html.append("") - - for test_case in tests_in_group: - test_case_name = test_case.stem - - summary_html.append(f"{test_case_name}") - - for type_checker in type_checkers: - try: - results_file = ( - root_dir - / "results" - / type_checker.name - / f"{test_case_name}.toml" - ) - with open(results_file, "rb") as f: - results = tomllib.load(f) - except FileNotFoundError: - results = {} - - raw_notes = results.get("notes", "").strip() - conformance = results.get("conformant", "Unknown") - if conformance == "Unknown": - # Try to look up the automated test results and use - # that if the test passes - automated = results.get("conformance_automated") - if automated == "Pass": - conformance = "Pass" - notes = "".join( - [f"

{note}

" for note in raw_notes.split("\n")] - ) - - conformance_classes = ( - "conformant" - if conformance == "Pass" - else "partially-conformant" - if conformance == "Partial" - else "not-conformant" - ) - - # Add an asterisk if there are notes to display for a "Pass". - if raw_notes != "" and conformance == "Pass": - conformance = "Pass*" - - conformance_cell = f"{conformance}" - if raw_notes != "": - conformance_classes = f"{conformance_classes} tooltip" - conformance_cell = f'{conformance_cell}
{notes}
' - - summary_html.append(f'{conformance_cell}') - - summary_html.append("") - - summary_html.append("") - - return "\n".join(summary_html) + with root_dir.joinpath("results", name, "version.toml").open("rb") as f: + data = tomllib.load(f) + except (FileNotFoundError, tomllib.TOMLDecodeError): + version = None + else: + version = data.get("version") or None + + # If version file cannot be found or has missing/invalid content, fall back to name. + if version is None: + version = f"{type_checker.name} ?.?.?" + + versions.append(version) + + return versions diff --git a/conformance/src/templates/base.html b/conformance/src/templates/base.html index 0bcd14262..08067e97e 100644 --- a/conformance/src/templates/base.html +++ b/conformance/src/templates/base.html @@ -1,186 +1,70 @@ - +{%- set columns = versions|length -%} + - - - - - Type System Test Results - - - - -
-
-

Python Type System Conformance Test Results

-

- While spec conformance is important for the ecosystem, we don't recommend using it - as the primary basis for choosing a type checker. It is not representative - of many of the things users typically care about. -

-
-
- - {{summary}} -
+ + + + Type System Test Results + + + +
+
+

Python Type System Conformance Test Results

+

+ While spec conformance is important for the ecosystem, we don't recommend using + it as the primary basis for choosing a type checker. It is not representative + of many of the things users typically care about. +

+
+
+ + + + + + + + + {%- for version in versions %} + + {%- endfor %} + + + + {%- for group in groups %} + + + + {%- for case in group.cases %} + + + {%- for result in case.results %} + {%- if result.notes %} + + {%- else %} + + {%- endif %} + {%- endfor %} + + {%- endfor %} + {%- endfor %} + +
{{ version }}
+ {{ group.name }} +
{{ case.name }} + {{ result.conformance }} +
+ {%- for note in result.notes %} +

{{ note }}

+ {%- endfor %} +
+ 		{{ result.conformance }}
+
-
- - + diff --git a/conformance/src/templates/style.css b/conformance/src/templates/style.css new file mode 100644 index 000000000..aa7709aff --- /dev/null +++ b/conformance/src/templates/style.css @@ -0,0 +1,162 @@ +body, +h1, +h2, +h3, +p, +table { + font-family: "system-ui", "Segoe UI", Helvetica, Arial, sans-serif; + margin: 0; + padding: 0; +} + +header { + background-color: #333; + color: #fff; + text-align: center; + padding: 10px 16px 12px; +} + +table { + margin: 0px; + padding: 0px; + width: 100%; + border-collapse: collapse; + font-family: "system-ui", "Segoe UI", Helvetica, Arial, sans-serif; + font-size: 10pt; + font-weight: normal; + font-style: normal; +} + +th, +td { + margin: 0px; + padding: 0px; + text-align: left; + border-bottom: 0px solid #000; + font-weight: normal; +} + +th:first-child, +td:first-child { + vertical-align: top; +} + +th:not(:first-child), +td:not(:first-child){ + text-align: center; +} + +.content_container { + margin: 20px; +} + +main { + margin-left: 20px; +} + +tbody > tr > th, +tbody > tr > td { + padding-left: 8px; + padding-right: 8px; + padding-top: 4px; + padding-bottom: 4px; + + margin-left: 8px; + margin-right: 8px; + margin-top: 4px; + margin-bottom: 4px; + + border-style: solid; + border-color: white; + border-left-width: 8px; + border-right-width: 8px; + border-top-width: 0; + border-bottom-width: 0; +} + +tbody > tr > th:not([colspan]) { + padding-left: 2em; +} + +tbody > tr > th[colspan] > a { + font-size: 10pt; + font-weight: bold; +} + +.col1 { + width: 30%; +} + +.col2 { + width: calc(70% / {{ columns }}); +} + +thead > tr > th { + font-size: 16px; + padding-bottom: 8px; + padding-top: 12px; + text-align: center; +} + +.conformant { + background-color: rgb(203, 255, 203); +} + +.partially-conformant { + background-color: rgb(251, 244, 166); +} + +.not-conformant { + background-color: rgb(242, 171, 171); +} + +.notes { + visibility: hidden; + position: absolute; + z-index: 2; + width: 400px; + color: black; + font-size: 12px; + background-color: white; + border: 1px solid black; + padding: 10px 15px 10px 15px; + text-align: left; +} + +.tooltip:hover .notes { + visibility: visible; +} + +.tooltip .notes p { + margin: 4px; +} + +.tooltip .notes { + top: 25px; + left: -80%; +} + +.tooltip .notes::before { + top: -5%; + left: 94%; +} + +.tooltip { + cursor: help; + position: relative; +} + +{# Add an asterisk if there are notes to display for a "Pass". -#} +.tooltip.conformant::after { + position: absolute; + content: "*"; +} + +.disclaimer { + margin: 6px auto 0; + max-width: 720px; + font-size: 12px; + line-height: 1.4; + font-weight: normal; + color: #d6d6d6; +} From ab1028c9e55ea4bb8b0c0b3be26014517e65f670 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:32 +0100 Subject: [PATCH 20/30] Improvements to styling --- conformance/results/results.html | 4633 ++++++++++++++------------- conformance/src/templates/base.html | 124 +- conformance/src/templates/style.css | 276 +- 3 files changed, 2581 insertions(+), 2452 deletions(-) diff --git a/conformance/results/results.html b/conformance/results/results.html index bb3c7989d..7d3bf0429 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -2,2346 +2,2411 @@ - - Type System Test Results + + + Python Type System Conformance Test Results -
-
-

Python Type System Conformance Test Results

-

- While spec conformance is important for the ecosystem, we don't recommend using - it as the primary basis for choosing a type checker. It is not representative - of many of the things users typically care about. -

-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
mypy 2.1.0pycroscope 0.4.0pyrefly 1.1.0pyright 1.1.410ty 0.0.50zuban 0.8.2
- Type annotations -
annotations_coroutinesPassPassPassPassPassPass
annotations_forward_refs - Partial -
-

Does not report error for a forward reference that is not enclosed in quotes.

-

Does not report error for use of quoted type with "|" operator (runtime error).

-

Incorrectly generates error for quoted type defined in class scope.

-
- 		- Partial -
-

Fails to reject "x" | int annotations that fail at runtime.

-

Rejects some valid quoted annotations.

-
- 		- Partial -
-

Types in quotes incorrectly refer to shadowing class member.

-

Does not reject some type forms that require quotes.

-
- 		Pass - Partial -
-

Resolves references in type annotations as referring to end-of-scope types (https://discuss.python.org/t/annotation-string-references-in-class-scope-in-conformance-tests/105439, https://github.com/python/typing/pull/2144)

-
- 		- Partial -
-

Incorrectly generates error for quoted type defined in class scope.

-
-
annotations_generators - Partial -
-

Does not report incompatible Generator type in `yield from` statement.

-
- 		PassPassPassPassPass
annotations_methods - Pass -
-

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

-
- 		PassPass - Pass -
-

Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.

-
- 		PassPass
annotations_typeexprPass - Partial -
-

Fails to reject various weird annotations

-
- 		PassPassPassPass
- Type forms -
typeforms_typeform - Partial -
-

Does not support assigning Union and GenericAlias objects to their runtime types.

-
- 		- Partial -
-

Fails to reject various weird annotations

-
- 		PassUnsupportedPassPass
- Special types in annotations -
specialtypes_anyPassPassPassPassPassPass
specialtypes_neverPass - Partial -
-

Does not enforce invariance in some contexts

-

Does not allow Any to be assigned to Never

-
- 		PassPassPassPass
specialtypes_nonePassPassPassPassPassPass
specialtypes_promotionsPassPassPassPassPassPass
specialtypes_type - Partial -
-

Does not treat `type` same as `type[Any]` for assert_type.

-

Does not allow access to unknown attributes from object of type `type[Any]`.

-
- 		PassPassPass - Partial -
-

Allows arbitrary attributes to be accessed on `TA` where `TA = typing.Type[typing.Any]` or `TA = type[typing.Any]`.

-

Treats `type` equivalently to `type[object]` rather than `type[typing.Any]`.

-
- 		Pass
- Generics -
generics_base_class - Partial -
-

Does not detect inconsistent type variable ordering.

-
- 		PassPassPassPassPass
generics_basicPassPassPassPass - Partial -
-

Incorrectly allows constrained type variables to be solved to a union of their constraints.

-
- 		Pass
generics_defaults - Partial -
-

Does not detect a TypeVar with a default used after a TypeVarTuple.

-

Does not fully support defaults on TypeVarTuple and ParamSpec.

-
- 		- Partial -
-

Numerous issues; does not support TypeVarTuple and ParamSpec defaults.

-
- 		PassPass - Partial -
-

Does not forbid a `TypeVar` immediately following a `TypeVarTuple` in a parameter list from having a default.

-

Does not support `TypeVarTuple`.

-
- 		Pass
generics_defaults_referential - Partial -
-

Does not correctly handle defaults referencing other TypeVars.

-
- 		- Partial -
-

Fails to apply default specializations in some cases.

-
- 		PassPassPassPass
generics_defaults_specialization - Partial -
-

Does not correctly resolve defaults when classes are used directly.

-
- 		PassPass - Partial -
-

Allows incorrect assignment to type[].

-
- 		- Partial -
-

Does not reject subscription of an already-specialized generic class.

-
- 		Pass
generics_paramspec_basicPassPassPassPassPassPass
generics_paramspec_componentsPassPassPassPass - Partial -
-

Incorrectly allows using `*args: P.args` and `**kwargs: P.kwargs` when `P` has not been put into scope by any other parameter annotation or enclosing scope.

-
- 		Pass
generics_paramspec_semanticsPassPassPass - Pass -
-

Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).

-
- 		PassPass
generics_paramspec_specializationPassPassPassPassPassPass
generics_scopingPassPass - Partial -
-

Does not implement several scoping checks/restrictions for generics

-
- 		Pass - Partial -
-

Does not reject `list[T]()` in the global scope, where `T` is an unbound type variable.

-

Does nto reject `alias: TypeAlias = list[T]` in the body scope of a class generic over a type variable `T`.

-
- 		Pass
generics_self_advanced - Partial -
-

Does not infer the type of an unannotated `self` parameter to be type `Self`.

-

Does not retain `Self` when calling method that returns `Self`.

-

Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`.

-

Does not retain `Self` when accessing attribute through `type[Self]`.

-
- 		Pass - Pass -
-

Treats attributes not initialized on the class as instance-only

-
- 		PassPassPass
generics_self_attributesPassPassPassPassPassPass
generics_self_basicPassPassPassPassPassPass
generics_self_protocolsPassPassPassPassPassPass
generics_self_usagePassPassPassPass - Partial -
-

Does not reject `Self` used in a return annotation when `self` is annotated using another type variable.

-
- 		Pass
generics_syntax_compatibilityPassPassPassPassPassPass
generics_syntax_declarationsPassPassPassPassPassPass
generics_syntax_infer_variance - Unsupported -
-

Type parameter syntax not yet supported.

-
- 		PassPassPassPassPass
generics_syntax_scoping - Partial -
-

Does not following runtime scoping rules for type parameters in all cases.

-
- 		- Partial -
-

Misses some details of scoping rules.

-
- 		PassPassPassPass
generics_type_erasure - Partial -
-

Infers Node[Never] instead of Node[Any] when argument is not provided.

-

False negative on instance attribute access on type(node).

-
- 		PassPassPass - Partial -
-

Does not reject access of generic instance variable from the class object.

-
- 		Pass
generics_typevartuple_args - Partial -
-

Does not enforce that tuples captured by TypeVarTuple are of the same length.

-
- 		PassPass - Partial -
-

Does not correctly solve TypeVarTuple with heterogeneous bounds.

-
- 		- Partial -
-

Supports PEP-646 unpacked tuples but not TypeVarTuple.

-
- 		Pass
generics_typevartuple_basic - Partial -
-

Does not enforce that tuples captured by TypeVarTuple are same length.

-
- 		PassPass - Partial -
-

Does not correctly solve TypeVarTuple with heterogeneous bounds.

-
- 		UnsupportedPass
generics_typevartuple_callablePassPassPassPassUnsupportedPass
generics_typevartuple_concatPassPassPassPassUnsupportedPass
generics_typevartuple_overloadsPassPassPassPassPassPass
generics_typevartuple_specialization - Partial -
-

Incorrectly specializes generic alias that includes a TypeVar and TypeVarTuple if no type arguments are provided.

-

Rejects specialization of generic type alias defined as a tuple containing a TypeVar.

-
- 		PassPassPassUnsupportedPass
generics_typevartuple_unpackPassPassPassPassUnsupportedPass
generics_upper_bound - Partial -
-

Does not reject use of type variable within an upper bound.

-
- 		PassPassPassPassPass
generics_variance - Partial -
-

Does not reject use of class-scoped TypeVar used in a base class when variance is incompatible.

-
- 		PassPassPassPassPass
generics_variance_inferencePassPassPassPassPassPass
- Type qualifiers -
qualifiers_annotated - Partial -
-

Does not allow ClassVar to be nested within Annotated.

-

Does not allow Final to be nested within Annotated.

-

Does not allow Required and NotRequired to be nested within Annotated.

-

Does not reject type[T] compatibility for type alias defined with Annotated.

-

Does not reject call of type alias defined with Annotated.

-
- 		- Partial -
-

Fails to reject various weird annotations.

-

False positive on lambda in Annotated.

-
- 		PassPassPassPass
qualifiers_final_annotation - Partial -
-

Does not treat use of Final name as if it was replaced by the literal in NamedTuple definition.

-

Does not allow conditional assignment of Final instance variable in __init__ method.

-

Does not allow redefinition of private class variable that is marked Final in parent class.

-

Does not report modification of local Final variable via "for" statement.

-
- 		PassPassPassPassPass
qualifiers_final_decoratorPassPassPassPassPassPass
- Class type compatibility -
classes_classvar - Partial -
-

Internal error if TypeVarTuple is used in ClassVar.

-

Does not reject use of ParamSpec in ClassVar.

-

Rejects ClassVar nested in Annotated.

-

Does not reject use of ClassVar in TypeAlias definition.

-
- 		PassPassPassPassPass
classes_overridePassPassPassPassPassPass
- Type aliases -
aliases_explicit - Partial -
-

Does not reject specialization of type alias that has already been implicitly specialized.

-
- 		PassPassPass - Partial -
-

Does not emit a diagnostic if a type alias to a union, in which every element of the union is implicitly specialized with `Unknown`, is invalidly specialized again.

-
- 		Pass
aliases_implicitPass - Partial -
-

Fails to handle various weird annotations.

-

Various bugs with resolving generic aliases.

-
- 		PassPass - Partial -
-

Does not reject variables with `Any` or `Unknown` types when used as implicit type aliases.

-

Falls short on full syntactic validation of type aliases.

-
- 		Pass
aliases_newtype - Partial -
-

`NewType`s are incorrectly considered to be classes.

-
- 		PassPassPassPassPass
aliases_recursivePassPassPassPassUnsupportedPass
aliases_type_statementPass - Partial -
-

Fails to reject various weird annotations.

-
- 		PassPass - Partial -
-

Does not reject circular definitions of type aliases.

-

Does not support `type` statements generic over `TypeVarTuple`s.

-
- 		Pass
aliases_typealiastype - Partial -
-

Incorrectly rejects some recursive type aliases using TypeAliasType.

-

Incorrectly rejects the use of a class-scoped TypeVar in a TypeAliasType definition.

-
- 		- Partial -
-

Rejects valid ParamSpec specialization.

-
- 		PassPass - Partial -
-

Does not reject specializing a type parameter in a generic type alias with a type inconsistent with the parameter's upper bound.

-

Does not reject declaring a type alias with a type variable that is not in scope.

-

Does not reject declaring a type alias with a non-literal tuple passed to the `type_params` parameter.

-

Does not reject cyclically defined type aliases.

-
- 		Pass
aliases_variancePassPassPassPassPassPass
- Literals -
literals_interactions - Pass -
-

Does not narrow `str` or `LiteralString` types to `Literal` string types via equality or containment checks.

-
- 		PassPassPass - Pass -
-

Deliberately does not allow `str` to be narrowed to literal string types through equality or containment checks due to the possibility of `str` subclasses that could have unexpected equality semantics.

-
- 		Pass
literals_literalstring - Unsupported -
-

Support for `LiteralString` is not implemented.

-
- 		PassPassPassPassPass
literals_parameterizations - Partial -
-

Does not reject tuple within Literal.

-
- 		- Partial -
-

Fails to reject various invalid literal parameterizations.

-
- 		PassPassPassPass
literals_semanticsPassPassPassPassPassPass
- Protocols -
protocols_class_objectsPass - Partial -
-

Abstract type[Proto] still allows protocol class objects in some paths.

-
- 		PassPass - Unsupported -
-

`type[Proto]` is not yet supported.

-

`ClassVar` protocol members are not yet supported.

-

`@property` protocol members are only partially supported.

-

A class object `C` is only considered to inhabit a protocol type with a method member `f` if `f` exists as an attribute on the metaclass of `C`.

-
- 		Pass
protocols_definition - Partial -
-

Does not detect protocol mismatch if concrete method is missing annotations.

-

Does not detect protocol mismatch if concrete method's parameters are position-only.

-
- 		PassPassPass - Partial -
-

Does not reject implicit instance attributes in `Protocol` methods.

-

Does not support `ClassVar` protocol members.

-

Incorrectly considers `ClassVar` attributes on concrete classes as satisfying non-`ClassVar` attribute members on protocols.

-

Only has partial support for `@property` protocol members.

-
- 		Pass
protocols_explicit - Pass -
-

Does not report unimplemented attributes for class that explicitly derives from protocol until it is instantiated.

-
- 		PassPassPass - Unsupported -
-

Allows implicitly abstract protocol methods to be called via `super()` on a protocol subclass.

-

Allows instantiation of abstract subclasses of protocol classes.

-
- 		Pass
protocols_genericPass - Partial -
-

Fails to reject duplicate generic/protocol bases.

-

Treats global object as a literal.

-
- 		PassPass - Partial -
-

Only partially supports `@property` protocol members.

-
- 		Pass
protocols_mergingPassPassPassPass - Partial -
-

Does not reject attempted instantiation of abstract subclasses of protocols.

-
- 		Pass
protocols_modulesPassPassPassPass - Partial -
-

Never considers a module as satisfying a protocol with a method member due to the fact that the method will never exist on the class `types.ModuleType`, only on a given specific module instance.

-
- 		Pass
protocols_recursivePassPassPassPass - Partial -
-

Fails to solve a type variable involving a recursive generic protocol.

-
- 		Pass
protocols_runtime_checkable - Partial -
-

Does not report unsafe overlap for runtime_checkable protocol.

-
- 		PassPassPass - Partial -
-

Does not reject `isinstance()` or `issubclass()` calls against runtime-checkable protocols where there is an unsafe overlap between the type of the first argument and the protocol.

-
- 		Pass
protocols_selfPassPassPassPassPassPass
protocols_subtypingPassPassPassPassPassPass
protocols_variancePassPassPassPassUnsupportedPass
- Callables -
callables_annotation - Partial -
-

Incorrectly treats "*args: T, **kwargs: T" as "..." when T is specialized to Any.

-

Does not treat "*args: Any, **kargs: Any" as "..." when separated by keyword parameter.

-
- 		Pass - Partial -
-

Parameter names are lost when resolving ParamSpec

-
- 		Pass - Partial -
-

Infers a callback protocol as being a gradual type if the callback has signature `__call__[T](self, *args: T, **kwargs: T)` and `T` has been explicitly specialized to `Any`.

-
- 		Pass
callables_kwargs - Partial -
-

Allows callable without kwargs to be assigned to callable with unpacked kwargs

-
- 		PassPassPassPassPass
callables_protocolPassPassPassPassPassPass
callables_subtypingPassPassPassPassPassPass
- Constructors -
constructors_call_init - Partial -
-

Does not report errors during binding to self parameter of __init__ method.

-

Does not reject use of class-scoped type variables in annotation of self parameter in __init__ method.

-
- 		PassPassPass - Partial -
-

Does not reject invalid argument types to an inherited constructor in a specialized subclass of a generic superclass.

-

Does not reject class-scoped type variables used in the `self` annotation.

-
- 		Pass
constructors_call_metaclass - Unsupported -
-

Does not honor metaclass __call__ method when evaluating constructor call.

-

Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

-
- 		PassPassPassPassPass
constructors_call_new - Partial -
-

Does not support __new__ return type that is not a subclass of the class being constructed.

-

Does not skip evaluation of __init__ based on __new__ return type.

-

Does not report errors during binding to cls parameter of __new__ method.

-
- 		PassPassPassPassPass
constructors_call_type - Partial -
-

Does not validate call to custom metaclass __call__ method through type[T].

-
- 		PassPassPass - Partial -
-

Has overly lenient handling of calls to `type[T]` if `T` is a type variable without an upper bound.

-
- 		Pass
constructors_callable - Partial -
-

Does not generate a union type for __new__ and __init__ when converting class to callable.

-

Does not ignore __init__ based on __new__ return type when converting class to callable.

-

Does not support __new__ return type that is different from class being constructed.

-
- 		Pass - Partial -
-

Converting constructor to callable does not preserve class-scoped type params.

-
- 		Pass - Partial -
-

Does not include `__init__` when `__new__` returns `Self`.

-

Does not ignore `__init__` when `__new__` returns `Any`.

-

Unions overload return types.

-
- 		Pass
constructors_consistency - Pass -
-

Does not report inconsistency between __new__ and __init__ (optional).

-
- 		PassPassPassPassPass
- Overloads -
overloads_basicPass - Partial -
-

Does less literal promotion than the test asks for.

-
- 		PassPassPassPass
overloads_consistencyPassPassPassPassPassPass
overloads_definitions - Partial -
-

Allows @override to be on all overloads and implementation, instead of just implementation.

-
- 		PassPassPassPassPass
overloads_definitions_stub - Partial -
-

Allows @override to appear in a stub file not on the first overload.

-
- 		- Unsupported -
-

Does not support checking stubs.

-
- 		PassPassPassPass
overloads_evaluation - Partial -
-

Does not expand boolean arguments to Literal[True] and Literal[False].

-

Does not expand enum arguments to literal variants.

-

Does not expand tuple arguments to possible combinations.

-

Does not evaluate Any in some cases where overload is ambiguous.

-

Evaluates Any in some cases where overload is not ambiguous.

-
- 		PassPass - Partial -
-

Does not evaluate Any in some cases where overload is ambiguous.

-
- 		PassPass
- Exceptions -
exceptions_context_managersPassPass - Partial -
-

Some error suppressing context managers are not detected

-
- 		PassUnsupportedPass
- Dataclasses -
dataclasses_descriptors - Partial -
-

Assumes descriptor behavior only when field is assigned in class body.

-

Does not correctly evaluate type of descriptor access.

-
- 		- Partial -
-

Conformance suite is questionable; see https://github.com/python/typing/issues/2259

-
- 		- Partial -
-

* Assumes descriptor behavior only when field is assigned in class body

-

* Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types

-
- 		Pass - Partial -
-

Only infers a descriptor `__get__` method as being called when a descriptor attribute is accessed on an instance if the descriptor attribute is present in the class namespace.

-
- 		Pass
dataclasses_final - Partial -
-

Wrongly requires a Final dataclass field to be initialized at class level.

-

Doesn't support Final nested inside ClassVar.

-
- 		PassPassPassPassPass
dataclasses_frozenPassPassPassPassPassPass
dataclasses_hash - Unsupported -
-

Does not synthesize `__hash__ = None` as a class attribute for unhashable dataclasses.

-

Does not report when an unhashable dataclass has `__hash__` called directly on an instance.

-

Does not report when dataclass is not compatible with Hashable protocol.

-
- 		PassPassPass - Partial -
-

Understands the `Hashable` protocol as equivalent to `object`.

-
- 		Pass
dataclasses_inheritancePassPassPassPassPassPass
dataclasses_kwonlyPassPassPassPassPassPass
dataclasses_match_argsPassPassPassPassPassPass
dataclasses_orderPassPassPassPassPassPass
dataclasses_postinitPassPassPassPassPassPass
dataclasses_slots - Partial -
-

Does not reject write to instance variable that is not defined in __slots__.

-
- 		PassPassPass - Partial -
-

Synthesizes a `__slots__` attribute but does not validate attribute assignments against `__slots__`.

-
- 		Pass
dataclasses_transform_classPassPassPassPassPassPass
dataclasses_transform_converter - Unsupported -
-

Converter parameter not yet supported.

-
- 		PassPassPassPassPass
dataclasses_transform_field - Partial -
-

Does not properly handle field constructor that has default value for `kw_only` or `init` parameter.

-
- 		PassPassPassPassPass
dataclasses_transform_func - Partial -
-

Does not handle `kw_only=False` override when `kw_only_default=True`.

-

Does not report error when `order=False` and comparison operators are used.

-
- 		PassPassPassPassPass
dataclasses_transform_metaPassPassPassPassPassPass
dataclasses_usage - Pass -
-

Does not detect unannotated usage of `dataclasses.field()`.

-
- 		PassPassPassPassPass
- Typed dictionaries -
typeddicts_alt_syntax - Pass -
-

Does not support keyword-argument form of alternative syntax (deprecated in 3.11).

-
- 		PassPassPassPassPass
typeddicts_class_syntax - Partial -
-

Does not support version-conditional items in TypedDict definitions.

-
- 		PassPass - Partial -
-

Does not support version-conditional items in TypedDict definitions.

-
- 		PassPass
typeddicts_extra_items - Unsupported -
-

Not supported.

-
- 		PassPassPassPassPass
typeddicts_finalPassPassPassPassPassPass
typeddicts_inheritancePassPassPassPassPassPass
typeddicts_operationsPassPassPassPassPassPass
typeddicts_readonlyPassPassPassPassPassPass
typeddicts_readonly_consistencyPassPassPassPassPassPass
typeddicts_readonly_inheritance - Partial -
-

Incorrectly rejects non-ReadOnly override of ReadOnly item.

-

Incorrectly rejects override of ReadOnly item with another ReadOnly item with narrower type.

-

Incorrectly rejects override of NotRequired ReadOnly item with a Required ReadOnly item.

-
- 		PassPassPassPassPass
typeddicts_readonly_kwargsPassPassPassPassPassPass
typeddicts_readonly_update - Partial -
-

Incorrectly allows update of ReadOnly item.

-

Incorrectly rejects update involving an item with Never type.

-
- 		PassPassPassPassPass
typeddicts_requiredPassPassPassPassPassPass
typeddicts_type_consistencyPass - Partial -
-

Considers TypedDicts to be assignable to plain dict types.

-
- 		PassPassPassPass
typeddicts_usagePassPassPassPassPassPass
- Tuples -
tuples_type_compat - Partial -
-

Does not support tuple narrowing based on `len()` type guard (optional).

-

Incorrectly marks a match case as unreachable.

-
- 		PassPassPassPassPass
tuples_type_formPassPassPassPassPassPass
tuples_unpacked - Partial -
-

"More than one unpack" error is missing in some cases.

-
- 		PassPassPassPassPass
- Named tuples -
namedtuples_define_class - Partial -
-

Does not reject override of named tuple attribute in child class.

-

Does not support version-conditional fields.

-
- 		- Partial -
-

Does not support precise type inference for slices over namedtuples.

-
- 		PassPassPassPass
namedtuples_define_functionalPassPassPassPassPassPass
namedtuples_type_compatPassPassPassPassPassPass
namedtuples_usage - Partial -
-

Does not reject attempt to delete named tuple field by name.

-
- 		PassPassPassPassPass
- Enumerations -
enums_behaviorsPassPassPassPassPassPass
enums_definitionPass - Partial -
-

Does not allow enum members to be conditional on version/platform checks.

-
- 		PassPassPassPass
enums_expansion - Partial -
-

Improperly applies narrowing to Flag subclass.

-
- 		PassPassPass - Partial -
-

Does not support `enum.Flag`.

-
- 		Pass
enums_member_names - Pass -
-

Does not support special-cased handling of member name literal types in some cases (optional).

-
- 		PassPassPassPassPass
enums_member_values - Partial -
-

Does not enforce declared type of `_value_`.

-

Does not enforce assigned tuple types for enum members (optional).

-
- 		PassPassPassPassPass
enums_members - Partial -
-

Does not treat attribute with annotation and no assignment as non-member.

-

Does not treat callables as non-members.

-

Does not honor `enum.member` as method decorator.

-

Does not properly handle aliased enum members.

-

Does not support `_ignore_` mechanism (optional).

-

Does not treat attributes with private names as non-members.

-
- 		PassPass - Pass -
-

Does not support `_ignore_` mechanism (optional).

-
- 		PassPass
- Type narrowing -
narrowing_typeguardPassPassPassPassPassPass
narrowing_typeisPassPassPassPass - Partial -
-

Intersects the pre-existing type with the top materialization of the bracketed type rather than the bracketed type itself.

-
- 		Pass
- Type checker directives -
directives_assert_typePassPassPassPassPassPass
directives_castPassPassPassPassPassPass
directives_deprecatedPassPassPassPass - Partial -
-

Does not detect calls to deprecated overloads.

-

Does not detect implicit calls to deprecated dunder methods, for example via operators.

-

Does not detect accesses of, or attempts to set, deprecated properties.

-
- 		Pass
directives_disjoint_basePass - Partial -
-

Does not reject invalid class definitions due to disjoint bases, but uses disjoint base information in type narrowing.

-
- 		Pass - Unsupported -
-

Does not support PEP 800 disjoint-base semantics.

-
- 		PassPass
directives_no_type_check - Partial -
-

Does not honor `@no_type_check` class decorator (allowed).

-

Does not reject invalid call of `@no_type_check` function.

-
- 		PassPass - Pass -
-

Does not honor `@no_type_check` class decorator (allowed).

-
- 		PassPass
directives_reveal_typePassPassPassPassPassPass
directives_type_checkingPassPassPassPassPassPass
directives_type_ignore - Partial -
-

Does not honor "# type: ignore" comment if comment includes additional text.

-
- 		PassPassPassPassPass
directives_type_ignore_file1PassPassPassPassPassPass
directives_type_ignore_file2PassPassPassPassPassPass
directives_version_platform - Pass -
-

Does not understand three-element form of sys.version checks.

-

Does not understand os.name checks.

-
- 		PassPassPassPassPass
- Historical and deprecated features -
historical_positional - Partial -
-

Does not reject positional-only parameter after non-positional-only parameter.

-

Treats keyword-only parameter as positional-only.

-

Applies legacy positional-only rules when PEP 570 syntax is used.

-
- 		PassPassPassPassPass
-
-
+
+

Python Type System Conformance Test Results

+

+ While specification conformance is important for the ecosystem, we don't recommend + using it as the primary basis for choosing a type checker. It is not representative + of many of the things users typically care about. +

+
+ + +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
mypy 2.1.0pycroscope 0.4.0pyrefly 1.1.0pyright 1.1.410ty 0.0.50zuban 0.8.2
+ Type annotations +
annotations_coroutinesPassPassPassPassPassPass
annotations_forward_refs + Partial +
    +
  • Does not report error for a forward reference that is not enclosed in quotes.
    • +
  • Does not report error for use of quoted type with "|" operator (runtime error).
    • +
  • Incorrectly generates error for quoted type defined in class scope.
    • +
+ 		+ Partial +
    +
  • Fails to reject "x" | int annotations that fail at runtime.
    • +
  • Rejects some valid quoted annotations.
    • +
+ 		+ Partial +
    +
  • Types in quotes incorrectly refer to shadowing class member.
    • +
  • Does not reject some type forms that require quotes.
    • +
+ 		Pass + Partial +
    +
  • Resolves references in type annotations as referring to end-of-scope types (https://discuss.python.org/t/annotation-string-references-in-class-scope-in-conformance-tests/105439, https://github.com/python/typing/pull/2144)
    • +
+ 		+ Partial +
    +
  • Incorrectly generates error for quoted type defined in class scope.
    • +
+
annotations_generators + Partial +
    +
  • Does not report incompatible Generator type in `yield from` statement.
    • +
+ 		PassPassPassPassPass
annotations_methods + Pass +
    +
  • Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.
    • +
+ 		PassPass + Pass +
    +
  • Type evaluation differs from other type checkers because of ambiguity in the spec related to method bindings.
    • +
+ 		PassPass
annotations_typeexprPass + Partial +
    +
  • Fails to reject various weird annotations
    • +
+ 		PassPassPassPass
+ Type forms +
typeforms_typeform + Partial +
    +
  • Does not support assigning Union and GenericAlias objects to their runtime types.
    • +
+ 		+ Partial +
    +
  • Fails to reject various weird annotations
    • +
+ 		PassUnsupportedPassPass
+ Special types in annotations +
specialtypes_anyPassPassPassPassPassPass
specialtypes_neverPass + Partial +
    +
  • Does not enforce invariance in some contexts
    • +
  • Does not allow Any to be assigned to Never
    • +
+ 		PassPassPassPass
specialtypes_nonePassPassPassPassPassPass
specialtypes_promotionsPassPassPassPassPassPass
specialtypes_type + Partial +
    +
  • Does not treat `type` same as `type[Any]` for assert_type.
    • +
  • Does not allow access to unknown attributes from object of type `type[Any]`.
    • +
+ 		PassPassPass + Partial +
    +
  • Allows arbitrary attributes to be accessed on `TA` where `TA = typing.Type[typing.Any]` or `TA = type[typing.Any]`.
    • +
  • Treats `type` equivalently to `type[object]` rather than `type[typing.Any]`.
    • +
+ 		Pass
+ Generics +
generics_base_class + Partial +
    +
  • Does not detect inconsistent type variable ordering.
    • +
+ 		PassPassPassPassPass
generics_basicPassPassPassPass + Partial +
    +
  • Incorrectly allows constrained type variables to be solved to a union of their constraints.
    • +
+ 		Pass
generics_defaults + Partial +
    +
  • Does not detect a TypeVar with a default used after a TypeVarTuple.
    • +
  • Does not fully support defaults on TypeVarTuple and ParamSpec.
    • +
+ 		+ Partial +
    +
  • Numerous issues; does not support TypeVarTuple and ParamSpec defaults.
    • +
+ 		PassPass + Partial +
    +
  • Does not forbid a `TypeVar` immediately following a `TypeVarTuple` in a parameter list from having a default.
    • +
  • Does not support `TypeVarTuple`.
    • +
+ 		Pass
generics_defaults_referential + Partial +
    +
  • Does not correctly handle defaults referencing other TypeVars.
    • +
+ 		+ Partial +
    +
  • Fails to apply default specializations in some cases.
    • +
+ 		PassPassPassPass
generics_defaults_specialization + Partial +
    +
  • Does not correctly resolve defaults when classes are used directly.
    • +
+ 		PassPass + Partial +
    +
  • Allows incorrect assignment to type[].
    • +
+ 		+ Partial +
    +
  • Does not reject subscription of an already-specialized generic class.
    • +
+ 		Pass
generics_paramspec_basicPassPassPassPassPassPass
generics_paramspec_componentsPassPassPassPass + Partial +
    +
  • Incorrectly allows using `*args: P.args` and `**kwargs: P.kwargs` when `P` has not been put into scope by any other parameter annotation or enclosing scope.
    • +
+ 		Pass
generics_paramspec_semanticsPassPassPass + Pass +
    +
  • Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).
    • +
+ 		PassPass
generics_paramspec_specializationPassPassPassPassPassPass
generics_scopingPassPass + Partial +
    +
  • Does not implement several scoping checks/restrictions for generics
    • +
+ 		Pass + Partial +
    +
  • Does not reject `list[T]()` in the global scope, where `T` is an unbound type variable.
    • +
  • Does nto reject `alias: TypeAlias = list[T]` in the body scope of a class generic over a type variable `T`.
    • +
+ 		Pass
generics_self_advanced + Partial +
    +
  • Does not infer the type of an unannotated `self` parameter to be type `Self`.
    • +
  • Does not retain `Self` when calling method that returns `Self`.
    • +
  • Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`.
    • +
  • Does not retain `Self` when accessing attribute through `type[Self]`.
    • +
+ 		Pass + Pass +
    +
  • Treats attributes not initialized on the class as instance-only
    • +
+ 		PassPassPass
generics_self_attributesPassPassPassPassPassPass
generics_self_basicPassPassPassPassPassPass
generics_self_protocolsPassPassPassPassPassPass
generics_self_usagePassPassPassPass + Partial +
    +
  • Does not reject `Self` used in a return annotation when `self` is annotated using another type variable.
    • +
+ 		Pass
generics_syntax_compatibilityPassPassPassPassPassPass
generics_syntax_declarationsPassPassPassPassPassPass
generics_syntax_infer_variance + Unsupported +
    +
  • Type parameter syntax not yet supported.
    • +
+ 		PassPassPassPassPass
generics_syntax_scoping + Partial +
    +
  • Does not following runtime scoping rules for type parameters in all cases.
    • +
+ 		+ Partial +
    +
  • Misses some details of scoping rules.
    • +
+ 		PassPassPassPass
generics_type_erasure + Partial +
    +
  • Infers Node[Never] instead of Node[Any] when argument is not provided.
    • +
  • False negative on instance attribute access on type(node).
    • +
+ 		PassPassPass + Partial +
    +
  • Does not reject access of generic instance variable from the class object.
    • +
+ 		Pass
generics_typevartuple_args + Partial +
    +
  • Does not enforce that tuples captured by TypeVarTuple are of the same length.
    • +
+ 		PassPass + Partial +
    +
  • Does not correctly solve TypeVarTuple with heterogeneous bounds.
    • +
+ 		+ Partial +
    +
  • Supports PEP-646 unpacked tuples but not TypeVarTuple.
    • +
+ 		Pass
generics_typevartuple_basic + Partial +
    +
  • Does not enforce that tuples captured by TypeVarTuple are same length.
    • +
+ 		PassPass + Partial +
    +
  • Does not correctly solve TypeVarTuple with heterogeneous bounds.
    • +
+ 		UnsupportedPass
generics_typevartuple_callablePassPassPassPassUnsupportedPass
generics_typevartuple_concatPassPassPassPassUnsupportedPass
generics_typevartuple_overloadsPassPassPassPassPassPass
generics_typevartuple_specialization + Partial +
    +
  • Incorrectly specializes generic alias that includes a TypeVar and TypeVarTuple if no type arguments are provided.
    • +
  • Rejects specialization of generic type alias defined as a tuple containing a TypeVar.
    • +
+ 		PassPassPassUnsupportedPass
generics_typevartuple_unpackPassPassPassPassUnsupportedPass
generics_upper_bound + Partial +
    +
  • Does not reject use of type variable within an upper bound.
    • +
+ 		PassPassPassPassPass
generics_variance + Partial +
    +
  • Does not reject use of class-scoped TypeVar used in a base class when variance is incompatible.
    • +
+ 		PassPassPassPassPass
generics_variance_inferencePassPassPassPassPassPass
+ Type qualifiers +
qualifiers_annotated + Partial +
    +
  • Does not allow ClassVar to be nested within Annotated.
    • +
  • Does not allow Final to be nested within Annotated.
    • +
  • Does not allow Required and NotRequired to be nested within Annotated.
    • +
  • Does not reject type[T] compatibility for type alias defined with Annotated.
    • +
  • Does not reject call of type alias defined with Annotated.
    • +
+ 		+ Partial +
    +
  • Fails to reject various weird annotations.
    • +
  • False positive on lambda in Annotated.
    • +
+ 		PassPassPassPass
qualifiers_final_annotation + Partial +
    +
  • Does not treat use of Final name as if it was replaced by the literal in NamedTuple definition.
    • +
  • Does not allow conditional assignment of Final instance variable in __init__ method.
    • +
  • Does not allow redefinition of private class variable that is marked Final in parent class.
    • +
  • Does not report modification of local Final variable via "for" statement.
    • +
+ 		PassPassPassPassPass
qualifiers_final_decoratorPassPassPassPassPassPass
+ Class type compatibility +
classes_classvar + Partial +
    +
  • Internal error if TypeVarTuple is used in ClassVar.
    • +
  • Does not reject use of ParamSpec in ClassVar.
    • +
  • Rejects ClassVar nested in Annotated.
    • +
  • Does not reject use of ClassVar in TypeAlias definition.
    • +
+ 		PassPassPassPassPass
classes_overridePassPassPassPassPassPass
+ Type aliases +
aliases_explicit + Partial +
    +
  • Does not reject specialization of type alias that has already been implicitly specialized.
    • +
+ 		PassPassPass + Partial +
    +
  • Does not emit a diagnostic if a type alias to a union, in which every element of the union is implicitly specialized with `Unknown`, is invalidly specialized again.
    • +
+ 		Pass
aliases_implicitPass + Partial +
    +
  • Fails to handle various weird annotations.
    • +
  • Various bugs with resolving generic aliases.
    • +
+ 		PassPass + Partial +
    +
  • Does not reject variables with `Any` or `Unknown` types when used as implicit type aliases.
    • +
  • Falls short on full syntactic validation of type aliases.
    • +
+ 		Pass
aliases_newtype + Partial +
    +
  • `NewType`s are incorrectly considered to be classes.
    • +
+ 		PassPassPassPassPass
aliases_recursivePassPassPassPassUnsupportedPass
aliases_type_statementPass + Partial +
    +
  • Fails to reject various weird annotations.
    • +
+ 		PassPass + Partial +
    +
  • Does not reject circular definitions of type aliases.
    • +
  • Does not support `type` statements generic over `TypeVarTuple`s.
    • +
+ 		Pass
aliases_typealiastype + Partial +
    +
  • Incorrectly rejects some recursive type aliases using TypeAliasType.
    • +
  • Incorrectly rejects the use of a class-scoped TypeVar in a TypeAliasType definition.
    • +
+ 		+ Partial +
    +
  • Rejects valid ParamSpec specialization.
    • +
+ 		PassPass + Partial +
    +
  • Does not reject specializing a type parameter in a generic type alias with a type inconsistent with the parameter's upper bound.
    • +
  • Does not reject declaring a type alias with a type variable that is not in scope.
    • +
  • Does not reject declaring a type alias with a non-literal tuple passed to the `type_params` parameter.
    • +
  • Does not reject cyclically defined type aliases.
    • +
+ 		Pass
aliases_variancePassPassPassPassPassPass
+ Literals +
literals_interactions + Pass +
    +
  • Does not narrow `str` or `LiteralString` types to `Literal` string types via equality or containment checks.
    • +
+ 		PassPassPass + Pass +
    +
  • Deliberately does not allow `str` to be narrowed to literal string types through equality or containment checks due to the possibility of `str` subclasses that could have unexpected equality semantics.
    • +
+ 		Pass
literals_literalstring + Unsupported +
    +
  • Support for `LiteralString` is not implemented.
    • +
+ 		PassPassPassPassPass
literals_parameterizations + Partial +
    +
  • Does not reject tuple within Literal.
    • +
+ 		+ Partial +
    +
  • Fails to reject various invalid literal parameterizations.
    • +
+ 		PassPassPassPass
literals_semanticsPassPassPassPassPassPass
+ Protocols +
protocols_class_objectsPass + Partial +
    +
  • Abstract type[Proto] still allows protocol class objects in some paths.
    • +
+ 		PassPass + Unsupported +
    +
  • `type[Proto]` is not yet supported.
    • +
  • `ClassVar` protocol members are not yet supported.
    • +
  • `@property` protocol members are only partially supported.
    • +
  • A class object `C` is only considered to inhabit a protocol type with a method member `f` if `f` exists as an attribute on the metaclass of `C`.
    • +
+ 		Pass
protocols_definition + Partial +
    +
  • Does not detect protocol mismatch if concrete method is missing annotations.
    • +
  • Does not detect protocol mismatch if concrete method's parameters are position-only.
    • +
+ 		PassPassPass + Partial +
    +
  • Does not reject implicit instance attributes in `Protocol` methods.
    • +
  • Does not support `ClassVar` protocol members.
    • +
  • Incorrectly considers `ClassVar` attributes on concrete classes as satisfying non-`ClassVar` attribute members on protocols.
    • +
  • Only has partial support for `@property` protocol members.
    • +
+ 		Pass
protocols_explicit + Pass +
    +
  • Does not report unimplemented attributes for class that explicitly derives from protocol until it is instantiated.
    • +
+ 		PassPassPass + Unsupported +
    +
  • Allows implicitly abstract protocol methods to be called via `super()` on a protocol subclass.
    • +
  • Allows instantiation of abstract subclasses of protocol classes.
    • +
+ 		Pass
protocols_genericPass + Partial +
    +
  • Fails to reject duplicate generic/protocol bases.
    • +
  • Treats global object as a literal.
    • +
+ 		PassPass + Partial +
    +
  • Only partially supports `@property` protocol members.
    • +
+ 		Pass
protocols_mergingPassPassPassPass + Partial +
    +
  • Does not reject attempted instantiation of abstract subclasses of protocols.
    • +
+ 		Pass
protocols_modulesPassPassPassPass + Partial +
    +
  • Never considers a module as satisfying a protocol with a method member due to the fact that the method will never exist on the class `types.ModuleType`, only on a given specific module instance.
    • +
+ 		Pass
protocols_recursivePassPassPassPass + Partial +
    +
  • Fails to solve a type variable involving a recursive generic protocol.
    • +
+ 		Pass
protocols_runtime_checkable + Partial +
    +
  • Does not report unsafe overlap for runtime_checkable protocol.
    • +
+ 		PassPassPass + Partial +
    +
  • Does not reject `isinstance()` or `issubclass()` calls against runtime-checkable protocols where there is an unsafe overlap between the type of the first argument and the protocol.
    • +
+ 		Pass
protocols_selfPassPassPassPassPassPass
protocols_subtypingPassPassPassPassPassPass
protocols_variancePassPassPassPassUnsupportedPass
+ Callables +
callables_annotation + Partial +
    +
  • Incorrectly treats "*args: T, **kwargs: T" as "..." when T is specialized to Any.
    • +
  • Does not treat "*args: Any, **kargs: Any" as "..." when separated by keyword parameter.
    • +
+ 		Pass + Partial +
    +
  • Parameter names are lost when resolving ParamSpec
    • +
+ 		Pass + Partial +
    +
  • Infers a callback protocol as being a gradual type if the callback has signature `__call__[T](self, *args: T, **kwargs: T)` and `T` has been explicitly specialized to `Any`.
    • +
+ 		Pass
callables_kwargs + Partial +
    +
  • Allows callable without kwargs to be assigned to callable with unpacked kwargs
    • +
+ 		PassPassPassPassPass
callables_protocolPassPassPassPassPassPass
callables_subtypingPassPassPassPassPassPass
+ Constructors +
constructors_call_init + Partial +
    +
  • Does not report errors during binding to self parameter of __init__ method.
    • +
  • Does not reject use of class-scoped type variables in annotation of self parameter in __init__ method.
    • +
+ 		PassPassPass + Partial +
    +
  • Does not reject invalid argument types to an inherited constructor in a specialized subclass of a generic superclass.
    • +
  • Does not reject class-scoped type variables used in the `self` annotation.
    • +
+ 		Pass
constructors_call_metaclass + Unsupported +
    +
  • Does not honor metaclass __call__ method when evaluating constructor call.
    • +
  • Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.
    • +
+ 		PassPassPassPassPass
constructors_call_new + Partial +
    +
  • Does not support __new__ return type that is not a subclass of the class being constructed.
    • +
  • Does not skip evaluation of __init__ based on __new__ return type.
    • +
  • Does not report errors during binding to cls parameter of __new__ method.
    • +
+ 		PassPassPassPassPass
constructors_call_type + Partial +
    +
  • Does not validate call to custom metaclass __call__ method through type[T].
    • +
+ 		PassPassPass + Partial +
    +
  • Has overly lenient handling of calls to `type[T]` if `T` is a type variable without an upper bound.
    • +
+ 		Pass
constructors_callable + Partial +
    +
  • Does not generate a union type for __new__ and __init__ when converting class to callable.
    • +
  • Does not ignore __init__ based on __new__ return type when converting class to callable.
    • +
  • Does not support __new__ return type that is different from class being constructed.
    • +
+ 		Pass + Partial +
    +
  • Converting constructor to callable does not preserve class-scoped type params.
    • +
+ 		Pass + Partial +
    +
  • Does not include `__init__` when `__new__` returns `Self`.
    • +
  • Does not ignore `__init__` when `__new__` returns `Any`.
    • +
  • Unions overload return types.
    • +
+ 		Pass
constructors_consistency + Pass +
    +
  • Does not report inconsistency between __new__ and __init__ (optional).
    • +
+ 		PassPassPassPassPass
+ Overloads +
overloads_basicPass + Partial +
    +
  • Does less literal promotion than the test asks for.
    • +
+ 		PassPassPassPass
overloads_consistencyPassPassPassPassPassPass
overloads_definitions + Partial +
    +
  • Allows @override to be on all overloads and implementation, instead of just implementation.
    • +
+ 		PassPassPassPassPass
overloads_definitions_stub + Partial +
    +
  • Allows @override to appear in a stub file not on the first overload.
    • +
+ 		+ Unsupported +
    +
  • Does not support checking stubs.
    • +
+ 		PassPassPassPass
overloads_evaluation + Partial +
    +
  • Does not expand boolean arguments to Literal[True] and Literal[False].
    • +
  • Does not expand enum arguments to literal variants.
    • +
  • Does not expand tuple arguments to possible combinations.
    • +
  • Does not evaluate Any in some cases where overload is ambiguous.
    • +
  • Evaluates Any in some cases where overload is not ambiguous.
    • +
+ 		PassPass + Partial +
    +
  • Does not evaluate Any in some cases where overload is ambiguous.
    • +
+ 		PassPass
+ Exceptions +
exceptions_context_managersPassPass + Partial +
    +
  • Some error suppressing context managers are not detected
    • +
+ 		PassUnsupportedPass
+ Dataclasses +
dataclasses_descriptors + Partial +
    +
  • Assumes descriptor behavior only when field is assigned in class body.
    • +
  • Does not correctly evaluate type of descriptor access.
    • +
+ 		+ Partial +
    +
  • Conformance suite is questionable; see https://github.com/python/typing/issues/2259
    • +
+ 		+ Partial +
    +
  • * Assumes descriptor behavior only when field is assigned in class body
    • +
  • * Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types
    • +
+ 		Pass + Partial +
    +
  • Only infers a descriptor `__get__` method as being called when a descriptor attribute is accessed on an instance if the descriptor attribute is present in the class namespace.
    • +
+ 		Pass
dataclasses_final + Partial +
    +
  • Wrongly requires a Final dataclass field to be initialized at class level.
    • +
  • Doesn't support Final nested inside ClassVar.
    • +
+ 		PassPassPassPassPass
dataclasses_frozenPassPassPassPassPassPass
dataclasses_hash + Unsupported +
    +
  • Does not synthesize `__hash__ = None` as a class attribute for unhashable dataclasses.
    • +
  • Does not report when an unhashable dataclass has `__hash__` called directly on an instance.
    • +
  • Does not report when dataclass is not compatible with Hashable protocol.
    • +
+ 		PassPassPass + Partial +
    +
  • Understands the `Hashable` protocol as equivalent to `object`.
    • +
+ 		Pass
dataclasses_inheritancePassPassPassPassPassPass
dataclasses_kwonlyPassPassPassPassPassPass
dataclasses_match_argsPassPassPassPassPassPass
dataclasses_orderPassPassPassPassPassPass
dataclasses_postinitPassPassPassPassPassPass
dataclasses_slots + Partial +
    +
  • Does not reject write to instance variable that is not defined in __slots__.
    • +
+ 		PassPassPass + Partial +
    +
  • Synthesizes a `__slots__` attribute but does not validate attribute assignments against `__slots__`.
    • +
+ 		Pass
dataclasses_transform_classPassPassPassPassPassPass
dataclasses_transform_converter + Unsupported +
    +
  • Converter parameter not yet supported.
    • +
+ 		PassPassPassPassPass
dataclasses_transform_field + Partial +
    +
  • Does not properly handle field constructor that has default value for `kw_only` or `init` parameter.
    • +
+ 		PassPassPassPassPass
dataclasses_transform_func + Partial +
    +
  • Does not handle `kw_only=False` override when `kw_only_default=True`.
    • +
  • Does not report error when `order=False` and comparison operators are used.
    • +
+ 		PassPassPassPassPass
dataclasses_transform_metaPassPassPassPassPassPass
dataclasses_usage + Pass +
    +
  • Does not detect unannotated usage of `dataclasses.field()`.
    • +
+ 		PassPassPassPassPass
+ Typed dictionaries +
typeddicts_alt_syntax + Pass +
    +
  • Does not support keyword-argument form of alternative syntax (deprecated in 3.11).
    • +
+ 		PassPassPassPassPass
typeddicts_class_syntax + Partial +
    +
  • Does not support version-conditional items in TypedDict definitions.
    • +
+ 		PassPass + Partial +
    +
  • Does not support version-conditional items in TypedDict definitions.
    • +
+ 		PassPass
typeddicts_extra_items + Unsupported +
    +
  • Not supported.
    • +
+ 		PassPassPassPassPass
typeddicts_finalPassPassPassPassPassPass
typeddicts_inheritancePassPassPassPassPassPass
typeddicts_operationsPassPassPassPassPassPass
typeddicts_readonlyPassPassPassPassPassPass
typeddicts_readonly_consistencyPassPassPassPassPassPass
typeddicts_readonly_inheritance + Partial +
    +
  • Incorrectly rejects non-ReadOnly override of ReadOnly item.
    • +
  • Incorrectly rejects override of ReadOnly item with another ReadOnly item with narrower type.
    • +
  • Incorrectly rejects override of NotRequired ReadOnly item with a Required ReadOnly item.
    • +
+ 		PassPassPassPassPass
typeddicts_readonly_kwargsPassPassPassPassPassPass
typeddicts_readonly_update + Partial +
    +
  • Incorrectly allows update of ReadOnly item.
    • +
  • Incorrectly rejects update involving an item with Never type.
    • +
+ 		PassPassPassPassPass
typeddicts_requiredPassPassPassPassPassPass
typeddicts_type_consistencyPass + Partial +
    +
  • Considers TypedDicts to be assignable to plain dict types.
    • +
+ 		PassPassPassPass
typeddicts_usagePassPassPassPassPassPass
+ Tuples +
tuples_type_compat + Partial +
    +
  • Does not support tuple narrowing based on `len()` type guard (optional).
    • +
  • Incorrectly marks a match case as unreachable.
    • +
+ 		PassPassPassPassPass
tuples_type_formPassPassPassPassPassPass
tuples_unpacked + Partial +
    +
  • "More than one unpack" error is missing in some cases.
    • +
+ 		PassPassPassPassPass
+ Named tuples +
namedtuples_define_class + Partial +
    +
  • Does not reject override of named tuple attribute in child class.
    • +
  • Does not support version-conditional fields.
    • +
+ 		+ Partial +
    +
  • Does not support precise type inference for slices over namedtuples.
    • +
+ 		PassPassPassPass
namedtuples_define_functionalPassPassPassPassPassPass
namedtuples_type_compatPassPassPassPassPassPass
namedtuples_usage + Partial +
    +
  • Does not reject attempt to delete named tuple field by name.
    • +
+ 		PassPassPassPassPass
+ Enumerations +
enums_behaviorsPassPassPassPassPassPass
enums_definitionPass + Partial +
    +
  • Does not allow enum members to be conditional on version/platform checks.
    • +
+ 		PassPassPassPass
enums_expansion + Partial +
    +
  • Improperly applies narrowing to Flag subclass.
    • +
+ 		PassPassPass + Partial +
    +
  • Does not support `enum.Flag`.
    • +
+ 		Pass
enums_member_names + Pass +
    +
  • Does not support special-cased handling of member name literal types in some cases (optional).
    • +
+ 		PassPassPassPassPass
enums_member_values + Partial +
    +
  • Does not enforce declared type of `_value_`.
    • +
  • Does not enforce assigned tuple types for enum members (optional).
    • +
+ 		PassPassPassPassPass
enums_members + Partial +
    +
  • Does not treat attribute with annotation and no assignment as non-member.
    • +
  • Does not treat callables as non-members.
    • +
  • Does not honor `enum.member` as method decorator.
    • +
  • Does not properly handle aliased enum members.
    • +
  • Does not support `_ignore_` mechanism (optional).
    • +
  • Does not treat attributes with private names as non-members.
    • +
+ 		PassPass + Pass +
    +
  • Does not support `_ignore_` mechanism (optional).
    • +
+ 		PassPass
+ Type narrowing +
narrowing_typeguardPassPassPassPassPassPass
narrowing_typeisPassPassPassPass + Partial +
    +
  • Intersects the pre-existing type with the top materialization of the bracketed type rather than the bracketed type itself.
    • +
+ 		Pass
+ Type checker directives +
directives_assert_typePassPassPassPassPassPass
directives_castPassPassPassPassPassPass
directives_deprecatedPassPassPassPass + Partial +
    +
  • Does not detect calls to deprecated overloads.
    • +
  • Does not detect implicit calls to deprecated dunder methods, for example via operators.
    • +
  • Does not detect accesses of, or attempts to set, deprecated properties.
    • +
+ 		Pass
directives_disjoint_basePass + Partial +
    +
  • Does not reject invalid class definitions due to disjoint bases, but uses disjoint base information in type narrowing.
    • +
+ 		Pass + Unsupported +
    +
  • Does not support PEP 800 disjoint-base semantics.
    • +
+ 		PassPass
directives_no_type_check + Partial +
    +
  • Does not honor `@no_type_check` class decorator (allowed).
    • +
  • Does not reject invalid call of `@no_type_check` function.
    • +
+ 		PassPass + Pass +
    +
  • Does not honor `@no_type_check` class decorator (allowed).
    • +
+ 		PassPass
directives_reveal_typePassPassPassPassPassPass
directives_type_checkingPassPassPassPassPassPass
directives_type_ignore + Partial +
    +
  • Does not honor "# type: ignore" comment if comment includes additional text.
    • +
+ 		PassPassPassPassPass
directives_type_ignore_file1PassPassPassPassPassPass
directives_type_ignore_file2PassPassPassPassPassPass
directives_version_platform + Pass +
    +
  • Does not understand three-element form of sys.version checks.
    • +
  • Does not understand os.name checks.
    • +
+ 		PassPassPassPassPass
+ Historical and deprecated features +
historical_positional + Partial +
    +
  • Does not reject positional-only parameter after non-positional-only parameter.
    • +
  • Treats keyword-only parameter as positional-only.
    • +
  • Applies legacy positional-only rules when PEP 570 syntax is used.
    • +
+ 		PassPassPassPassPass
+
+ \ No newline at end of file diff --git a/conformance/src/templates/base.html b/conformance/src/templates/base.html index 08067e97e..0cb9a47c6 100644 --- a/conformance/src/templates/base.html +++ b/conformance/src/templates/base.html @@ -3,8 +3,9 @@ - - Type System Test Results + + + Python Type System Conformance Test Results -
-
-

Python Type System Conformance Test Results

-

- While spec conformance is important for the ecosystem, we don't recommend using - it as the primary basis for choosing a type checker. It is not representative - of many of the things users typically care about. -

-
-
- - - - - - - - - {%- for version in versions %} - - {%- endfor %} - - - - {%- for group in groups %} - - - - {%- for case in group.cases %} - - - {%- for result in case.results %} - {%- if result.notes %} - - {%- else %} - - {%- endif %} - {%- endfor %} - +
+

Python Type System Conformance Test Results

+

+ While specification conformance is important for the ecosystem, we don't recommend + using it as the primary basis for choosing a type checker. It is not representative + of many of the things users typically care about. +

+
+ + + +
+
+
{{ version }}
- {{ group.name }} -
{{ case.name }} - {{ result.conformance }} -
- {%- for note in result.notes %} -

{{ note }}

- {%- endfor %} -
- 		{{ result.conformance }}
+ + + + + + + + {%- for version in versions %} + {%- endfor %} + + + + {%- for group in groups %} + + + + {%- for case in group.cases %} + + + {%- for result in case.results %} + {%- if result.notes %} + + {%- else %} + + {%- endif %} {%- endfor %} - -
{{ version }}
+ {{ group.name }} +
{{ case.name }} + {{ result.conformance }} +
    + {%- for note in result.notes %} +
  • {{ note }}
    • + {%- endfor %} +
+ 		{{ result.conformance }}
-
-
+ + {%- endfor %} + {%- endfor %} + + + + diff --git a/conformance/src/templates/style.css b/conformance/src/templates/style.css index aa7709aff..bd2835010 100644 --- a/conformance/src/templates/style.css +++ b/conformance/src/templates/style.css @@ -1,162 +1,210 @@ -body, -h1, -h2, -h3, -p, -table { +:root { + --dark-main-bg: #333; + --dark-main-fg: #fff; + --dark-muted-fg: #ddd; + --dark-selection-bg: #ec6e4a; + --dark-shadow: #999; + --light-main-bg: #fff; + --light-main-fg: #000; + --light-muted-fg: #666; + --light-selection-bg: #ec6e4a; + --light-shadow: #ccc; + + --conformant: rgb(203, 255, 203); + --partially-conformant: rgb(251, 244, 166); + --not-conformant: rgb(242, 171, 171); + + color-scheme: light dark; + &:has(#color-scheme option[value="light"]:checked) { + color-scheme: light; + } + &:has(#color-scheme option[value="dark"]:checked) { + color-scheme: dark; + } +} + +* { + box-sizing: border-box; +} + +::selection { + background: light-dark(var(--light-selection-bg), var(--dark-selection-bg)); + color: light-dark(var(--dark-main-fg), var(--light-main-fg)); +} + +html { + background: light-dark(var(--light-main-bg), var(--dark-main-bg)); + color: light-dark(var(--light-main-fg), var(--dark-main-fg)); font-family: "system-ui", "Segoe UI", Helvetica, Arial, sans-serif; + font-size: 0.85rem; +} + +body { margin: 0; padding: 0; } header { - background-color: #333; + background-color: #2b5b84; + box-shadow: inset 0 0 50px rgba(0, 0, 0, 0.03), inset 0 0 20px rgba(0, 0, 0, 0.03); color: #fff; + padding: 1em; text-align: center; - padding: 10px 16px 12px; -} - -table { - margin: 0px; - padding: 0px; - width: 100%; - border-collapse: collapse; - font-family: "system-ui", "Segoe UI", Helvetica, Arial, sans-serif; - font-size: 10pt; - font-weight: normal; - font-style: normal; } -th, -td { - margin: 0px; - padding: 0px; - text-align: left; - border-bottom: 0px solid #000; - font-weight: normal; +main { + padding: 2em; } -th:first-child, -td:first-child { - vertical-align: top; +a { + text-underline-offset: 2px; } -th:not(:first-child), -td:not(:first-child){ - text-align: center; +table { + border-collapse: collapse; + inline-size: 100%; + margin: 0; + padding: 0; } -.content_container { - margin: 20px; -} +th, +td { + border-color: transparent; + border-style: solid; + border-width: 0 10px; + margin: 0; + padding: 3px; + position: relative; + text-align: start; -main { - margin-left: 20px; + &:not(:first-child) { + text-align: center; + } } -tbody > tr > th, -tbody > tr > td { - padding-left: 8px; - padding-right: 8px; - padding-top: 4px; - padding-bottom: 4px; - - margin-left: 8px; - margin-right: 8px; - margin-top: 4px; - margin-bottom: 4px; - - border-style: solid; - border-color: white; - border-left-width: 8px; - border-right-width: 8px; - border-top-width: 0; - border-bottom-width: 0; +thead { + background: light-dark(var(--light-main-bg), var(--dark-main-bg)); + inset-block-start: 0; + position: sticky; + z-index: 3; + & > tr > th { + padding-block: 1em; + } } tbody > tr > th:not([colspan]) { - padding-left: 2em; -} - -tbody > tr > th[colspan] > a { - font-size: 10pt; - font-weight: bold; + font-weight: normal; + padding-inline-start: 2em; } .col1 { - width: 30%; + inline-size: 30%; } .col2 { - width: calc(70% / {{ columns }}); -} - -thead > tr > th { - font-size: 16px; - padding-bottom: 8px; - padding-top: 12px; - text-align: center; + inline-size: calc(70% / {{ columns }}); } .conformant { - background-color: rgb(203, 255, 203); + background-color: var(--conformant); + color: var(--light-main-fg); } .partially-conformant { - background-color: rgb(251, 244, 166); + background-color: var(--partially-conformant); + color: var(--light-main-fg); } .not-conformant { - background-color: rgb(242, 171, 171); -} - -.notes { - visibility: hidden; - position: absolute; - z-index: 2; - width: 400px; - color: black; - font-size: 12px; - background-color: white; - border: 1px solid black; - padding: 10px 15px 10px 15px; - text-align: left; -} - -.tooltip:hover .notes { - visibility: visible; -} - -.tooltip .notes p { - margin: 4px; + background-color: var(--not-conformant); + color: var(--light-main-fg); +} + +tbody > tr:hover { + th:not([colspan]) { + background-color: light-dark(#eee, #444); + border-color: light-dark(#eee, #444); + } + th:not([colspan]) ~ td { + border-color: light-dark(#eee, #444); + } + .conformant { + background-color: hsl(from var(--conformant) h s calc(l - 5)); + } + .partially-conformant { + background-color: hsl(from var(--partially-conformant) h s calc(l - 5)); + } + .not-conformant { + background-color: hsl(from var(--not-conformant) h s calc(l - 5)); + } } -.tooltip .notes { - top: 25px; - left: -80%; -} +.tooltip { + cursor: help; -.tooltip .notes::before { - top: -5%; - left: 94%; + /* Add an asterisk if there are notes to display for a "Pass". */ + &.conformant::after { + content: "*"; + position: absolute; + } + + .notes { + background-color: light-dark(var(--light-main-bg), var(--dark-main-bg)); + border-radius: 5px; + box-shadow: 0 0 3px light-dark(var(--light-shadow), var(--dark-shadow)); + color: light-dark(var(--light-main-fg), var(--dark-main-fg)); + cursor: default; + font-size: .8em; + inset-block-start: calc(50% - 5px); + inset-inline-end: 50%; + line-height: 1.6; + list-style-position: inside; + min-inline-size: 500px; + padding: 10px 15px; + position: absolute; + text-align: start; + visibility: hidden; + z-index: 2; + & > li { + text-indent: 1.3em hanging; + } + } + + &:hover .notes { + visibility: visible; + } } -.tooltip { - cursor: help; - position: relative; +.disclaimer { + color: var(--dark-muted-fg); + font-size: 0.9rem; + font-weight: normal; + line-height: 1.4; + margin: 1rem auto; + max-inline-size: 720px; } -{# Add an asterisk if there are notes to display for a "Pass". -#} -.tooltip.conformant::after { +form:has(#color-scheme) { + background: inherit; + inset-block-start: 1rem; + inset-inline-end: 1rem; position: absolute; - content: "*"; } -.disclaimer { - margin: 6px auto 0; - max-width: 720px; - font-size: 12px; - line-height: 1.4; - font-weight: normal; - color: #d6d6d6; +#color-scheme { + appearance: none; + background: inherit; + border-color: var(--dark-muted-fg); + border-radius: 5px; + border-style: solid; + border-width: 1px; + color: inherit; + padding: 0.5rem; + &:focus { + border-color: inherit; + outline: none; + } + &:hover { + border-color: inherit; + } } From 1847d038151fbd59bbb49b3d987a876ca5fc1579 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:32 +0100 Subject: [PATCH 21/30] Add markdown rendering of notes --- conformance/pyproject.toml | 2 + conformance/results/results.html | 182 +++++++++++++++------------- conformance/src/reporting.py | 15 ++- conformance/src/templates/style.css | 8 ++ conformance/uv.lock | 22 ++++ 5 files changed, 142 insertions(+), 87 deletions(-) diff --git a/conformance/pyproject.toml b/conformance/pyproject.toml index 428adad56..17c4adfd2 100644 --- a/conformance/pyproject.toml +++ b/conformance/pyproject.toml @@ -4,12 +4,14 @@ version = "0.1.0" requires-python = "==3.12.*" dependencies = [ "jinja2", + "markdown", "mypy", "pycroscope", "pyrefly", "pyright", "tomlkit", "ty", + "types-markdown", "zuban", ] diff --git a/conformance/results/results.html b/conformance/results/results.html index 7d3bf0429..965af67bc 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -68,6 +68,13 @@ text-underline-offset: 2px; } + code { + background: light-dark(#eee, #666); + border: 1px solid light-dark(#ddd, #888); + border-radius: 3px; + padding: 0 2px; + } + table { border-collapse: collapse; inline-size: 100%; @@ -174,6 +181,7 @@ visibility: hidden; z-index: 2; & > li { + padding: 3px 0; text-indent: 1.3em hanging; } } @@ -273,14 +281,14 @@

Python Type System Conformance Test Results

Partial
  • Does not report error for a forward reference that is not enclosed in quotes.
    • -
  • Does not report error for use of quoted type with "|" operator (runtime error).
    • +
  • Does not report error for use of quoted type with "|" operator (runtime error).
  • Incorrectly generates error for quoted type defined in class scope.
Partial
    -
  • Fails to reject "x" | int annotations that fail at runtime.
    • +
  • Fails to reject "x" | int annotations that fail at runtime.
  • Rejects some valid quoted annotations.
@@ -310,7 +318,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not report incompatible Generator type in `yield from` statement.
    • +
  • Does not report incompatible Generator type in yield from statement.
Pass @@ -428,8 +436,8 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not treat `type` same as `type[Any]` for assert_type.
    • -
  • Does not allow access to unknown attributes from object of type `type[Any]`.
    • +
  • Does not treat type same as type[Any] for assert_type.
    • +
  • Does not allow access to unknown attributes from object of type type[Any].
Pass @@ -438,8 +446,8 @@

Python Type System Conformance Test Results

Partial
    -
  • Allows arbitrary attributes to be accessed on `TA` where `TA = typing.Type[typing.Any]` or `TA = type[typing.Any]`.
    • -
  • Treats `type` equivalently to `type[object]` rather than `type[typing.Any]`.
    • +
  • Allows arbitrary attributes to be accessed on TA where TA = typing.Type[typing.Any] or TA = type[typing.Any].
    • +
  • Treats type equivalently to type[object] rather than type[typing.Any].
Pass @@ -497,8 +505,8 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not forbid a `TypeVar` immediately following a `TypeVarTuple` in a parameter list from having a default.
    • -
  • Does not support `TypeVarTuple`.
    • +
  • Does not forbid a TypeVar immediately following a TypeVarTuple in a parameter list from having a default.
    • +
  • Does not support TypeVarTuple.
Pass @@ -564,7 +572,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Incorrectly allows using `*args: P.args` and `**kwargs: P.kwargs` when `P` has not been put into scope by any other parameter annotation or enclosing scope.
    • +
  • Incorrectly allows using *args: P.args and **kwargs: P.kwargs when P has not been put into scope by any other parameter annotation or enclosing scope.
Pass @@ -577,7 +585,7 @@

Python Type System Conformance Test Results

Pass
    -
  • Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).
    • +
  • Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).
Pass @@ -606,8 +614,8 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not reject `list[T]()` in the global scope, where `T` is an unbound type variable.
    • -
  • Does nto reject `alias: TypeAlias = list[T]` in the body scope of a class generic over a type variable `T`.
    • +
  • Does not reject list[T]() in the global scope, where T is an unbound type variable.
    • +
  • Does nto reject alias: TypeAlias = list[T] in the body scope of a class generic over a type variable T.
Pass @@ -617,10 +625,10 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not infer the type of an unannotated `self` parameter to be type `Self`.
    • -
  • Does not retain `Self` when calling method that returns `Self`.
    • -
  • Does not infer the type of an unannotated `cls` parameter to be type `type[Self]`.
    • -
  • Does not retain `Self` when accessing attribute through `type[Self]`.
    • +
  • Does not infer the type of an unannotated self parameter to be type Self.
    • +
  • Does not retain Self when calling method that returns Self.
    • +
  • Does not infer the type of an unannotated cls parameter to be type type[Self].
    • +
  • Does not retain Self when accessing attribute through type[Self].
Pass @@ -670,7 +678,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not reject `Self` used in a return annotation when `self` is annotated using another type variable.
    • +
  • Does not reject Self used in a return annotation when self is annotated using another type variable.
Pass @@ -912,9 +920,9 @@

Python Type System Conformance Test Results

Partial
  • Does not treat use of Final name as if it was replaced by the literal in NamedTuple definition.
    • -
  • Does not allow conditional assignment of Final instance variable in __init__ method.
    • +
  • Does not allow conditional assignment of Final instance variable in init method.
  • Does not allow redefinition of private class variable that is marked Final in parent class.
    • -
  • Does not report modification of local Final variable via "for" statement.
    • +
  • Does not report modification of local Final variable via "for" statement.
Pass @@ -982,7 +990,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not emit a diagnostic if a type alias to a union, in which every element of the union is implicitly specialized with `Unknown`, is invalidly specialized again.
    • +
  • Does not emit a diagnostic if a type alias to a union, in which every element of the union is implicitly specialized with Unknown, is invalidly specialized again.
Pass @@ -1002,7 +1010,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not reject variables with `Any` or `Unknown` types when used as implicit type aliases.
    • +
  • Does not reject variables with Any or Unknown types when used as implicit type aliases.
  • Falls short on full syntactic validation of type aliases.
@@ -1013,7 +1021,7 @@

Python Type System Conformance Test Results

Partial
    -
  • `NewType`s are incorrectly considered to be classes.
    • +
  • NewTypes are incorrectly considered to be classes.
Pass @@ -1046,7 +1054,7 @@

Python Type System Conformance Test Results

Partial
  • Does not reject circular definitions of type aliases.
    • -
  • Does not support `type` statements generic over `TypeVarTuple`s.
    • +
  • Does not support type statements generic over TypeVarTuples.
Pass @@ -1071,9 +1079,9 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not reject specializing a type parameter in a generic type alias with a type inconsistent with the parameter's upper bound.
    • +
  • Does not reject specializing a type parameter in a generic type alias with a type inconsistent with the parameter's upper bound.
  • Does not reject declaring a type alias with a type variable that is not in scope.
    • -
  • Does not reject declaring a type alias with a non-literal tuple passed to the `type_params` parameter.
    • +
  • Does not reject declaring a type alias with a non-literal tuple passed to the type_params parameter.
  • Does not reject cyclically defined type aliases.
@@ -1098,7 +1106,7 @@

Python Type System Conformance Test Results

Pass
    -
  • Does not narrow `str` or `LiteralString` types to `Literal` string types via equality or containment checks.
    • +
  • Does not narrow str or LiteralString types to Literal string types via equality or containment checks.
Pass @@ -1107,7 +1115,7 @@

Python Type System Conformance Test Results

Pass
    -
  • Deliberately does not allow `str` to be narrowed to literal string types through equality or containment checks due to the possibility of `str` subclasses that could have unexpected equality semantics.
    • +
  • Deliberately does not allow str to be narrowed to literal string types through equality or containment checks due to the possibility of str subclasses that could have unexpected equality semantics.
Pass @@ -1117,7 +1125,7 @@

Python Type System Conformance Test Results

Unsupported
    -
  • Support for `LiteralString` is not implemented.
    • +
  • Support for LiteralString is not implemented.
Pass @@ -1173,10 +1181,10 @@

Python Type System Conformance Test Results

Unsupported
    -
  • `type[Proto]` is not yet supported.
    • -
  • `ClassVar` protocol members are not yet supported.
    • -
  • `@property` protocol members are only partially supported.
    • -
  • A class object `C` is only considered to inhabit a protocol type with a method member `f` if `f` exists as an attribute on the metaclass of `C`.
    • +
  • type[Proto] is not yet supported.
    • +
  • ClassVar protocol members are not yet supported.
    • +
  • @property protocol members are only partially supported.
    • +
  • A class object C is only considered to inhabit a protocol type with a method member f if f exists as an attribute on the metaclass of C.
Pass @@ -1187,7 +1195,7 @@

Python Type System Conformance Test Results

Partial
  • Does not detect protocol mismatch if concrete method is missing annotations.
    • -
  • Does not detect protocol mismatch if concrete method's parameters are position-only.
    • +
  • Does not detect protocol mismatch if concrete method's parameters are position-only.
Pass @@ -1196,10 +1204,10 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not reject implicit instance attributes in `Protocol` methods.
    • -
  • Does not support `ClassVar` protocol members.
    • -
  • Incorrectly considers `ClassVar` attributes on concrete classes as satisfying non-`ClassVar` attribute members on protocols.
    • -
  • Only has partial support for `@property` protocol members.
    • +
  • Does not reject implicit instance attributes in Protocol methods.
    • +
  • Does not support ClassVar protocol members.
    • +
  • Incorrectly considers ClassVar attributes on concrete classes as satisfying non-ClassVar attribute members on protocols.
    • +
  • Only has partial support for @property protocol members.
Pass @@ -1218,7 +1226,7 @@

Python Type System Conformance Test Results

Unsupported
    -
  • Allows implicitly abstract protocol methods to be called via `super()` on a protocol subclass.
    • +
  • Allows implicitly abstract protocol methods to be called via super() on a protocol subclass.
  • Allows instantiation of abstract subclasses of protocol classes.
@@ -1239,7 +1247,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Only partially supports `@property` protocol members.
    • +
  • Only partially supports @property protocol members.
Pass @@ -1267,7 +1275,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Never considers a module as satisfying a protocol with a method member due to the fact that the method will never exist on the class `types.ModuleType`, only on a given specific module instance.
    • +
  • Never considers a module as satisfying a protocol with a method member due to the fact that the method will never exist on the class types.ModuleType, only on a given specific module instance.
Pass @@ -1300,7 +1308,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not reject `isinstance()` or `issubclass()` calls against runtime-checkable protocols where there is an unsafe overlap between the type of the first argument and the protocol.
    • +
  • Does not reject isinstance() or issubclass() calls against runtime-checkable protocols where there is an unsafe overlap between the type of the first argument and the protocol.
Pass @@ -1342,8 +1350,8 @@

Python Type System Conformance Test Results

Partial
    -
  • Incorrectly treats "*args: T, **kwargs: T" as "..." when T is specialized to Any.
    • -
  • Does not treat "*args: Any, **kargs: Any" as "..." when separated by keyword parameter.
    • +
  • Incorrectly treats "args: T, *kwargs: T" as "..." when T is specialized to Any.
    • +
  • Does not treat "args: Any, *kargs: Any" as "..." when separated by keyword parameter.
Pass @@ -1357,7 +1365,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Infers a callback protocol as being a gradual type if the callback has signature `__call__[T](self, *args: T, **kwargs: T)` and `T` has been explicitly specialized to `Any`.
    • +
  • Infers a callback protocol as being a gradual type if the callback has signature __call__[T](self, *args: T, **kwargs: T) and T has been explicitly specialized to Any.
Pass @@ -1404,8 +1412,8 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not report errors during binding to self parameter of __init__ method.
    • -
  • Does not reject use of class-scoped type variables in annotation of self parameter in __init__ method.
    • +
  • Does not report errors during binding to self parameter of init method.
    • +
  • Does not reject use of class-scoped type variables in annotation of self parameter in init method.
Pass @@ -1415,7 +1423,7 @@

Python Type System Conformance Test Results

Partial
  • Does not reject invalid argument types to an inherited constructor in a specialized subclass of a generic superclass.
    • -
  • Does not reject class-scoped type variables used in the `self` annotation.
    • +
  • Does not reject class-scoped type variables used in the self annotation.
Pass @@ -1425,8 +1433,8 @@

Python Type System Conformance Test Results

Unsupported
    -
  • Does not honor metaclass __call__ method when evaluating constructor call.
    • -
  • Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.
    • +
  • Does not honor metaclass call method when evaluating constructor call.
    • +
  • Does not skip evaluation of new and init if custom metaclass call returns non-class.
Pass @@ -1440,9 +1448,9 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not support __new__ return type that is not a subclass of the class being constructed.
    • -
  • Does not skip evaluation of __init__ based on __new__ return type.
    • -
  • Does not report errors during binding to cls parameter of __new__ method.
    • +
  • Does not support new return type that is not a subclass of the class being constructed.
    • +
  • Does not skip evaluation of init based on new return type.
    • +
  • Does not report errors during binding to cls parameter of new method.
Pass @@ -1456,7 +1464,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not validate call to custom metaclass __call__ method through type[T].
    • +
  • Does not validate call to custom metaclass call method through type[T].
Pass @@ -1465,7 +1473,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Has overly lenient handling of calls to `type[T]` if `T` is a type variable without an upper bound.
    • +
  • Has overly lenient handling of calls to type[T] if T is a type variable without an upper bound.
Pass @@ -1475,9 +1483,9 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not generate a union type for __new__ and __init__ when converting class to callable.
    • -
  • Does not ignore __init__ based on __new__ return type when converting class to callable.
    • -
  • Does not support __new__ return type that is different from class being constructed.
    • +
  • Does not generate a union type for new and init when converting class to callable.
    • +
  • Does not ignore init based on new return type when converting class to callable.
    • +
  • Does not support new return type that is different from class being constructed.
Pass @@ -1491,8 +1499,8 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not include `__init__` when `__new__` returns `Self`.
    • -
  • Does not ignore `__init__` when `__new__` returns `Any`.
    • +
  • Does not include __init__ when __new__ returns Self.
    • +
  • Does not ignore __init__ when __new__ returns Any.
  • Unions overload return types.
@@ -1503,7 +1511,7 @@

Python Type System Conformance Test Results

Pass
    -
  • Does not report inconsistency between __new__ and __init__ (optional).
    • +
  • Does not report inconsistency between new and init (optional).
Pass @@ -1638,15 +1646,19 @@

Python Type System Conformance Test Results

Partial
    -
  • * Assumes descriptor behavior only when field is assigned in class body
    • -
  • * Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types
    • +
    • +
    • Assumes descriptor behavior only when field is assigned in class body
      • +
    • +
    • +
    • Doesn't allow non-data descriptors or data descriptors with differing __get__ and __set__ types
      • +
Pass Partial
    -
  • Only infers a descriptor `__get__` method as being called when a descriptor attribute is accessed on an instance if the descriptor attribute is present in the class namespace.
    • +
  • Only infers a descriptor __get__ method as being called when a descriptor attribute is accessed on an instance if the descriptor attribute is present in the class namespace.
Pass @@ -1657,7 +1669,7 @@

Python Type System Conformance Test Results

Partial
  • Wrongly requires a Final dataclass field to be initialized at class level.
    • -
  • Doesn't support Final nested inside ClassVar.
    • +
  • Doesn't support Final nested inside ClassVar.
Pass @@ -1680,8 +1692,8 @@

Python Type System Conformance Test Results

Unsupported
    -
  • Does not synthesize `__hash__ = None` as a class attribute for unhashable dataclasses.
    • -
  • Does not report when an unhashable dataclass has `__hash__` called directly on an instance.
    • +
  • Does not synthesize __hash__ = None as a class attribute for unhashable dataclasses.
    • +
  • Does not report when an unhashable dataclass has __hash__ called directly on an instance.
  • Does not report when dataclass is not compatible with Hashable protocol.
@@ -1691,7 +1703,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Understands the `Hashable` protocol as equivalent to `object`.
    • +
  • Understands the Hashable protocol as equivalent to object.
Pass @@ -1746,7 +1758,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not reject write to instance variable that is not defined in __slots__.
    • +
  • Does not reject write to instance variable that is not defined in slots.
Pass @@ -1755,7 +1767,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Synthesizes a `__slots__` attribute but does not validate attribute assignments against `__slots__`.
    • +
  • Synthesizes a __slots__ attribute but does not validate attribute assignments against __slots__.
Pass @@ -1788,7 +1800,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not properly handle field constructor that has default value for `kw_only` or `init` parameter.
    • +
  • Does not properly handle field constructor that has default value for kw_only or init parameter.
Pass @@ -1802,8 +1814,8 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not handle `kw_only=False` override when `kw_only_default=True`.
    • -
  • Does not report error when `order=False` and comparison operators are used.
    • +
  • Does not handle kw_only=False override when kw_only_default=True.
    • +
  • Does not report error when order=False and comparison operators are used.
Pass @@ -1826,7 +1838,7 @@

Python Type System Conformance Test Results

Pass
    -
  • Does not detect unannotated usage of `dataclasses.field()`.
    • +
  • Does not detect unannotated usage of dataclasses.field().
Pass @@ -2014,7 +2026,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not support tuple narrowing based on `len()` type guard (optional).
    • +
  • Does not support tuple narrowing based on len() type guard (optional).
  • Incorrectly marks a match case as unreachable.
@@ -2038,7 +2050,7 @@

Python Type System Conformance Test Results

Partial
    -
  • "More than one unpack" error is missing in some cases.
    • +
  • "More than one unpack" error is missing in some cases.
Pass @@ -2146,7 +2158,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not support `enum.Flag`.
    • +
  • Does not support enum.Flag.
Pass @@ -2170,7 +2182,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not enforce declared type of `_value_`.
    • +
  • Does not enforce declared type of _value_.
  • Does not enforce assigned tuple types for enum members (optional).
@@ -2187,9 +2199,9 @@

Python Type System Conformance Test Results

  • Does not treat attribute with annotation and no assignment as non-member.
  • Does not treat callables as non-members.
    • -
  • Does not honor `enum.member` as method decorator.
    • +
  • Does not honor enum.member as method decorator.
  • Does not properly handle aliased enum members.
    • -
  • Does not support `_ignore_` mechanism (optional).
    • +
  • Does not support _ignore_ mechanism (optional).
  • Does not treat attributes with private names as non-members.
@@ -2198,7 +2210,7 @@

Python Type System Conformance Test Results

Pass
    -
  • Does not support `_ignore_` mechanism (optional).
    • +
  • Does not support _ignore_ mechanism (optional).
Pass @@ -2295,8 +2307,8 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not honor `@no_type_check` class decorator (allowed).
    • -
  • Does not reject invalid call of `@no_type_check` function.
    • +
  • Does not honor @no_type_check class decorator (allowed).
    • +
  • Does not reject invalid call of @no_type_check function.
Pass @@ -2304,7 +2316,7 @@

Python Type System Conformance Test Results

Pass
    -
  • Does not honor `@no_type_check` class decorator (allowed).
    • +
  • Does not honor @no_type_check class decorator (allowed).
Pass @@ -2333,7 +2345,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not honor "# type: ignore" comment if comment includes additional text.
    • +
  • Does not honor "# type: ignore" comment if comment includes additional text.
Pass diff --git a/conformance/src/reporting.py b/conformance/src/reporting.py index 167abc4c7..56e397ace 100644 --- a/conformance/src/reporting.py +++ b/conformance/src/reporting.py @@ -9,6 +9,8 @@ from pathlib import Path import jinja2 +import markdown +import markupsafe from test_groups import get_test_cases, get_test_groups from type_checker import TYPE_CHECKERS, TypeChecker @@ -18,7 +20,7 @@ class TestResult: type_checker: str conformance: str - notes: list[str] = field(default_factory=list) + notes: list[markupsafe.Markup] = field(default_factory=list) @dataclass(frozen=True, kw_only=True, slots=True) @@ -116,10 +118,19 @@ def _get_groups( automated = data.get("conformance_automated") conformance = "Pass" if automated == "Pass" else "Unknown" + notes = [ + markupsafe.Markup( + markdown.markdown(note, output_format="html") + .removeprefix("

") + .removesuffix("

") + ) + for note in data.get("notes", "").strip().splitlines() + ] + result = TestResult( type_checker=type_checker.name, conformance=conformance, - notes=data.get("notes", "").strip().splitlines(), + notes=notes, ) case.results.append(result) diff --git a/conformance/src/templates/style.css b/conformance/src/templates/style.css index bd2835010..d8512af95 100644 --- a/conformance/src/templates/style.css +++ b/conformance/src/templates/style.css @@ -60,6 +60,13 @@ a { text-underline-offset: 2px; } +code { + background: light-dark(#eee, #666); + border: 1px solid light-dark(#ddd, #888); + border-radius: 3px; + padding: 0 2px; +} + table { border-collapse: collapse; inline-size: 100%; @@ -166,6 +173,7 @@ tbody > tr:hover { visibility: hidden; z-index: 2; & > li { + padding: 3px 0; text-indent: 1.3em hanging; } } diff --git a/conformance/uv.lock b/conformance/uv.lock index e66d5428d..2aadb047b 100644 --- a/conformance/uv.lock +++ b/conformance/uv.lock @@ -68,6 +68,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/59/67/a6739ac96e28b7855808bdb0370e250606104a859750d209e5a0716fe7ab/librt-0.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:2f10cf143e4a9bb0f4f5af568a00df94a2d69ef41c2579584454bb0fe5cc642c", size = 103470, upload-time = "2026-05-10T18:16:10.369Z" }, ] +[[package]] +name = "markdown" +version = "3.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/f4/69fa6ed85ae003c2378ffa8f6d2e3234662abd02c10d216c0ba96081a238/markdown-3.10.2.tar.gz", hash = "sha256:994d51325d25ad8aa7ce4ebaec003febcce822c3f8c911e3b17c52f7f589f950", size = 368805, upload-time = "2026-02-09T14:57:26.942Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl", hash = "sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36", size = 108180, upload-time = "2026-02-09T14:57:25.787Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -216,6 +225,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/4c/0c1ca628c5da7840e16801caa0bfeed1241e1113d8a5156a34245d4fa927/ty-0.0.50-py3-none-win_arm64.whl", hash = "sha256:96a84d970b59f2eddb92a4af3ba9906f24bda118cf487d923765ccd4ca24627b", size = 11635811, upload-time = "2026-06-17T21:37:07.191Z" }, ] +[[package]] +name = "types-markdown" +version = "3.10.2.20260508" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/62/4ca11db82799590117d9b6ef8fcd6515039cff80838304face5440f7d95c/types_markdown-3.10.2.20260508.tar.gz", hash = "sha256:64c405a30decd46cf78443bbcde8331f561f83752cc6d962ff3f5ccaabe26a8e", size = 19818, upload-time = "2026-05-08T04:49:45.095Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/9d/eae7125bcf063ff5f79aadad58f3d37675263941f84e025bf53de1735fe1/types_markdown-3.10.2.20260508-py3-none-any.whl", hash = "sha256:2ee5e462c7b821c27c3d8f75f4da873d3e3c87d24d7f2d741f8f4261e4024a99", size = 25811, upload-time = "2026-05-08T04:49:43.962Z" }, +] + [[package]] name = "typeshed-client" version = "2.11.0" @@ -235,24 +253,28 @@ version = "0.1.0" source = { virtual = "." } dependencies = [ { name = "jinja2" }, + { name = "markdown" }, { name = "mypy" }, { name = "pycroscope" }, { name = "pyrefly" }, { name = "pyright" }, { name = "tomlkit" }, { name = "ty" }, + { name = "types-markdown" }, { name = "zuban" }, ] [package.metadata] requires-dist = [ { name = "jinja2" }, + { name = "markdown" }, { name = "mypy" }, { name = "pycroscope" }, { name = "pyrefly" }, { name = "pyright" }, { name = "tomlkit" }, { name = "ty" }, + { name = "types-markdown" }, { name = "zuban" }, ] From d821c3c8688e6d4d369a3773c1d8f1ea42ce4861 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:32 +0100 Subject: [PATCH 22/30] Ensure existing notes use markdown syntax --- .../results/mypy/aliases_typealiastype.toml | 4 +- .../mypy/annotations_forward_refs.toml | 2 +- .../results/mypy/annotations_generators.toml | 2 +- .../results/mypy/callables_annotation.toml | 4 +- .../results/mypy/callables_kwargs.toml | 2 +- .../results/mypy/classes_classvar.toml | 8 +- .../results/mypy/constructors_call_init.toml | 4 +- .../mypy/constructors_call_metaclass.toml | 4 +- .../results/mypy/constructors_call_new.toml | 6 +- .../results/mypy/constructors_call_type.toml | 2 +- .../results/mypy/constructors_callable.toml | 6 +- .../mypy/constructors_consistency.toml | 2 +- .../results/mypy/dataclasses_final.toml | 4 +- .../results/mypy/dataclasses_hash.toml | 2 +- .../results/mypy/dataclasses_slots.toml | 2 +- .../results/mypy/directives_type_ignore.toml | 2 +- .../mypy/directives_version_platform.toml | 4 +- conformance/results/mypy/enums_expansion.toml | 2 +- .../results/mypy/generics_defaults.toml | 4 +- .../mypy/generics_defaults_referential.toml | 2 +- .../results/mypy/generics_type_erasure.toml | 4 +- .../mypy/generics_typevartuple_args.toml | 2 +- .../mypy/generics_typevartuple_basic.toml | 2 +- .../generics_typevartuple_specialization.toml | 4 +- .../results/mypy/generics_variance.toml | 2 +- .../mypy/literals_parameterizations.toml | 2 +- .../results/mypy/overloads_definitions.toml | 2 +- .../mypy/overloads_definitions_stub.toml | 2 +- .../results/mypy/overloads_evaluation.toml | 6 +- .../mypy/protocols_runtime_checkable.toml | 2 +- .../results/mypy/qualifiers_annotated.toml | 10 +- .../mypy/qualifiers_final_annotation.toml | 8 +- .../results/mypy/specialtypes_type.toml | 2 +- .../results/mypy/typeddicts_class_syntax.toml | 2 +- .../results/mypy/typeddicts_extra_items.toml | 3 - .../mypy/typeddicts_readonly_inheritance.toml | 6 +- .../mypy/typeddicts_readonly_update.toml | 4 +- .../results/mypy/typeforms_typeform.toml | 2 +- .../pycroscope/aliases_typealiastype.toml | 2 +- .../pycroscope/annotations_forward_refs.toml | 2 +- .../pycroscope/dataclasses_descriptors.toml | 2 +- .../results/pycroscope/generics_defaults.toml | 2 +- .../pycroscope/protocols_class_objects.toml | 2 +- .../pycroscope/qualifiers_annotated.toml | 2 +- .../pycroscope/specialtypes_never.toml | 2 +- .../typeddicts_type_consistency.toml | 2 +- .../results/pyrefly/callables_annotation.toml | 2 +- .../pyrefly/dataclasses_descriptors.toml | 4 +- .../pyrefly/exceptions_context_managers.toml | 2 +- .../results/pyrefly/generics_scoping.toml | 2 +- .../pyrefly/generics_self_advanced.toml | 2 +- .../generics_defaults_specialization.toml | 2 +- .../pyright/generics_paramspec_semantics.toml | 2 +- .../pyright/generics_typevartuple_args.toml | 2 +- .../pyright/generics_typevartuple_basic.toml | 2 +- .../results/pyright/overloads_basic.toml | 2 - .../results/pyright/overloads_evaluation.toml | 2 +- .../pyright/typeddicts_class_syntax.toml | 2 +- conformance/results/results.html | 187 +++++++++--------- .../results/ty/annotations_forward_refs.toml | 2 +- conformance/results/ty/generics_scoping.toml | 2 +- .../ty/generics_typevartuple_args.toml | 2 +- .../results/zuban/enums_member_values.toml | 2 - conformance/results/zuban/enums_members.toml | 2 - 64 files changed, 177 insertions(+), 195 deletions(-) diff --git a/conformance/results/mypy/aliases_typealiastype.toml b/conformance/results/mypy/aliases_typealiastype.toml index 1a034e810..b757d9edd 100644 --- a/conformance/results/mypy/aliases_typealiastype.toml +++ b/conformance/results/mypy/aliases_typealiastype.toml @@ -1,7 +1,7 @@ conformant = "Partial" notes = """ -Incorrectly rejects some recursive type aliases using TypeAliasType. -Incorrectly rejects the use of a class-scoped TypeVar in a TypeAliasType definition. +Incorrectly rejects some recursive type aliases using `TypeAliasType`. +Incorrectly rejects the use of a class-scoped `TypeVar` in a `TypeAliasType` definition. """ output = """ aliases_typealiastype.py:20: error: Cannot resolve name "GoodAlias5" (possible cyclic definition) [misc] diff --git a/conformance/results/mypy/annotations_forward_refs.toml b/conformance/results/mypy/annotations_forward_refs.toml index fd9c48687..15211054e 100644 --- a/conformance/results/mypy/annotations_forward_refs.toml +++ b/conformance/results/mypy/annotations_forward_refs.toml @@ -1,7 +1,7 @@ conformant = "Partial" notes = """ Does not report error for a forward reference that is not enclosed in quotes. -Does not report error for use of quoted type with "|" operator (runtime error). +Does not report error for use of quoted type with `|` operator (runtime error). Incorrectly generates error for quoted type defined in class scope. """ output = """ diff --git a/conformance/results/mypy/annotations_generators.toml b/conformance/results/mypy/annotations_generators.toml index e14ede973..3e86b6ced 100644 --- a/conformance/results/mypy/annotations_generators.toml +++ b/conformance/results/mypy/annotations_generators.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Does not report incompatible Generator type in `yield from` statement. +Does not report incompatible `Generator` type in `yield from` statement. """ output = """ annotations_generators.py:51: error: Missing return statement [return] diff --git a/conformance/results/mypy/callables_annotation.toml b/conformance/results/mypy/callables_annotation.toml index 11daf6205..f19579654 100644 --- a/conformance/results/mypy/callables_annotation.toml +++ b/conformance/results/mypy/callables_annotation.toml @@ -1,7 +1,7 @@ conformant = "Partial" notes = """ -Incorrectly treats "*args: T, **kwargs: T" as "..." when T is specialized to Any. -Does not treat "*args: Any, **kargs: Any" as "..." when separated by keyword parameter. +Incorrectly treats `*args: T, **kwargs: T` as `...` when `T` is specialized to `Any`. +Does not treat `*args: Any, **kargs: Any` as `...` when separated by keyword parameter. """ output = """ callables_annotation.py:25: error: Too few arguments [call-arg] diff --git a/conformance/results/mypy/callables_kwargs.toml b/conformance/results/mypy/callables_kwargs.toml index 571931b13..0586ed15b 100644 --- a/conformance/results/mypy/callables_kwargs.toml +++ b/conformance/results/mypy/callables_kwargs.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Allows callable without kwargs to be assigned to callable with unpacked kwargs +Allows callable without kwargs to be assigned to callable with unpacked kwargs. """ output = """ callables_kwargs.py:46: error: Missing named argument "v1" for "func1" [call-arg] diff --git a/conformance/results/mypy/classes_classvar.toml b/conformance/results/mypy/classes_classvar.toml index bddc8f0b6..853c49684 100644 --- a/conformance/results/mypy/classes_classvar.toml +++ b/conformance/results/mypy/classes_classvar.toml @@ -1,9 +1,9 @@ conformant = "Partial" notes = """ -Internal error if TypeVarTuple is used in ClassVar. -Does not reject use of ParamSpec in ClassVar. -Rejects ClassVar nested in Annotated. -Does not reject use of ClassVar in TypeAlias definition. +Internal error if `TypeVarTuple` is used in `ClassVar`. +Does not reject use of `ParamSpec` in `ClassVar`. +Rejects `ClassVar` nested in `Annotated`. +Does not reject use of `ClassVar` in `TypeAlias` definition. """ output = """ classes_classvar.py:38: error: ClassVar[...] must have at most one type argument [valid-type] diff --git a/conformance/results/mypy/constructors_call_init.toml b/conformance/results/mypy/constructors_call_init.toml index d8680ec9d..f2d7df715 100644 --- a/conformance/results/mypy/constructors_call_init.toml +++ b/conformance/results/mypy/constructors_call_init.toml @@ -1,7 +1,7 @@ conformant = "Partial" notes = """ -Does not report errors during binding to self parameter of __init__ method. -Does not reject use of class-scoped type variables in annotation of self parameter in __init__ method. +Does not report errors during binding to self parameter of `__init__` method. +Does not reject use of class-scoped type variables in annotation of self parameter in `__init__` method. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/mypy/constructors_call_metaclass.toml b/conformance/results/mypy/constructors_call_metaclass.toml index fb84e85db..a06679970 100644 --- a/conformance/results/mypy/constructors_call_metaclass.toml +++ b/conformance/results/mypy/constructors_call_metaclass.toml @@ -1,7 +1,7 @@ conformant = "Unsupported" notes = """ -Does not honor metaclass __call__ method when evaluating constructor call. -Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class. +Does not honor metaclass `__call__` method when evaluating constructor call. +Does not skip evaluation of `__new__` and `__init__` if custom metaclass call returns non-class. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/mypy/constructors_call_new.toml b/conformance/results/mypy/constructors_call_new.toml index a325d9e54..49860361e 100644 --- a/conformance/results/mypy/constructors_call_new.toml +++ b/conformance/results/mypy/constructors_call_new.toml @@ -1,8 +1,8 @@ conformant = "Partial" notes = """ -Does not support __new__ return type that is not a subclass of the class being constructed. -Does not skip evaluation of __init__ based on __new__ return type. -Does not report errors during binding to cls parameter of __new__ method. +Does not support `__new__` return type that is not a subclass of the class being constructed. +Does not skip evaluation of `__init__` based on `__new__` return type. +Does not report errors during binding to `cls` parameter of `__new__` method. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/mypy/constructors_call_type.toml b/conformance/results/mypy/constructors_call_type.toml index 7ba573760..c94564c53 100644 --- a/conformance/results/mypy/constructors_call_type.toml +++ b/conformance/results/mypy/constructors_call_type.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Does not validate call to custom metaclass __call__ method through type[T]. +Does not validate call to custom metaclass `__call__` method through `type[T]`. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/mypy/constructors_callable.toml b/conformance/results/mypy/constructors_callable.toml index af573bd2e..edf923406 100644 --- a/conformance/results/mypy/constructors_callable.toml +++ b/conformance/results/mypy/constructors_callable.toml @@ -1,8 +1,8 @@ conformant = "Partial" notes = """ -Does not generate a union type for __new__ and __init__ when converting class to callable. -Does not ignore __init__ based on __new__ return type when converting class to callable. -Does not support __new__ return type that is different from class being constructed. +Does not generate a union type for `__new__` and `__init__` when converting class to callable. +Does not ignore `__init__` based on `__new__` return type when converting class to callable. +Does not support `__new__` return type that is different from class being constructed. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/mypy/constructors_consistency.toml b/conformance/results/mypy/constructors_consistency.toml index 654ebaa96..5195baef1 100644 --- a/conformance/results/mypy/constructors_consistency.toml +++ b/conformance/results/mypy/constructors_consistency.toml @@ -1,6 +1,6 @@ conformant = "Pass" notes = """ -Does not report inconsistency between __new__ and __init__ (optional). +Does not report inconsistency between `__new__` and `__init__` (optional). """ conformance_automated = "Pass" errors_diff = """ diff --git a/conformance/results/mypy/dataclasses_final.toml b/conformance/results/mypy/dataclasses_final.toml index ddfb29575..ea61c82bc 100644 --- a/conformance/results/mypy/dataclasses_final.toml +++ b/conformance/results/mypy/dataclasses_final.toml @@ -1,7 +1,7 @@ conformant = "Partial" notes = """ -Wrongly requires a Final dataclass field to be initialized at class level. -Doesn't support Final nested inside ClassVar. +Wrongly requires a `Final` dataclass field to be initialized at class level. +Doesn't support `Final` nested inside `ClassVar`. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/mypy/dataclasses_hash.toml b/conformance/results/mypy/dataclasses_hash.toml index 08809153e..e4c6a663f 100644 --- a/conformance/results/mypy/dataclasses_hash.toml +++ b/conformance/results/mypy/dataclasses_hash.toml @@ -2,7 +2,7 @@ conformant = "Unsupported" notes = """ Does not synthesize `__hash__ = None` as a class attribute for unhashable dataclasses. Does not report when an unhashable dataclass has `__hash__` called directly on an instance. -Does not report when dataclass is not compatible with Hashable protocol. +Does not report when dataclass is not compatible with `Hashable` protocol. """ output = """ dataclasses_hash.py:14: error: Expression is of type "Callable[[object], int]", not "None" [assert-type] diff --git a/conformance/results/mypy/dataclasses_slots.toml b/conformance/results/mypy/dataclasses_slots.toml index 4a6d76337..7addfa5c1 100644 --- a/conformance/results/mypy/dataclasses_slots.toml +++ b/conformance/results/mypy/dataclasses_slots.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Does not reject write to instance variable that is not defined in __slots__. +Does not reject write to instance variable that is not defined in `__slots__`. """ output = """ dataclasses_slots.py:11: error: "DC1" both defines "__slots__" and is used with "slots=True" [misc] diff --git a/conformance/results/mypy/directives_type_ignore.toml b/conformance/results/mypy/directives_type_ignore.toml index 318b2b2a2..576b2b0d9 100644 --- a/conformance/results/mypy/directives_type_ignore.toml +++ b/conformance/results/mypy/directives_type_ignore.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Does not honor "# type: ignore" comment if comment includes additional text. +Does not honor `# type: ignore` comment if comment includes additional text. """ output = """ directives_type_ignore.py:11: error: Invalid "type: ignore" comment [syntax] diff --git a/conformance/results/mypy/directives_version_platform.toml b/conformance/results/mypy/directives_version_platform.toml index e1fcc5822..80c6eecff 100644 --- a/conformance/results/mypy/directives_version_platform.toml +++ b/conformance/results/mypy/directives_version_platform.toml @@ -1,7 +1,7 @@ conformant = "Pass" notes = """ -Does not understand three-element form of sys.version checks. -Does not understand os.name checks. +Does not understand three-element form of `sys.version` checks. +Does not understand `os.name` checks. """ output = """ directives_version_platform.py:26: error: Expression is of type "int | str", not "int" [assert-type] diff --git a/conformance/results/mypy/enums_expansion.toml b/conformance/results/mypy/enums_expansion.toml index 9858edfea..8b6a14aa0 100644 --- a/conformance/results/mypy/enums_expansion.toml +++ b/conformance/results/mypy/enums_expansion.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Improperly applies narrowing to Flag subclass. +Improperly applies narrowing to `Flag` subclass. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/mypy/generics_defaults.toml b/conformance/results/mypy/generics_defaults.toml index 5b18af90b..ce9174469 100644 --- a/conformance/results/mypy/generics_defaults.toml +++ b/conformance/results/mypy/generics_defaults.toml @@ -1,7 +1,7 @@ conformant = "Partial" notes = """ -Does not detect a TypeVar with a default used after a TypeVarTuple. -Does not fully support defaults on TypeVarTuple and ParamSpec. +Does not detect a `TypeVar` with a default used after a `TypeVarTuple`. +Does not fully support defaults on `TypeVarTuple` and `ParamSpec`. """ output = """ generics_defaults.py:24: error: "T" cannot appear after "DefaultStrT" in type parameter list because it has no default type [misc] diff --git a/conformance/results/mypy/generics_defaults_referential.toml b/conformance/results/mypy/generics_defaults_referential.toml index e8484dc7d..6bedb442a 100644 --- a/conformance/results/mypy/generics_defaults_referential.toml +++ b/conformance/results/mypy/generics_defaults_referential.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Does not correctly handle defaults referencing other TypeVars. +Does not correctly handle defaults referencing other `TypeVar`s. """ output = """ generics_defaults_referential.py:23: error: Expression is of type "type[slice[StartT, StopT, StepT]]", not "type[slice[int, int, int | None]]" [assert-type] diff --git a/conformance/results/mypy/generics_type_erasure.toml b/conformance/results/mypy/generics_type_erasure.toml index 46c739fee..dd13b4464 100644 --- a/conformance/results/mypy/generics_type_erasure.toml +++ b/conformance/results/mypy/generics_type_erasure.toml @@ -1,7 +1,7 @@ conformant = "Partial" notes = """ -Infers Node[Never] instead of Node[Any] when argument is not provided. -False negative on instance attribute access on type(node). +Infers `Node[Never]` instead of `Node[Any]` when argument is not provided. +False negative on instance attribute access on `type(node)`. """ output = """ generics_type_erasure.py:19: error: Expression is of type "Node[Never]", not "Node[Any]" [assert-type] diff --git a/conformance/results/mypy/generics_typevartuple_args.toml b/conformance/results/mypy/generics_typevartuple_args.toml index 43d9fa078..5920b7325 100644 --- a/conformance/results/mypy/generics_typevartuple_args.toml +++ b/conformance/results/mypy/generics_typevartuple_args.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Does not enforce that tuples captured by TypeVarTuple are of the same length. +Does not enforce that tuples captured by `TypeVarTuple` are of the same length. """ output = """ generics_typevartuple_args.py:33: error: Argument 3 to "exec_le" has incompatible type "str"; expected "Env" [arg-type] diff --git a/conformance/results/mypy/generics_typevartuple_basic.toml b/conformance/results/mypy/generics_typevartuple_basic.toml index 37140c379..c9a734127 100644 --- a/conformance/results/mypy/generics_typevartuple_basic.toml +++ b/conformance/results/mypy/generics_typevartuple_basic.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Does not enforce that tuples captured by TypeVarTuple are same length. +Does not enforce that tuples captured by `TypeVarTuple` are same length. """ output = """ generics_typevartuple_basic.py:42: error: Argument 1 to "Array" has incompatible type "Height"; expected "tuple[Height, Width]" [arg-type] diff --git a/conformance/results/mypy/generics_typevartuple_specialization.toml b/conformance/results/mypy/generics_typevartuple_specialization.toml index 7d23a81f1..280f9266d 100644 --- a/conformance/results/mypy/generics_typevartuple_specialization.toml +++ b/conformance/results/mypy/generics_typevartuple_specialization.toml @@ -1,7 +1,7 @@ conformant = "Partial" notes = """ -Incorrectly specializes generic alias that includes a TypeVar and TypeVarTuple if no type arguments are provided. -Rejects specialization of generic type alias defined as a tuple containing a TypeVar. +Incorrectly specializes generic alias that includes a `TypeVar` and `TypeVarTuple` if no type arguments are provided. +Rejects specialization of generic type alias defined as a tuple containing a `TypeVar`. """ output = """ generics_typevartuple_specialization.py:85: error: Argument 1 to "takes_float_array_with_specific_shape" has incompatible type "Array2[float, *tuple[Any, ...]]"; expected "Array2[float, Height, Width]" [arg-type] diff --git a/conformance/results/mypy/generics_variance.toml b/conformance/results/mypy/generics_variance.toml index 30810aacf..2dff34a72 100644 --- a/conformance/results/mypy/generics_variance.toml +++ b/conformance/results/mypy/generics_variance.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Does not reject use of class-scoped TypeVar used in a base class when variance is incompatible. +Does not reject use of class-scoped `TypeVar` used in a base class when variance is incompatible. """ output = """ generics_variance.py:14: error: TypeVar cannot be both covariant and contravariant [misc] diff --git a/conformance/results/mypy/literals_parameterizations.toml b/conformance/results/mypy/literals_parameterizations.toml index 730d531db..eac516519 100644 --- a/conformance/results/mypy/literals_parameterizations.toml +++ b/conformance/results/mypy/literals_parameterizations.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Does not reject tuple within Literal. +Does not reject tuple within `Literal`. """ output = """ literals_parameterizations.py:41: error: Invalid type: Literal[...] cannot contain arbitrary expressions [valid-type] diff --git a/conformance/results/mypy/overloads_definitions.toml b/conformance/results/mypy/overloads_definitions.toml index 80b29a62f..e653284f3 100644 --- a/conformance/results/mypy/overloads_definitions.toml +++ b/conformance/results/mypy/overloads_definitions.toml @@ -1,7 +1,7 @@ conformant = "Partial" conformance_automated = "Fail" notes = """ -Allows @override to be on all overloads and implementation, instead of just implementation. +Allows `@override` to be on all overloads and implementation, instead of just implementation. """ errors_diff = """ Lines 226, 227, 228, 231, 232: Expected error (tag 'override_impl') diff --git a/conformance/results/mypy/overloads_definitions_stub.toml b/conformance/results/mypy/overloads_definitions_stub.toml index 51a1aa393..1a4dc4fa7 100644 --- a/conformance/results/mypy/overloads_definitions_stub.toml +++ b/conformance/results/mypy/overloads_definitions_stub.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Allows @override to appear in a stub file not on the first overload. +Allows `@override` to appear in a stub file not on the first overload. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/mypy/overloads_evaluation.toml b/conformance/results/mypy/overloads_evaluation.toml index 4ddeaca2d..61c5af6a8 100644 --- a/conformance/results/mypy/overloads_evaluation.toml +++ b/conformance/results/mypy/overloads_evaluation.toml @@ -1,10 +1,10 @@ conformant = "Partial" notes = """ -Does not expand boolean arguments to Literal[True] and Literal[False]. +Does not expand boolean arguments to `Literal[True]` and `Literal[False]`. Does not expand enum arguments to literal variants. Does not expand tuple arguments to possible combinations. -Does not evaluate Any in some cases where overload is ambiguous. -Evaluates Any in some cases where overload is not ambiguous. +Does not evaluate `Any` in some cases where overload is ambiguous. +Evaluates `Any` in some cases where overload is not ambiguous. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/mypy/protocols_runtime_checkable.toml b/conformance/results/mypy/protocols_runtime_checkable.toml index 63124dc56..859ffd725 100644 --- a/conformance/results/mypy/protocols_runtime_checkable.toml +++ b/conformance/results/mypy/protocols_runtime_checkable.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Does not report unsafe overlap for runtime_checkable protocol. +Does not report unsafe overlap for `runtime_checkable` protocol. """ output = """ protocols_runtime_checkable.py:23: error: Only @runtime_checkable protocols can be used with instance and class checks [misc] diff --git a/conformance/results/mypy/qualifiers_annotated.toml b/conformance/results/mypy/qualifiers_annotated.toml index 6fba939f9..e3cfd8ab9 100644 --- a/conformance/results/mypy/qualifiers_annotated.toml +++ b/conformance/results/mypy/qualifiers_annotated.toml @@ -1,10 +1,10 @@ conformant = "Partial" notes = """ -Does not allow ClassVar to be nested within Annotated. -Does not allow Final to be nested within Annotated. -Does not allow Required and NotRequired to be nested within Annotated. -Does not reject type[T] compatibility for type alias defined with Annotated. -Does not reject call of type alias defined with Annotated. +Does not allow `ClassVar` to be nested within `Annotated`. +Does not allow `Final` to be nested within `Annotated`. +Does not allow `Required` and `NotRequired` to be nested within `Annotated`. +Does not reject `type[T]` compatibility for type alias defined with `Annotated`. +Does not reject call of type alias defined with `Annotated`. """ output = """ qualifiers_annotated.py:38: error: Bracketed expression "[...]" is not valid as a type [valid-type] diff --git a/conformance/results/mypy/qualifiers_final_annotation.toml b/conformance/results/mypy/qualifiers_final_annotation.toml index 2d0cea337..137e9f808 100644 --- a/conformance/results/mypy/qualifiers_final_annotation.toml +++ b/conformance/results/mypy/qualifiers_final_annotation.toml @@ -1,9 +1,9 @@ conformant = "Partial" notes = """ -Does not treat use of Final name as if it was replaced by the literal in NamedTuple definition. -Does not allow conditional assignment of Final instance variable in __init__ method. -Does not allow redefinition of private class variable that is marked Final in parent class. -Does not report modification of local Final variable via "for" statement. +Does not treat use of `Final` name as if it was replaced by the literal in `NamedTuple` definition. +Does not allow conditional assignment of `Final` instance variable in `__init__` method. +Does not allow redefinition of private class variable that is marked `Final` in parent class. +Does not report modification of local `Final` variable via `for` statement. """ output = """ qualifiers_final_annotation.py:16: error: Type in Final[...] can only be omitted if there is an initializer [misc] diff --git a/conformance/results/mypy/specialtypes_type.toml b/conformance/results/mypy/specialtypes_type.toml index ea8d59f2a..d83279230 100644 --- a/conformance/results/mypy/specialtypes_type.toml +++ b/conformance/results/mypy/specialtypes_type.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Does not treat `type` same as `type[Any]` for assert_type. +Does not treat `type` same as `type[Any]` for `assert_type`. Does not allow access to unknown attributes from object of type `type[Any]`. """ output = """ diff --git a/conformance/results/mypy/typeddicts_class_syntax.toml b/conformance/results/mypy/typeddicts_class_syntax.toml index 29220811d..a005f5d63 100644 --- a/conformance/results/mypy/typeddicts_class_syntax.toml +++ b/conformance/results/mypy/typeddicts_class_syntax.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Does not support version-conditional items in TypedDict definitions. +Does not support version-conditional items in `TypedDict` definitions. """ output = """ typeddicts_class_syntax.py:30: error: Invalid statement in TypedDict definition; expected "field_name: field_type" [misc] diff --git a/conformance/results/mypy/typeddicts_extra_items.toml b/conformance/results/mypy/typeddicts_extra_items.toml index e7c58f8bc..0938d870e 100644 --- a/conformance/results/mypy/typeddicts_extra_items.toml +++ b/conformance/results/mypy/typeddicts_extra_items.toml @@ -1,7 +1,4 @@ conformant = "Unsupported" -notes = """ -Not supported. -""" conformance_automated = "Fail" errors_diff = """ Line 215: Expected 1 errors diff --git a/conformance/results/mypy/typeddicts_readonly_inheritance.toml b/conformance/results/mypy/typeddicts_readonly_inheritance.toml index 46bc4ce00..4bd44ff08 100644 --- a/conformance/results/mypy/typeddicts_readonly_inheritance.toml +++ b/conformance/results/mypy/typeddicts_readonly_inheritance.toml @@ -1,8 +1,8 @@ conformant = "Partial" notes = """ -Incorrectly rejects non-ReadOnly override of ReadOnly item. -Incorrectly rejects override of ReadOnly item with another ReadOnly item with narrower type. -Incorrectly rejects override of NotRequired ReadOnly item with a Required ReadOnly item. +Incorrectly rejects non-`ReadOnly` override of `ReadOnly` item. +Incorrectly rejects override of `ReadOnly` item with another `ReadOnly` item with narrower type. +Incorrectly rejects override of `NotRequired ReadOnly` item with a `Required ReadOnly` item. """ output = """ typeddicts_readonly_inheritance.py:19: error: Overwriting TypedDict field "name" while extending [misc] diff --git a/conformance/results/mypy/typeddicts_readonly_update.toml b/conformance/results/mypy/typeddicts_readonly_update.toml index c53706fc4..aa147b3ee 100644 --- a/conformance/results/mypy/typeddicts_readonly_update.toml +++ b/conformance/results/mypy/typeddicts_readonly_update.toml @@ -1,7 +1,7 @@ conformant = "Partial" notes = """ -Incorrectly allows update of ReadOnly item. -Incorrectly rejects update involving an item with Never type. +Incorrectly allows update of `ReadOnly` item. +Incorrectly rejects update involving an item with `Never` type. """ output = """ typeddicts_readonly_update.py:34: error: Argument 1 to "update" of "TypedDict" has incompatible type "B"; expected "TypedDict({'x'?=: int, 'y': int})" [typeddict-item] diff --git a/conformance/results/mypy/typeforms_typeform.toml b/conformance/results/mypy/typeforms_typeform.toml index 483dc9723..14f1ba107 100644 --- a/conformance/results/mypy/typeforms_typeform.toml +++ b/conformance/results/mypy/typeforms_typeform.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Does not support assigning Union and GenericAlias objects to their runtime types. +Does not support assigning `Union` and `GenericAlias` objects to their runtime types. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/pycroscope/aliases_typealiastype.toml b/conformance/results/pycroscope/aliases_typealiastype.toml index 2708a2ee4..09556fcfd 100644 --- a/conformance/results/pycroscope/aliases_typealiastype.toml +++ b/conformance/results/pycroscope/aliases_typealiastype.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Rejects valid ParamSpec specialization. +Rejects valid `ParamSpec` specialization. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/pycroscope/annotations_forward_refs.toml b/conformance/results/pycroscope/annotations_forward_refs.toml index 11cedda9e..63fbfc009 100644 --- a/conformance/results/pycroscope/annotations_forward_refs.toml +++ b/conformance/results/pycroscope/annotations_forward_refs.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Fails to reject "x" | int annotations that fail at runtime. +Fails to reject `"x" | int` annotations that fail at runtime. Rejects some valid quoted annotations. """ conformance_automated = "Fail" diff --git a/conformance/results/pycroscope/dataclasses_descriptors.toml b/conformance/results/pycroscope/dataclasses_descriptors.toml index e5410991c..5b89c7ef9 100644 --- a/conformance/results/pycroscope/dataclasses_descriptors.toml +++ b/conformance/results/pycroscope/dataclasses_descriptors.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Conformance suite is questionable; see https://github.com/python/typing/issues/2259 +Conformance suite is questionable; see """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/pycroscope/generics_defaults.toml b/conformance/results/pycroscope/generics_defaults.toml index 93d9e5fb4..3e3cc4f3a 100644 --- a/conformance/results/pycroscope/generics_defaults.toml +++ b/conformance/results/pycroscope/generics_defaults.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Numerous issues; does not support TypeVarTuple and ParamSpec defaults. +Numerous issues; does not support `TypeVarTuple` and `ParamSpec` defaults. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/pycroscope/protocols_class_objects.toml b/conformance/results/pycroscope/protocols_class_objects.toml index e40edf15f..0df48dfc7 100644 --- a/conformance/results/pycroscope/protocols_class_objects.toml +++ b/conformance/results/pycroscope/protocols_class_objects.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Abstract type[Proto] still allows protocol class objects in some paths. +Abstract `type[Proto]` still allows protocol class objects in some paths. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/pycroscope/qualifiers_annotated.toml b/conformance/results/pycroscope/qualifiers_annotated.toml index 97000ee40..13fd7743f 100644 --- a/conformance/results/pycroscope/qualifiers_annotated.toml +++ b/conformance/results/pycroscope/qualifiers_annotated.toml @@ -1,7 +1,7 @@ conformant = "Partial" notes = """ Fails to reject various weird annotations. -False positive on lambda in Annotated. +False positive on lambda in `Annotated`. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/pycroscope/specialtypes_never.toml b/conformance/results/pycroscope/specialtypes_never.toml index e6f61291a..47dfa3ac0 100644 --- a/conformance/results/pycroscope/specialtypes_never.toml +++ b/conformance/results/pycroscope/specialtypes_never.toml @@ -1,7 +1,7 @@ conformant = "Partial" notes = """ Does not enforce invariance in some contexts -Does not allow Any to be assigned to Never +Does not allow `Any` to be assigned to `Never` """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/pycroscope/typeddicts_type_consistency.toml b/conformance/results/pycroscope/typeddicts_type_consistency.toml index 62d6661c8..cf2aaa072 100644 --- a/conformance/results/pycroscope/typeddicts_type_consistency.toml +++ b/conformance/results/pycroscope/typeddicts_type_consistency.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Considers TypedDicts to be assignable to plain dict types. +Considers `TypedDict`s to be assignable to plain `dict` types. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/pyrefly/callables_annotation.toml b/conformance/results/pyrefly/callables_annotation.toml index efbdf98e8..01f1ec75e 100644 --- a/conformance/results/pyrefly/callables_annotation.toml +++ b/conformance/results/pyrefly/callables_annotation.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Parameter names are lost when resolving ParamSpec +Parameter names are lost when resolving `ParamSpec`. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/pyrefly/dataclasses_descriptors.toml b/conformance/results/pyrefly/dataclasses_descriptors.toml index 03625f94d..5f36bae96 100644 --- a/conformance/results/pyrefly/dataclasses_descriptors.toml +++ b/conformance/results/pyrefly/dataclasses_descriptors.toml @@ -1,7 +1,7 @@ conformant = "Partial" notes = """ -* Assumes descriptor behavior only when field is assigned in class body -* Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types +Assumes descriptor behavior only when field is assigned in class body +Doesn't allow non-data descriptors or data descriptors with differing `__get__` and `__set__` types """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/pyrefly/exceptions_context_managers.toml b/conformance/results/pyrefly/exceptions_context_managers.toml index d9b225e74..fe2a3282d 100644 --- a/conformance/results/pyrefly/exceptions_context_managers.toml +++ b/conformance/results/pyrefly/exceptions_context_managers.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Some error suppressing context managers are not detected +Some error suppressing context managers are not detected. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/pyrefly/generics_scoping.toml b/conformance/results/pyrefly/generics_scoping.toml index 236c49f06..0a4b9c40e 100644 --- a/conformance/results/pyrefly/generics_scoping.toml +++ b/conformance/results/pyrefly/generics_scoping.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Does not implement several scoping checks/restrictions for generics +Does not implement several scoping checks/restrictions for generics. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/pyrefly/generics_self_advanced.toml b/conformance/results/pyrefly/generics_self_advanced.toml index fb11f7c63..797430280 100644 --- a/conformance/results/pyrefly/generics_self_advanced.toml +++ b/conformance/results/pyrefly/generics_self_advanced.toml @@ -1,6 +1,6 @@ conformant = "Pass" notes = """ -Treats attributes not initialized on the class as instance-only +Treats attributes not initialized on the class as instance-only. """ conformance_automated = "Pass" errors_diff = """ diff --git a/conformance/results/pyright/generics_defaults_specialization.toml b/conformance/results/pyright/generics_defaults_specialization.toml index 28c6cdced..f1d5bc216 100644 --- a/conformance/results/pyright/generics_defaults_specialization.toml +++ b/conformance/results/pyright/generics_defaults_specialization.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Allows incorrect assignment to type[]. +Allows incorrect assignment to `type[]`. """ output = """ generics_defaults_specialization.py:30:15 - error: Too many type arguments provided for "MyAlias[DefaultStrT@MyAlias]"; expected 1 but received 2 (reportInvalidTypeForm) diff --git a/conformance/results/pyright/generics_paramspec_semantics.toml b/conformance/results/pyright/generics_paramspec_semantics.toml index e24a0a9c8..ba9b447a9 100644 --- a/conformance/results/pyright/generics_paramspec_semantics.toml +++ b/conformance/results/pyright/generics_paramspec_semantics.toml @@ -1,6 +1,6 @@ conformant = "Pass" notes = """ -Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed). +Constraint solver doesn't find common type for two signatures captured by a single `ParamSpec` (allowed). """ output = """ generics_paramspec_semantics.py:26:6 - error: Expected 2 more positional arguments (reportCallIssue) diff --git a/conformance/results/pyright/generics_typevartuple_args.toml b/conformance/results/pyright/generics_typevartuple_args.toml index 65c6a1992..a2756d559 100644 --- a/conformance/results/pyright/generics_typevartuple_args.toml +++ b/conformance/results/pyright/generics_typevartuple_args.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Does not correctly solve TypeVarTuple with heterogeneous bounds. +Does not correctly solve `TypeVarTuple` with heterogeneous bounds. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/pyright/generics_typevartuple_basic.toml b/conformance/results/pyright/generics_typevartuple_basic.toml index 7e5cfa403..4021237e7 100644 --- a/conformance/results/pyright/generics_typevartuple_basic.toml +++ b/conformance/results/pyright/generics_typevartuple_basic.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Does not correctly solve TypeVarTuple with heterogeneous bounds. +Does not correctly solve `TypeVarTuple` with heterogeneous bounds. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/pyright/overloads_basic.toml b/conformance/results/pyright/overloads_basic.toml index 14bfb58bf..78bfb673a 100644 --- a/conformance/results/pyright/overloads_basic.toml +++ b/conformance/results/pyright/overloads_basic.toml @@ -1,6 +1,4 @@ conformant = "Pass" -notes = """ -""" output = """ overloads_basic.py:39:1 - error: No overloads for "__getitem__" match the provided arguments (reportCallIssue) overloads_basic.py:39:1 - error: Argument of type "Literal['']" cannot be assigned to parameter "__s" of type "slice[Any, Any, Any]" in function "__getitem__" diff --git a/conformance/results/pyright/overloads_evaluation.toml b/conformance/results/pyright/overloads_evaluation.toml index 7b41dc4e1..cb63679bf 100644 --- a/conformance/results/pyright/overloads_evaluation.toml +++ b/conformance/results/pyright/overloads_evaluation.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Does not evaluate Any in some cases where overload is ambiguous. +Does not evaluate `Any` in some cases where overload is ambiguous. """ conformance_automated = "Fail" errors_diff = """ diff --git a/conformance/results/pyright/typeddicts_class_syntax.toml b/conformance/results/pyright/typeddicts_class_syntax.toml index e77eac4c6..b1c81614c 100644 --- a/conformance/results/pyright/typeddicts_class_syntax.toml +++ b/conformance/results/pyright/typeddicts_class_syntax.toml @@ -1,6 +1,6 @@ conformant = "Partial" notes = """ -Does not support version-conditional items in TypedDict definitions. +Does not support version-conditional items in `TypedDict` definitions. """ output = """ typeddicts_class_syntax.py:30:5 - error: TypedDict classes can contain only type annotations (reportGeneralTypeIssues) diff --git a/conformance/results/results.html b/conformance/results/results.html index 965af67bc..45868508b 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -281,14 +281,14 @@

Python Type System Conformance Test Results

Partial
  • Does not report error for a forward reference that is not enclosed in quotes.
    • -
  • Does not report error for use of quoted type with "|" operator (runtime error).
    • +
  • Does not report error for use of quoted type with | operator (runtime error).
  • Incorrectly generates error for quoted type defined in class scope.
Partial
    -
  • Fails to reject "x" | int annotations that fail at runtime.
    • +
  • Fails to reject "x" | int annotations that fail at runtime.
  • Rejects some valid quoted annotations.
@@ -303,7 +303,7 @@

Python Type System Conformance Test Results

Partial @@ -318,7 +318,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not report incompatible Generator type in yield from statement.
    • +
  • Does not report incompatible Generator type in yield from statement.
Pass @@ -370,7 +370,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not support assigning Union and GenericAlias objects to their runtime types.
    • +
  • Does not support assigning Union and GenericAlias objects to their runtime types.
@@ -405,7 +405,7 @@

Python Type System Conformance Test Results

Partial
  • Does not enforce invariance in some contexts
    • -
  • Does not allow Any to be assigned to Never
    • +
  • Does not allow Any to be assigned to Never
Pass @@ -436,7 +436,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not treat type same as type[Any] for assert_type.
    • +
  • Does not treat type same as type[Any] for assert_type.
  • Does not allow access to unknown attributes from object of type type[Any].
@@ -490,14 +490,14 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not detect a TypeVar with a default used after a TypeVarTuple.
    • -
  • Does not fully support defaults on TypeVarTuple and ParamSpec.
    • +
  • Does not detect a TypeVar with a default used after a TypeVarTuple.
    • +
  • Does not fully support defaults on TypeVarTuple and ParamSpec.
Partial
    -
  • Numerous issues; does not support TypeVarTuple and ParamSpec defaults.
    • +
  • Numerous issues; does not support TypeVarTuple and ParamSpec defaults.
Pass @@ -516,7 +516,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not correctly handle defaults referencing other TypeVars.
    • +
  • Does not correctly handle defaults referencing other TypeVars.
@@ -543,7 +543,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Allows incorrect assignment to type[].
    • +
  • Allows incorrect assignment to type[].
@@ -585,7 +585,7 @@

Python Type System Conformance Test Results

Pass
    -
  • Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).
    • +
  • Constraint solver doesn't find common type for two signatures captured by a single ParamSpec (allowed).
Pass @@ -607,7 +607,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not implement several scoping checks/restrictions for generics
    • +
  • Does not implement several scoping checks/restrictions for generics.
Pass @@ -615,7 +615,7 @@

Python Type System Conformance Test Results

Partial
  • Does not reject list[T]() in the global scope, where T is an unbound type variable.
    • -
  • Does nto reject alias: TypeAlias = list[T] in the body scope of a class generic over a type variable T.
    • +
  • Does not reject alias: TypeAlias = list[T] in the body scope of a class generic over a type variable T.
Pass @@ -635,7 +635,7 @@

Python Type System Conformance Test Results

Pass
    -
  • Treats attributes not initialized on the class as instance-only
    • +
  • Treats attributes not initialized on the class as instance-only.
Pass @@ -739,8 +739,8 @@

Python Type System Conformance Test Results

Partial
    -
  • Infers Node[Never] instead of Node[Any] when argument is not provided.
    • -
  • False negative on instance attribute access on type(node).
    • +
  • Infers Node[Never] instead of Node[Any] when argument is not provided.
    • +
  • False negative on instance attribute access on type(node).
Pass @@ -759,7 +759,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not enforce that tuples captured by TypeVarTuple are of the same length.
    • +
  • Does not enforce that tuples captured by TypeVarTuple are of the same length.
Pass @@ -767,13 +767,13 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not correctly solve TypeVarTuple with heterogeneous bounds.
    • +
  • Does not correctly solve TypeVarTuple with heterogeneous bounds.
Partial
    -
  • Supports PEP-646 unpacked tuples but not TypeVarTuple.
    • +
  • Supports PEP-646 unpacked tuples but not TypeVarTuple.
Pass @@ -783,7 +783,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not enforce that tuples captured by TypeVarTuple are same length.
    • +
  • Does not enforce that tuples captured by TypeVarTuple are same length.
Pass @@ -791,7 +791,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not correctly solve TypeVarTuple with heterogeneous bounds.
    • +
  • Does not correctly solve TypeVarTuple with heterogeneous bounds.
Unsupported @@ -829,8 +829,8 @@

Python Type System Conformance Test Results

Partial
    -
  • Incorrectly specializes generic alias that includes a TypeVar and TypeVarTuple if no type arguments are provided.
    • -
  • Rejects specialization of generic type alias defined as a tuple containing a TypeVar.
    • +
  • Incorrectly specializes generic alias that includes a TypeVar and TypeVarTuple if no type arguments are provided.
    • +
  • Rejects specialization of generic type alias defined as a tuple containing a TypeVar.
Pass @@ -867,7 +867,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not reject use of class-scoped TypeVar used in a base class when variance is incompatible.
    • +
  • Does not reject use of class-scoped TypeVar used in a base class when variance is incompatible.
Pass @@ -895,18 +895,18 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not allow ClassVar to be nested within Annotated.
    • -
  • Does not allow Final to be nested within Annotated.
    • -
  • Does not allow Required and NotRequired to be nested within Annotated.
    • -
  • Does not reject type[T] compatibility for type alias defined with Annotated.
    • -
  • Does not reject call of type alias defined with Annotated.
    • +
  • Does not allow ClassVar to be nested within Annotated.
    • +
  • Does not allow Final to be nested within Annotated.
    • +
  • Does not allow Required and NotRequired to be nested within Annotated.
    • +
  • Does not reject type[T] compatibility for type alias defined with Annotated.
    • +
  • Does not reject call of type alias defined with Annotated.
Partial
  • Fails to reject various weird annotations.
    • -
  • False positive on lambda in Annotated.
    • +
  • False positive on lambda in Annotated.
Pass @@ -919,10 +919,10 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not treat use of Final name as if it was replaced by the literal in NamedTuple definition.
    • -
  • Does not allow conditional assignment of Final instance variable in init method.
    • -
  • Does not allow redefinition of private class variable that is marked Final in parent class.
    • -
  • Does not report modification of local Final variable via "for" statement.
    • +
  • Does not treat use of Final name as if it was replaced by the literal in NamedTuple definition.
    • +
  • Does not allow conditional assignment of Final instance variable in __init__ method.
    • +
  • Does not allow redefinition of private class variable that is marked Final in parent class.
    • +
  • Does not report modification of local Final variable via for statement.
Pass @@ -950,10 +950,10 @@

Python Type System Conformance Test Results

Partial
    -
  • Internal error if TypeVarTuple is used in ClassVar.
    • -
  • Does not reject use of ParamSpec in ClassVar.
    • -
  • Rejects ClassVar nested in Annotated.
    • -
  • Does not reject use of ClassVar in TypeAlias definition.
    • +
  • Internal error if TypeVarTuple is used in ClassVar.
    • +
  • Does not reject use of ParamSpec in ClassVar.
    • +
  • Rejects ClassVar nested in Annotated.
    • +
  • Does not reject use of ClassVar in TypeAlias definition.
Pass @@ -1064,14 +1064,14 @@

Python Type System Conformance Test Results

Partial
    -
  • Incorrectly rejects some recursive type aliases using TypeAliasType.
    • -
  • Incorrectly rejects the use of a class-scoped TypeVar in a TypeAliasType definition.
    • +
  • Incorrectly rejects some recursive type aliases using TypeAliasType.
    • +
  • Incorrectly rejects the use of a class-scoped TypeVar in a TypeAliasType definition.
Partial
    -
  • Rejects valid ParamSpec specialization.
    • +
  • Rejects valid ParamSpec specialization.
Pass @@ -1139,7 +1139,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not reject tuple within Literal.
    • +
  • Does not reject tuple within Literal.
@@ -1173,7 +1173,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Abstract type[Proto] still allows protocol class objects in some paths.
    • +
  • Abstract type[Proto] still allows protocol class objects in some paths.
Pass @@ -1299,7 +1299,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not report unsafe overlap for runtime_checkable protocol.
    • +
  • Does not report unsafe overlap for runtime_checkable protocol.
Pass @@ -1350,15 +1350,15 @@

Python Type System Conformance Test Results

Partial
    -
  • Incorrectly treats "args: T, *kwargs: T" as "..." when T is specialized to Any.
    • -
  • Does not treat "args: Any, *kargs: Any" as "..." when separated by keyword parameter.
    • +
  • Incorrectly treats *args: T, **kwargs: T as ... when T is specialized to Any.
    • +
  • Does not treat *args: Any, **kargs: Any as ... when separated by keyword parameter.
Pass Partial
    -
  • Parameter names are lost when resolving ParamSpec
    • +
  • Parameter names are lost when resolving ParamSpec.
Pass @@ -1375,7 +1375,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Allows callable without kwargs to be assigned to callable with unpacked kwargs
    • +
  • Allows callable without kwargs to be assigned to callable with unpacked kwargs.
Pass @@ -1412,8 +1412,8 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not report errors during binding to self parameter of init method.
    • -
  • Does not reject use of class-scoped type variables in annotation of self parameter in init method.
    • +
  • Does not report errors during binding to self parameter of __init__ method.
    • +
  • Does not reject use of class-scoped type variables in annotation of self parameter in __init__ method.
Pass @@ -1433,8 +1433,8 @@

Python Type System Conformance Test Results

Unsupported
    -
  • Does not honor metaclass call method when evaluating constructor call.
    • -
  • Does not skip evaluation of new and init if custom metaclass call returns non-class.
    • +
  • Does not honor metaclass __call__ method when evaluating constructor call.
    • +
  • Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.
Pass @@ -1448,9 +1448,9 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not support new return type that is not a subclass of the class being constructed.
    • -
  • Does not skip evaluation of init based on new return type.
    • -
  • Does not report errors during binding to cls parameter of new method.
    • +
  • Does not support __new__ return type that is not a subclass of the class being constructed.
    • +
  • Does not skip evaluation of __init__ based on __new__ return type.
    • +
  • Does not report errors during binding to cls parameter of __new__ method.
Pass @@ -1464,7 +1464,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not validate call to custom metaclass call method through type[T].
    • +
  • Does not validate call to custom metaclass __call__ method through type[T].
Pass @@ -1483,9 +1483,9 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not generate a union type for new and init when converting class to callable.
    • -
  • Does not ignore init based on new return type when converting class to callable.
    • -
  • Does not support new return type that is different from class being constructed.
    • +
  • Does not generate a union type for __new__ and __init__ when converting class to callable.
    • +
  • Does not ignore __init__ based on __new__ return type when converting class to callable.
    • +
  • Does not support __new__ return type that is different from class being constructed.
Pass @@ -1511,7 +1511,7 @@

Python Type System Conformance Test Results

Pass
    -
  • Does not report inconsistency between new and init (optional).
    • +
  • Does not report inconsistency between __new__ and __init__ (optional).
Pass @@ -1553,7 +1553,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Allows @override to be on all overloads and implementation, instead of just implementation.
    • +
  • Allows @override to be on all overloads and implementation, instead of just implementation.
Pass @@ -1567,7 +1567,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Allows @override to appear in a stub file not on the first overload.
    • +
  • Allows @override to appear in a stub file not on the first overload.
@@ -1586,11 +1586,11 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not expand boolean arguments to Literal[True] and Literal[False].
    • +
  • Does not expand boolean arguments to Literal[True] and Literal[False].
  • Does not expand enum arguments to literal variants.
  • Does not expand tuple arguments to possible combinations.
    • -
  • Does not evaluate Any in some cases where overload is ambiguous.
    • -
  • Evaluates Any in some cases where overload is not ambiguous.
    • +
  • Does not evaluate Any in some cases where overload is ambiguous.
    • +
  • Evaluates Any in some cases where overload is not ambiguous.
Pass @@ -1598,7 +1598,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not evaluate Any in some cases where overload is ambiguous.
    • +
  • Does not evaluate Any in some cases where overload is ambiguous.
Pass @@ -1616,7 +1616,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Some error suppressing context managers are not detected
    • +
  • Some error suppressing context managers are not detected.
Pass @@ -1640,18 +1640,14 @@

Python Type System Conformance Test Results

Partial Partial
    -
    • -
    • Assumes descriptor behavior only when field is assigned in class body
      • -
    • -
    • -
    • Doesn't allow non-data descriptors or data descriptors with differing __get__ and __set__ types
      • -
    • +
  • Assumes descriptor behavior only when field is assigned in class body
    • +
  • Doesn't allow non-data descriptors or data descriptors with differing __get__ and __set__ types
Pass @@ -1668,8 +1664,8 @@

Python Type System Conformance Test Results

Partial
    -
  • Wrongly requires a Final dataclass field to be initialized at class level.
    • -
  • Doesn't support Final nested inside ClassVar.
    • +
  • Wrongly requires a Final dataclass field to be initialized at class level.
    • +
  • Doesn't support Final nested inside ClassVar.
Pass @@ -1694,7 +1690,7 @@

Python Type System Conformance Test Results

  • Does not synthesize __hash__ = None as a class attribute for unhashable dataclasses.
  • Does not report when an unhashable dataclass has __hash__ called directly on an instance.
    • -
  • Does not report when dataclass is not compatible with Hashable protocol.
    • +
  • Does not report when dataclass is not compatible with Hashable protocol.
Pass @@ -1758,7 +1754,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not reject write to instance variable that is not defined in slots.
    • +
  • Does not reject write to instance variable that is not defined in __slots__.
Pass @@ -1871,7 +1867,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not support version-conditional items in TypedDict definitions.
    • +
  • Does not support version-conditional items in TypedDict definitions.
Pass @@ -1879,7 +1875,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not support version-conditional items in TypedDict definitions.
    • +
  • Does not support version-conditional items in TypedDict definitions.
Pass @@ -1887,12 +1883,7 @@

Python Type System Conformance Test Results

typeddicts_extra_items - - Unsupported -
    -
  • Not supported.
    • -
- + Unsupported Pass Pass Pass @@ -1949,9 +1940,9 @@

Python Type System Conformance Test Results

Partial
    -
  • Incorrectly rejects non-ReadOnly override of ReadOnly item.
    • -
  • Incorrectly rejects override of ReadOnly item with another ReadOnly item with narrower type.
    • -
  • Incorrectly rejects override of NotRequired ReadOnly item with a Required ReadOnly item.
    • +
  • Incorrectly rejects non-ReadOnly override of ReadOnly item.
    • +
  • Incorrectly rejects override of ReadOnly item with another ReadOnly item with narrower type.
    • +
  • Incorrectly rejects override of NotRequired ReadOnly item with a Required ReadOnly item.
Pass @@ -1974,8 +1965,8 @@

Python Type System Conformance Test Results

Partial
    -
  • Incorrectly allows update of ReadOnly item.
    • -
  • Incorrectly rejects update involving an item with Never type.
    • +
  • Incorrectly allows update of ReadOnly item.
    • +
  • Incorrectly rejects update involving an item with Never type.
Pass @@ -1999,7 +1990,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Considers TypedDicts to be assignable to plain dict types.
    • +
  • Considers TypedDicts to be assignable to plain dict types.
Pass @@ -2149,7 +2140,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Improperly applies narrowing to Flag subclass.
    • +
  • Improperly applies narrowing to Flag subclass.
Pass @@ -2345,7 +2336,7 @@

Python Type System Conformance Test Results

Partial
    -
  • Does not honor "# type: ignore" comment if comment includes additional text.
    • +
  • Does not honor # type: ignore comment if comment includes additional text.
Pass @@ -2377,8 +2368,8 @@

Python Type System Conformance Test Results

Pass
    -
  • Does not understand three-element form of sys.version checks.
    • -
  • Does not understand os.name checks.
    • +
  • Does not understand three-element form of sys.version checks.
    • +
  • Does not understand os.name checks.
Pass diff --git a/conformance/results/ty/annotations_forward_refs.toml b/conformance/results/ty/annotations_forward_refs.toml index 5944683d0..d176548df 100644 --- a/conformance/results/ty/annotations_forward_refs.toml +++ b/conformance/results/ty/annotations_forward_refs.toml @@ -1,7 +1,7 @@ conformance_automated = "Fail" conformant = "Partial" notes = """ -Resolves references in type annotations as referring to end-of-scope types (https://discuss.python.org/t/annotation-string-references-in-class-scope-in-conformance-tests/105439, https://github.com/python/typing/pull/2144) +Resolves references in type annotations as referring to end-of-scope types (, ) """ errors_diff = """ Line 87: Unexpected errors ['annotations_forward_refs.py:87:9: error[invalid-type-form] Function `int` is not valid in a type expression'] diff --git a/conformance/results/ty/generics_scoping.toml b/conformance/results/ty/generics_scoping.toml index b37df7322..330fcd0a8 100644 --- a/conformance/results/ty/generics_scoping.toml +++ b/conformance/results/ty/generics_scoping.toml @@ -2,7 +2,7 @@ conformance_automated = "Fail" conformant = "Partial" notes = """ Does not reject `list[T]()` in the global scope, where `T` is an unbound type variable. -Does nto reject `alias: TypeAlias = list[T]` in the body scope of a class generic over a type variable `T`. +Does not reject `alias: TypeAlias = list[T]` in the body scope of a class generic over a type variable `T`. """ errors_diff = """ Line 98: Expected 1 errors diff --git a/conformance/results/ty/generics_typevartuple_args.toml b/conformance/results/ty/generics_typevartuple_args.toml index 7de7af540..826f3333d 100644 --- a/conformance/results/ty/generics_typevartuple_args.toml +++ b/conformance/results/ty/generics_typevartuple_args.toml @@ -1,7 +1,7 @@ conformance_automated = "Fail" conformant = "Partial" notes = """ -Supports PEP-646 unpacked tuples but not TypeVarTuple. +Supports PEP-646 unpacked tuples but not `TypeVarTuple`. """ errors_diff = """ Line 33: Expected 1 errors diff --git a/conformance/results/zuban/enums_member_values.toml b/conformance/results/zuban/enums_member_values.toml index 9705e064b..0e71e68fa 100644 --- a/conformance/results/zuban/enums_member_values.toml +++ b/conformance/results/zuban/enums_member_values.toml @@ -1,5 +1,3 @@ -notes = """ -""" conformance_automated = "Pass" errors_diff = """ """ diff --git a/conformance/results/zuban/enums_members.toml b/conformance/results/zuban/enums_members.toml index 5176996e0..532bcb874 100644 --- a/conformance/results/zuban/enums_members.toml +++ b/conformance/results/zuban/enums_members.toml @@ -1,5 +1,3 @@ -notes = """ -""" conformance_automated = "Pass" errors_diff = """ """ From b188375673dc0647c03e6b7edc35c4a5bcae4417 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:32 +0100 Subject: [PATCH 23/30] Add a favicon using an emoji --- conformance/results/results.html | 1 + conformance/src/templates/base.html | 1 + 2 files changed, 2 insertions(+) diff --git a/conformance/results/results.html b/conformance/results/results.html index 45868508b..3876fc0c8 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -225,6 +225,7 @@ } } +
diff --git a/conformance/src/templates/base.html b/conformance/src/templates/base.html index 0cb9a47c6..2457df6bf 100644 --- a/conformance/src/templates/base.html +++ b/conformance/src/templates/base.html @@ -11,6 +11,7 @@ {% include "style.css" %} {%- endfilter %} +
From ecbec59daa76bfb3327865eec37265e1b8b729e1 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Wed, 13 May 2026 21:07:32 +0100 Subject: [PATCH 24/30] Improve structure of table Add `th[scope]` attributes and split groups into separate ``s. --- conformance/results/results.html | 378 +++++++++++++++------------- conformance/src/templates/base.html | 12 +- 2 files changed, 215 insertions(+), 175 deletions(-) diff --git a/conformance/results/results.html b/conformance/results/results.html index 3876fc0c8..254a2aba0 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -252,23 +252,23 @@

Python Type System Conformance Test Results

- - mypy 2.1.0 - pycroscope 0.4.0 - pyrefly 1.1.0 - pyright 1.1.410 - ty 0.0.50 - zuban 0.8.2 + + mypy 2.1.0 + pycroscope 0.4.0 + pyrefly 1.1.0 + pyright 1.1.410 + ty 0.0.50 + zuban 0.8.2 - + Type annotations - annotations_coroutines + annotations_coroutines Pass Pass Pass @@ -277,7 +277,7 @@

Python Type System Conformance Test Results

Pass - annotations_forward_refs + annotations_forward_refs Partial
    @@ -315,7 +315,7 @@

    Python Type System Conformance Test Results

    - annotations_generators + annotations_generators Partial
      @@ -329,7 +329,7 @@

      Python Type System Conformance Test Results

      Pass - annotations_methods + annotations_methods Pass
        @@ -348,7 +348,7 @@

        Python Type System Conformance Test Results

        Pass - annotations_typeexpr + annotations_typeexpr Pass Partial @@ -361,13 +361,15 @@

        Python Type System Conformance Test Results

        Pass Pass + + - + Type forms - typeforms_typeform + typeforms_typeform Partial
          @@ -385,13 +387,15 @@

          Python Type System Conformance Test Results

          Pass Pass + + - + Special types in annotations - specialtypes_any + specialtypes_any Pass Pass Pass @@ -400,7 +404,7 @@

          Python Type System Conformance Test Results

          Pass - specialtypes_never + specialtypes_never Pass Partial @@ -415,7 +419,7 @@

          Python Type System Conformance Test Results

          Pass - specialtypes_none + specialtypes_none Pass Pass Pass @@ -424,7 +428,7 @@

          Python Type System Conformance Test Results

          Pass - specialtypes_promotions + specialtypes_promotions Pass Pass Pass @@ -433,7 +437,7 @@

          Python Type System Conformance Test Results

          Pass - specialtypes_type + specialtypes_type Partial
            @@ -453,13 +457,15 @@

            Python Type System Conformance Test Results

            Pass + + - + Generics - generics_base_class + generics_base_class Partial
              @@ -473,7 +479,7 @@

              Python Type System Conformance Test Results

              Pass - generics_basic + generics_basic Pass Pass Pass @@ -487,7 +493,7 @@

              Python Type System Conformance Test Results

              Pass - generics_defaults + generics_defaults Partial
                @@ -513,7 +519,7 @@

                Python Type System Conformance Test Results

                Pass - generics_defaults_referential + generics_defaults_referential Partial
                  @@ -532,7 +538,7 @@

                  Python Type System Conformance Test Results

                  Pass - generics_defaults_specialization + generics_defaults_specialization Partial
                    @@ -556,7 +562,7 @@

                    Python Type System Conformance Test Results

                    Pass - generics_paramspec_basic + generics_paramspec_basic Pass Pass Pass @@ -565,7 +571,7 @@

                    Python Type System Conformance Test Results

                    Pass - generics_paramspec_components + generics_paramspec_components Pass Pass Pass @@ -579,7 +585,7 @@

                    Python Type System Conformance Test Results

                    Pass - generics_paramspec_semantics + generics_paramspec_semantics Pass Pass Pass @@ -593,7 +599,7 @@

                    Python Type System Conformance Test Results

                    Pass - generics_paramspec_specialization + generics_paramspec_specialization Pass Pass Pass @@ -602,7 +608,7 @@

                    Python Type System Conformance Test Results

                    Pass - generics_scoping + generics_scoping Pass Pass @@ -622,7 +628,7 @@

                    Python Type System Conformance Test Results

                    Pass - generics_self_advanced + generics_self_advanced Partial
                      @@ -644,7 +650,7 @@

                      Python Type System Conformance Test Results

                      Pass - generics_self_attributes + generics_self_attributes Pass Pass Pass @@ -653,7 +659,7 @@

                      Python Type System Conformance Test Results

                      Pass - generics_self_basic + generics_self_basic Pass Pass Pass @@ -662,7 +668,7 @@

                      Python Type System Conformance Test Results

                      Pass - generics_self_protocols + generics_self_protocols Pass Pass Pass @@ -671,7 +677,7 @@

                      Python Type System Conformance Test Results

                      Pass - generics_self_usage + generics_self_usage Pass Pass Pass @@ -685,7 +691,7 @@

                      Python Type System Conformance Test Results

                      Pass - generics_syntax_compatibility + generics_syntax_compatibility Pass Pass Pass @@ -694,7 +700,7 @@

                      Python Type System Conformance Test Results

                      Pass - generics_syntax_declarations + generics_syntax_declarations Pass Pass Pass @@ -703,7 +709,7 @@

                      Python Type System Conformance Test Results

                      Pass - generics_syntax_infer_variance + generics_syntax_infer_variance Unsupported
                        @@ -717,7 +723,7 @@

                        Python Type System Conformance Test Results

                        Pass - generics_syntax_scoping + generics_syntax_scoping Partial
                          @@ -736,7 +742,7 @@

                          Python Type System Conformance Test Results

                          Pass - generics_type_erasure + generics_type_erasure Partial
                            @@ -756,7 +762,7 @@

                            Python Type System Conformance Test Results

                            Pass - generics_typevartuple_args + generics_typevartuple_args Partial
                              @@ -780,7 +786,7 @@

                              Python Type System Conformance Test Results

                              Pass - generics_typevartuple_basic + generics_typevartuple_basic Partial
                                @@ -799,7 +805,7 @@

                                Python Type System Conformance Test Results

                                Pass - generics_typevartuple_callable + generics_typevartuple_callable Pass Pass Pass @@ -808,7 +814,7 @@

                                Python Type System Conformance Test Results

                                Pass - generics_typevartuple_concat + generics_typevartuple_concat Pass Pass Pass @@ -817,7 +823,7 @@

                                Python Type System Conformance Test Results

                                Pass - generics_typevartuple_overloads + generics_typevartuple_overloads Pass Pass Pass @@ -826,7 +832,7 @@

                                Python Type System Conformance Test Results

                                Pass - generics_typevartuple_specialization + generics_typevartuple_specialization Partial
                                  @@ -841,7 +847,7 @@

                                  Python Type System Conformance Test Results

                                  Pass - generics_typevartuple_unpack + generics_typevartuple_unpack Pass Pass Pass @@ -850,7 +856,7 @@

                                  Python Type System Conformance Test Results

                                  Pass - generics_upper_bound + generics_upper_bound Partial
                                    @@ -864,7 +870,7 @@

                                    Python Type System Conformance Test Results

                                    Pass - generics_variance + generics_variance Partial
                                      @@ -878,7 +884,7 @@

                                      Python Type System Conformance Test Results

                                      Pass - generics_variance_inference + generics_variance_inference Pass Pass Pass @@ -886,13 +892,15 @@

                                      Python Type System Conformance Test Results

                                      Pass Pass + + - + Type qualifiers - qualifiers_annotated + qualifiers_annotated Partial
                                        @@ -916,7 +924,7 @@

                                        Python Type System Conformance Test Results

                                        Pass - qualifiers_final_annotation + qualifiers_final_annotation Partial
                                          @@ -933,7 +941,7 @@

                                          Python Type System Conformance Test Results

                                          Pass - qualifiers_final_decorator + qualifiers_final_decorator Pass Pass Pass @@ -941,13 +949,15 @@

                                          Python Type System Conformance Test Results

                                          Pass Pass + + - + Class type compatibility - classes_classvar + classes_classvar Partial
                                            @@ -964,7 +974,7 @@

                                            Python Type System Conformance Test Results

                                            Pass - classes_override + classes_override Pass Pass Pass @@ -972,13 +982,15 @@

                                            Python Type System Conformance Test Results

                                            Pass Pass + + - + Type aliases - aliases_explicit + aliases_explicit Partial
                                              @@ -997,7 +1009,7 @@

                                              Python Type System Conformance Test Results

                                              Pass - aliases_implicit + aliases_implicit Pass Partial @@ -1018,7 +1030,7 @@

                                              Python Type System Conformance Test Results

                                              Pass - aliases_newtype + aliases_newtype Partial
                                                @@ -1032,7 +1044,7 @@

                                                Python Type System Conformance Test Results

                                                Pass - aliases_recursive + aliases_recursive Pass Pass Pass @@ -1041,7 +1053,7 @@

                                                Python Type System Conformance Test Results

                                                Pass - aliases_type_statement + aliases_type_statement Pass Partial @@ -1061,7 +1073,7 @@

                                                Python Type System Conformance Test Results

                                                Pass - aliases_typealiastype + aliases_typealiastype Partial
                                                  @@ -1089,7 +1101,7 @@

                                                  Python Type System Conformance Test Results

                                                  Pass - aliases_variance + aliases_variance Pass Pass Pass @@ -1097,13 +1109,15 @@

                                                  Python Type System Conformance Test Results

                                                  Pass Pass + + - + Literals - literals_interactions + literals_interactions Pass
                                                    @@ -1122,7 +1136,7 @@

                                                    Python Type System Conformance Test Results

                                                    Pass - literals_literalstring + literals_literalstring Unsupported
                                                      @@ -1136,7 +1150,7 @@

                                                      Python Type System Conformance Test Results

                                                      Pass - literals_parameterizations + literals_parameterizations Partial
                                                        @@ -1155,7 +1169,7 @@

                                                        Python Type System Conformance Test Results

                                                        Pass - literals_semantics + literals_semantics Pass Pass Pass @@ -1163,13 +1177,15 @@

                                                        Python Type System Conformance Test Results

                                                        Pass Pass + + - + Protocols - protocols_class_objects + protocols_class_objects Pass Partial @@ -1191,7 +1207,7 @@

                                                        Python Type System Conformance Test Results

                                                        Pass - protocols_definition + protocols_definition Partial
                                                          @@ -1214,7 +1230,7 @@

                                                          Python Type System Conformance Test Results

                                                          Pass - protocols_explicit + protocols_explicit Pass
                                                            @@ -1234,7 +1250,7 @@

                                                            Python Type System Conformance Test Results

                                                            Pass - protocols_generic + protocols_generic Pass Partial @@ -1254,7 +1270,7 @@

                                                            Python Type System Conformance Test Results

                                                            Pass - protocols_merging + protocols_merging Pass Pass Pass @@ -1268,7 +1284,7 @@

                                                            Python Type System Conformance Test Results

                                                            Pass - protocols_modules + protocols_modules Pass Pass Pass @@ -1282,7 +1298,7 @@

                                                            Python Type System Conformance Test Results

                                                            Pass - protocols_recursive + protocols_recursive Pass Pass Pass @@ -1296,7 +1312,7 @@

                                                            Python Type System Conformance Test Results

                                                            Pass - protocols_runtime_checkable + protocols_runtime_checkable Partial
                                                              @@ -1315,7 +1331,7 @@

                                                              Python Type System Conformance Test Results

                                                              Pass - protocols_self + protocols_self Pass Pass Pass @@ -1324,7 +1340,7 @@

                                                              Python Type System Conformance Test Results

                                                              Pass - protocols_subtyping + protocols_subtyping Pass Pass Pass @@ -1333,7 +1349,7 @@

                                                              Python Type System Conformance Test Results

                                                              Pass - protocols_variance + protocols_variance Pass Pass Pass @@ -1341,13 +1357,15 @@

                                                              Python Type System Conformance Test Results

                                                              Unsupported Pass + + - + Callables - callables_annotation + callables_annotation Partial
                                                                @@ -1372,7 +1390,7 @@

                                                                Python Type System Conformance Test Results

                                                                Pass - callables_kwargs + callables_kwargs Partial
                                                                  @@ -1386,7 +1404,7 @@

                                                                  Python Type System Conformance Test Results

                                                                  Pass - callables_protocol + callables_protocol Pass Pass Pass @@ -1395,7 +1413,7 @@

                                                                  Python Type System Conformance Test Results

                                                                  Pass - callables_subtyping + callables_subtyping Pass Pass Pass @@ -1403,13 +1421,15 @@

                                                                  Python Type System Conformance Test Results

                                                                  Pass Pass + + - + Constructors - constructors_call_init + constructors_call_init Partial
                                                                    @@ -1430,7 +1450,7 @@

                                                                    Python Type System Conformance Test Results

                                                                    Pass - constructors_call_metaclass + constructors_call_metaclass Unsupported
                                                                      @@ -1445,7 +1465,7 @@

                                                                      Python Type System Conformance Test Results

                                                                      Pass - constructors_call_new + constructors_call_new Partial
                                                                        @@ -1461,7 +1481,7 @@

                                                                        Python Type System Conformance Test Results

                                                                        Pass - constructors_call_type + constructors_call_type Partial
                                                                          @@ -1480,7 +1500,7 @@

                                                                          Python Type System Conformance Test Results

                                                                          Pass - constructors_callable + constructors_callable Partial
                                                                            @@ -1508,7 +1528,7 @@

                                                                            Python Type System Conformance Test Results

                                                                            Pass - constructors_consistency + constructors_consistency Pass
                                                                              @@ -1521,13 +1541,15 @@

                                                                              Python Type System Conformance Test Results

                                                                              Pass Pass + + - + Overloads - overloads_basic + overloads_basic Pass Partial @@ -1541,7 +1563,7 @@

                                                                              Python Type System Conformance Test Results

                                                                              Pass - overloads_consistency + overloads_consistency Pass Pass Pass @@ -1550,7 +1572,7 @@

                                                                              Python Type System Conformance Test Results

                                                                              Pass - overloads_definitions + overloads_definitions Partial
                                                                                @@ -1564,7 +1586,7 @@

                                                                                Python Type System Conformance Test Results

                                                                                Pass - overloads_definitions_stub + overloads_definitions_stub Partial
                                                                                  @@ -1583,7 +1605,7 @@

                                                                                  Python Type System Conformance Test Results

                                                                                  Pass - overloads_evaluation + overloads_evaluation Partial
                                                                                    @@ -1605,13 +1627,15 @@

                                                                                    Python Type System Conformance Test Results

                                                                                    Pass Pass + + - + Exceptions - exceptions_context_managers + exceptions_context_managers Pass Pass @@ -1624,13 +1648,15 @@

                                                                                    Python Type System Conformance Test Results

                                                                                    Unsupported Pass + + - + Dataclasses - dataclasses_descriptors + dataclasses_descriptors Partial
                                                                                      @@ -1661,7 +1687,7 @@

                                                                                      Python Type System Conformance Test Results

                                                                                      Pass - dataclasses_final + dataclasses_final Partial
                                                                                        @@ -1676,7 +1702,7 @@

                                                                                        Python Type System Conformance Test Results

                                                                                        Pass - dataclasses_frozen + dataclasses_frozen Pass Pass Pass @@ -1685,7 +1711,7 @@

                                                                                        Python Type System Conformance Test Results

                                                                                        Pass - dataclasses_hash + dataclasses_hash Unsupported
                                                                                          @@ -1706,7 +1732,7 @@

                                                                                          Python Type System Conformance Test Results

                                                                                          Pass - dataclasses_inheritance + dataclasses_inheritance Pass Pass Pass @@ -1715,7 +1741,7 @@

                                                                                          Python Type System Conformance Test Results

                                                                                          Pass - dataclasses_kwonly + dataclasses_kwonly Pass Pass Pass @@ -1724,7 +1750,7 @@

                                                                                          Python Type System Conformance Test Results

                                                                                          Pass - dataclasses_match_args + dataclasses_match_args Pass Pass Pass @@ -1733,7 +1759,7 @@

                                                                                          Python Type System Conformance Test Results

                                                                                          Pass - dataclasses_order + dataclasses_order Pass Pass Pass @@ -1742,7 +1768,7 @@

                                                                                          Python Type System Conformance Test Results

                                                                                          Pass - dataclasses_postinit + dataclasses_postinit Pass Pass Pass @@ -1751,7 +1777,7 @@

                                                                                          Python Type System Conformance Test Results

                                                                                          Pass - dataclasses_slots + dataclasses_slots Partial
                                                                                            @@ -1770,7 +1796,7 @@

                                                                                            Python Type System Conformance Test Results

                                                                                            Pass - dataclasses_transform_class + dataclasses_transform_class Pass Pass Pass @@ -1779,7 +1805,7 @@

                                                                                            Python Type System Conformance Test Results

                                                                                            Pass - dataclasses_transform_converter + dataclasses_transform_converter Unsupported
                                                                                              @@ -1793,7 +1819,7 @@

                                                                                              Python Type System Conformance Test Results

                                                                                              Pass - dataclasses_transform_field + dataclasses_transform_field Partial
                                                                                                @@ -1807,7 +1833,7 @@

                                                                                                Python Type System Conformance Test Results

                                                                                                Pass - dataclasses_transform_func + dataclasses_transform_func Partial
                                                                                                  @@ -1822,7 +1848,7 @@

                                                                                                  Python Type System Conformance Test Results

                                                                                                  Pass - dataclasses_transform_meta + dataclasses_transform_meta Pass Pass Pass @@ -1831,7 +1857,7 @@

                                                                                                  Python Type System Conformance Test Results

                                                                                                  Pass - dataclasses_usage + dataclasses_usage Pass
                                                                                                    @@ -1844,13 +1870,15 @@

                                                                                                    Python Type System Conformance Test Results

                                                                                                    Pass Pass + + - + Typed dictionaries - typeddicts_alt_syntax + typeddicts_alt_syntax Pass
                                                                                                      @@ -1864,7 +1892,7 @@

                                                                                                      Python Type System Conformance Test Results

                                                                                                      Pass - typeddicts_class_syntax + typeddicts_class_syntax Partial
                                                                                                        @@ -1883,7 +1911,7 @@

                                                                                                        Python Type System Conformance Test Results

                                                                                                        Pass - typeddicts_extra_items + typeddicts_extra_items Unsupported Pass Pass @@ -1892,7 +1920,7 @@

                                                                                                        Python Type System Conformance Test Results

                                                                                                        Pass - typeddicts_final + typeddicts_final Pass Pass Pass @@ -1901,7 +1929,7 @@

                                                                                                        Python Type System Conformance Test Results

                                                                                                        Pass - typeddicts_inheritance + typeddicts_inheritance Pass Pass Pass @@ -1910,7 +1938,7 @@

                                                                                                        Python Type System Conformance Test Results

                                                                                                        Pass - typeddicts_operations + typeddicts_operations Pass Pass Pass @@ -1919,7 +1947,7 @@

                                                                                                        Python Type System Conformance Test Results

                                                                                                        Pass - typeddicts_readonly + typeddicts_readonly Pass Pass Pass @@ -1928,7 +1956,7 @@

                                                                                                        Python Type System Conformance Test Results

                                                                                                        Pass - typeddicts_readonly_consistency + typeddicts_readonly_consistency Pass Pass Pass @@ -1937,7 +1965,7 @@

                                                                                                        Python Type System Conformance Test Results

                                                                                                        Pass - typeddicts_readonly_inheritance + typeddicts_readonly_inheritance Partial
                                                                                                          @@ -1953,7 +1981,7 @@

                                                                                                          Python Type System Conformance Test Results

                                                                                                          Pass - typeddicts_readonly_kwargs + typeddicts_readonly_kwargs Pass Pass Pass @@ -1962,7 +1990,7 @@

                                                                                                          Python Type System Conformance Test Results

                                                                                                          Pass - typeddicts_readonly_update + typeddicts_readonly_update Partial
                                                                                                            @@ -1977,7 +2005,7 @@

                                                                                                            Python Type System Conformance Test Results

                                                                                                            Pass - typeddicts_required + typeddicts_required Pass Pass Pass @@ -1986,7 +2014,7 @@

                                                                                                            Python Type System Conformance Test Results

                                                                                                            Pass - typeddicts_type_consistency + typeddicts_type_consistency Pass Partial @@ -2000,7 +2028,7 @@

                                                                                                            Python Type System Conformance Test Results

                                                                                                            Pass - typeddicts_usage + typeddicts_usage Pass Pass Pass @@ -2008,13 +2036,15 @@

                                                                                                            Python Type System Conformance Test Results

                                                                                                            Pass Pass + + - + Tuples - tuples_type_compat + tuples_type_compat Partial
                                                                                                              @@ -2029,7 +2059,7 @@

                                                                                                              Python Type System Conformance Test Results

                                                                                                              Pass - tuples_type_form + tuples_type_form Pass Pass Pass @@ -2038,7 +2068,7 @@

                                                                                                              Python Type System Conformance Test Results

                                                                                                              Pass - tuples_unpacked + tuples_unpacked Partial
                                                                                                                @@ -2051,13 +2081,15 @@

                                                                                                                Python Type System Conformance Test Results

                                                                                                                Pass Pass + + - + Named tuples - namedtuples_define_class + namedtuples_define_class Partial
                                                                                                                  @@ -2077,7 +2109,7 @@

                                                                                                                  Python Type System Conformance Test Results

                                                                                                                  Pass - namedtuples_define_functional + namedtuples_define_functional Pass Pass Pass @@ -2086,7 +2118,7 @@

                                                                                                                  Python Type System Conformance Test Results

                                                                                                                  Pass - namedtuples_type_compat + namedtuples_type_compat Pass Pass Pass @@ -2095,7 +2127,7 @@

                                                                                                                  Python Type System Conformance Test Results

                                                                                                                  Pass - namedtuples_usage + namedtuples_usage Partial
                                                                                                                    @@ -2108,13 +2140,15 @@

                                                                                                                    Python Type System Conformance Test Results

                                                                                                                    Pass Pass + + - + Enumerations - enums_behaviors + enums_behaviors Pass Pass Pass @@ -2123,7 +2157,7 @@

                                                                                                                    Python Type System Conformance Test Results

                                                                                                                    Pass - enums_definition + enums_definition Pass Partial @@ -2137,7 +2171,7 @@

                                                                                                                    Python Type System Conformance Test Results

                                                                                                                    Pass - enums_expansion + enums_expansion Partial
                                                                                                                      @@ -2156,7 +2190,7 @@

                                                                                                                      Python Type System Conformance Test Results

                                                                                                                      Pass - enums_member_names + enums_member_names Pass
                                                                                                                        @@ -2170,7 +2204,7 @@

                                                                                                                        Python Type System Conformance Test Results

                                                                                                                        Pass - enums_member_values + enums_member_values Partial
                                                                                                                          @@ -2185,7 +2219,7 @@

                                                                                                                          Python Type System Conformance Test Results

                                                                                                                          Pass - enums_members + enums_members Partial
                                                                                                                            @@ -2208,13 +2242,15 @@

                                                                                                                            Python Type System Conformance Test Results

                                                                                                                            Pass Pass + + - + Type narrowing - narrowing_typeguard + narrowing_typeguard Pass Pass Pass @@ -2223,7 +2259,7 @@

                                                                                                                            Python Type System Conformance Test Results

                                                                                                                            Pass - narrowing_typeis + narrowing_typeis Pass Pass Pass @@ -2236,13 +2272,15 @@

                                                                                                                            Python Type System Conformance Test Results

                                                                                                                            Pass + + - + Type checker directives - directives_assert_type + directives_assert_type Pass Pass Pass @@ -2251,7 +2289,7 @@

                                                                                                                            Python Type System Conformance Test Results

                                                                                                                            Pass - directives_cast + directives_cast Pass Pass Pass @@ -2260,7 +2298,7 @@

                                                                                                                            Python Type System Conformance Test Results

                                                                                                                            Pass - directives_deprecated + directives_deprecated Pass Pass Pass @@ -2276,7 +2314,7 @@

                                                                                                                            Python Type System Conformance Test Results

                                                                                                                            Pass - directives_disjoint_base + directives_disjoint_base Pass Partial @@ -2295,7 +2333,7 @@

                                                                                                                            Python Type System Conformance Test Results

                                                                                                                            Pass - directives_no_type_check + directives_no_type_check Partial
                                                                                                                              @@ -2315,7 +2353,7 @@

                                                                                                                              Python Type System Conformance Test Results

                                                                                                                              Pass - directives_reveal_type + directives_reveal_type Pass Pass Pass @@ -2324,7 +2362,7 @@

                                                                                                                              Python Type System Conformance Test Results

                                                                                                                              Pass - directives_type_checking + directives_type_checking Pass Pass Pass @@ -2333,7 +2371,7 @@

                                                                                                                              Python Type System Conformance Test Results

                                                                                                                              Pass - directives_type_ignore + directives_type_ignore Partial
                                                                                                                                @@ -2347,7 +2385,7 @@

                                                                                                                                Python Type System Conformance Test Results

                                                                                                                                Pass - directives_type_ignore_file1 + directives_type_ignore_file1 Pass Pass Pass @@ -2356,7 +2394,7 @@

                                                                                                                                Python Type System Conformance Test Results

                                                                                                                                Pass - directives_type_ignore_file2 + directives_type_ignore_file2 Pass Pass Pass @@ -2365,7 +2403,7 @@

                                                                                                                                Python Type System Conformance Test Results

                                                                                                                                Pass - directives_version_platform + directives_version_platform Pass
                                                                                                                                  @@ -2379,13 +2417,15 @@

                                                                                                                                  Python Type System Conformance Test Results

                                                                                                                                  Pass Pass + + - + Historical and deprecated features - historical_positional + historical_positional Partial
                                                                                                                                    diff --git a/conformance/src/templates/base.html b/conformance/src/templates/base.html index 2457df6bf..79281f681 100644 --- a/conformance/src/templates/base.html +++ b/conformance/src/templates/base.html @@ -38,22 +38,22 @@

                                                                                                                                    Python Type System Conformance Test Results

                                                                                                                                    - + {%- for version in versions %} - {{ version }} + {{ version }} {%- endfor %} + {%- for group in groups %} - {%- for group in groups %} - + {{ group.name }} {%- for case in group.cases %} - {{ case.name }} + {{ case.name }} {%- for result in case.results %} {%- if result.notes %} @@ -70,8 +70,8 @@

                                                                                                                                    Python Type System Conformance Test Results

                                                                                                                                    {%- endfor %} {%- endfor %} - {%- endfor %} + {%- endfor %}