Black format everything and update flake8 linter (#583)

Updated version of black
Fix lints targeting removing 3.7 compatibility.
Update output formatting.

.github/workflows/main.yml

@@ -89,10 +89,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.8"
       - name: Run formatters and linters
         run: |
-          pip3 install black isort flake8-pyi flake8-noqa flake8-bugbear
+          pip3 install black==24.3.0 isort flake8 flake8-pyi flake8-noqa flake8-bugbear
           black --check --extend-exclude '(_pb2_grpc|_pb2).pyi?$' .
           isort --check . --diff
           flake8 .
@@ -107,7 +109,9 @@ jobs:
     runs-on: windows-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.8"
       - name: Read versions
         run: echo ::set-output name=PROTOBUF_VERSION::$(grep "^protobuf>=" test_requirements.txt | cut -f2 -d=)
         id: read_versions

CHANGELOG.md

@@ -1,5 +1,7 @@
 ## Upcoming
 
+- Remove 3.7 compatibility for typing_extensions.final/Literal
+
 ## 3.5.0
 
 - Add gRPC aio stub and servicer generation (#489)

mypy_protobuf/extensions_pb2.pyi

@@ -2,21 +2,17 @@
 @generated by mypy-protobuf.  Do not edit manually!
 isort:skip_file
 """
+
 import builtins
 import google.protobuf.descriptor
 import google.protobuf.descriptor_pb2
 import google.protobuf.internal.extension_dict
 import google.protobuf.message
-import sys
-
-if sys.version_info >= (3, 8):
-    import typing as typing_extensions
-else:
-    import typing_extensions
+import typing
 
 DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
 
-@typing_extensions.final
+@typing.final
 class FieldOptions(google.protobuf.message.Message):
     DESCRIPTOR: google.protobuf.descriptor.Descriptor
 
@@ -36,7 +32,7 @@ class FieldOptions(google.protobuf.message.Message):
         keytype: builtins.str = ...,
         valuetype: builtins.str = ...,
     ) -> None: ...
-    def ClearField(self, field_name: typing_extensions.Literal["casttype", b"casttype", "keytype", b"keytype", "valuetype", b"valuetype"]) -> None: ...
+    def ClearField(self, field_name: typing.Literal["casttype", b"casttype", "keytype", b"keytype", "valuetype", b"valuetype"]) -> None: ...
 
 global___FieldOptions = FieldOptions
 

mypy_protobuf/main.py

@@ -169,9 +169,7 @@ def _import(self, path: str, name: str) -> str:
         """
         if path == "typing_extensions":
             stabilization = {
-                "Literal": (3, 8),
                 "TypeAlias": (3, 10),
-                "final": (3, 8),
             }
             assert name in stabilization
             if not self.typing_extensions_min or self.typing_extensions_min < stabilization[name]:
@@ -407,7 +405,7 @@ def write_messages(
 
             class_name = desc.name if desc.name not in PYTHON_RESERVED else "_r_" + desc.name
             message_class = self._import("google.protobuf.message", "Message")
-            wl("@{}", self._import("typing_extensions", "final"))
+            wl("@{}", self._import("typing", "final"))
             wl(f"class {class_name}({message_class}{addl_base}):")
             with self._indent():
                 scl = scl_prefix + [i]
@@ -438,12 +436,16 @@ def write_messages(
                     if field.name in PYTHON_RESERVED:
                         continue
                     field_type = self.python_type(field)
-
                     if is_scalar(field) and field.label != d.FieldDescriptorProto.LABEL_REPEATED:
                         # Scalar non repeated fields are r/w
                         wl(f"{field.name}: {field_type}")
                         self._write_comments(scl + [d.DescriptorProto.FIELD_FIELD_NUMBER, idx])
-                    else:
+
+                for idx, field in enumerate(desc.field):
+                    if field.name in PYTHON_RESERVED:
+                        continue
+                    field_type = self.python_type(field)
+                    if not (is_scalar(field) and field.label != d.FieldDescriptorProto.LABEL_REPEATED):
                         # r/o Getters for non-scalar fields and scalar-repeated fields
                         scl_field = scl + [d.DescriptorProto.FIELD_FIELD_NUMBER, idx]
                         wl("@property")
@@ -452,6 +454,7 @@ def write_messages(
                         if self._has_comments(scl_field):
                             with self._indent():
                                 self._write_comments(scl_field)
+                            wl("")
 
                 self.write_extensions(desc.extension, scl + [d.DescriptorProto.EXTENSION_FIELD_NUMBER])
 
@@ -506,14 +509,14 @@ def write_stringly_typed_fields(self, desc: d.DescriptorProto) -> None:
         if hf_fields:
             wl(
                 "def HasField(self, field_name: {}[{}]) -> {}: ...",
-                self._import("typing_extensions", "Literal"),
+                self._import("typing", "Literal"),
                 hf_fields_text,
                 self._builtin("bool"),
             )
         if cf_fields:
             wl(
                 "def ClearField(self, field_name: {}[{}]) -> None: ...",
-                self._import("typing_extensions", "Literal"),
+                self._import("typing", "Literal"),
                 cf_fields_text,
             )
 
@@ -522,10 +525,10 @@ def write_stringly_typed_fields(self, desc: d.DescriptorProto) -> None:
                 wl("@{}", self._import("typing", "overload"))
             wl(
                 "def WhichOneof(self, oneof_group: {}[{}]) -> {}[{}] | None: ...",
-                self._import("typing_extensions", "Literal"),
+                self._import("typing", "Literal"),
                 # Accepts both str and bytes
                 f'"{wo_field}", b"{wo_field}"',
-                self._import("typing_extensions", "Literal"),
+                self._import("typing", "Literal"),
                 # Returns `str`
                 ", ".join(f'"{m}"' for m in members),
             )
@@ -599,6 +602,7 @@ def write_methods(
                 with self._indent():
                     if not self._write_comments(scl_method):
                         wl("...")
+            wl("")
 
     def write_services(
         self,
@@ -620,7 +624,6 @@ def write_services(
                 if self._write_comments(scl):
                     wl("")
                 self.write_methods(service, class_name, is_abstract=True, scl_prefix=scl)
-            wl("")
 
             # The stub client
             stub_class_name = service.name + "_Stub"
@@ -633,7 +636,6 @@ def write_services(
                     self._import("google.protobuf.service", "RpcChannel"),
                 )
                 self.write_methods(service, stub_class_name, is_abstract=False, scl_prefix=scl)
-            wl("")
 
     def _import_casttype(self, casttype: str) -> str:
         split = casttype.split(".")
@@ -709,16 +711,14 @@ def write_grpc_async_hacks(self) -> None:
         wl = self._write_line
         # _MaybeAsyncIterator[Req] is supertyped by Iterator[Req] and AsyncIterator[Req].
         # So both can be used in the contravariant function parameter position.
-        wl("_T = {}('_T')", self._import("typing", "TypeVar"))
+        wl('_T = {}("_T")', self._import("typing", "TypeVar"))
         wl("")
         wl(
-            "class _MaybeAsyncIterator({}[_T], {}[_T], metaclass={}):",
+            "class _MaybeAsyncIterator({}[_T], {}[_T], metaclass={}): ...",
             self._import("collections.abc", "AsyncIterator"),
             self._import("collections.abc", "Iterator"),
             self._import("abc", "ABCMeta"),
         )
-        with self._indent():
-            wl("...")
         wl("")
 
         # _ServicerContext is supertyped by grpc.ServicerContext and grpc.aio.ServicerContext
@@ -758,6 +758,7 @@ def write_grpc_methods(self, service: d.ServiceDescriptorProto, scl_prefix: Sour
                 with self._indent():
                     if not self._write_comments(scl):
                         wl("...")
+            wl("")
 
     def write_grpc_stub_methods(self, service: d.ServiceDescriptorProto, scl_prefix: SourceCodeLocation, is_async: bool = False) -> None:
         wl = self._write_line
@@ -774,6 +775,7 @@ def write_grpc_stub_methods(self, service: d.ServiceDescriptorProto, scl_prefix:
                 wl("{},", self._output_type(method))
             wl("]")
             self._write_comments(scl)
+            wl("")
 
     def write_grpc_services(
         self,
@@ -799,7 +801,6 @@ def write_grpc_services(
                 channel = f"{self._import('typing', 'Union')}[{self._import('grpc', 'Channel')}, {self._import('grpc.aio', 'Channel')}]"
                 wl("def __init__(self, channel: {}) -> None: ...", channel)
                 self.write_grpc_stub_methods(service, scl)
-            wl("")
 
             # The (fake) async stub client
             wl(
@@ -811,7 +812,6 @@ def write_grpc_services(
                     wl("")
                 # No __init__ since this isn't a real class (yet), and requires manual casting to work.
                 self.write_grpc_stub_methods(service, scl, is_async=True)
-            wl("")
 
             # The service definition interface
             wl(
@@ -823,7 +823,6 @@ def write_grpc_services(
                 if self._write_comments(scl):
                     wl("")
                 self.write_grpc_methods(service, scl)
-            wl("")
             server = self._import("grpc", "Server")
             aserver = self._import("grpc.aio", "Server")
             wl(
@@ -925,8 +924,9 @@ def write(self) -> str:
         if self.lines:
             assert self.lines[0].startswith('"""')
             self.lines[0] = f'"""{HEADER}{self.lines[0][3:]}'
+            self._write_line("")
         else:
-            self._write_line(f'"""{HEADER}"""')
+            self._write_line(f'"""{HEADER}"""\n')
 
         for reexport_idx in self.fd.public_dependency:
             reexport_file = self.fd.dependency[reexport_idx]

test/generated/google/protobuf/duration_pb2.pyi

@@ -31,20 +31,16 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 """
+
 import builtins
 import google.protobuf.descriptor
 import google.protobuf.internal.well_known_types
 import google.protobuf.message
-import sys
-
-if sys.version_info >= (3, 8):
-    import typing as typing_extensions
-else:
-    import typing_extensions
+import typing
 
 DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
 
-@typing_extensions.final
+@typing.final
 class Duration(google.protobuf.message.Message, google.protobuf.internal.well_known_types.Duration):
     """A Duration represents a signed, fixed-length span of time represented
     as a count of seconds and fractions of seconds at nanosecond
@@ -129,6 +125,6 @@ class Duration(google.protobuf.message.Message, google.protobuf.internal.well_kn
         seconds: builtins.int = ...,
         nanos: builtins.int = ...,
     ) -> None: ...
-    def ClearField(self, field_name: typing_extensions.Literal["nanos", b"nanos", "seconds", b"seconds"]) -> None: ...
+    def ClearField(self, field_name: typing.Literal["nanos", b"nanos", "seconds", b"seconds"]) -> None: ...
 
 global___Duration = Duration

test/generated/mypy_protobuf/extensions_pb2.pyi

@@ -2,21 +2,17 @@
 @generated by mypy-protobuf.  Do not edit manually!
 isort:skip_file
 """
+
 import builtins
 import google.protobuf.descriptor
 import google.protobuf.descriptor_pb2
 import google.protobuf.internal.extension_dict
 import google.protobuf.message
-import sys
-
-if sys.version_info >= (3, 8):
-    import typing as typing_extensions
-else:
-    import typing_extensions
+import typing
 
 DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
 
-@typing_extensions.final
+@typing.final
 class FieldOptions(google.protobuf.message.Message):
     DESCRIPTOR: google.protobuf.descriptor.Descriptor
 
@@ -36,7 +32,7 @@ class FieldOptions(google.protobuf.message.Message):
         keytype: builtins.str = ...,
         valuetype: builtins.str = ...,
     ) -> None: ...
-    def ClearField(self, field_name: typing_extensions.Literal["casttype", b"casttype", "keytype", b"keytype", "valuetype", b"valuetype"]) -> None: ...
+    def ClearField(self, field_name: typing.Literal["casttype", b"casttype", "keytype", b"keytype", "valuetype", b"valuetype"]) -> None: ...
 
 global___FieldOptions = FieldOptions
 

test/generated/testproto/Capitalized/Capitalized_pb2.pyi

@@ -2,19 +2,15 @@
 @generated by mypy-protobuf.  Do not edit manually!
 isort:skip_file
 """
+
 import builtins
 import google.protobuf.descriptor
 import google.protobuf.message
-import sys
-
-if sys.version_info >= (3, 8):
-    import typing as typing_extensions
-else:
-    import typing_extensions
+import typing
 
 DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
 
-@typing_extensions.final
+@typing.final
 class lower(google.protobuf.message.Message):
     DESCRIPTOR: google.protobuf.descriptor.Descriptor
 
@@ -25,11 +21,11 @@ class lower(google.protobuf.message.Message):
         *,
         a: builtins.int = ...,
     ) -> None: ...
-    def ClearField(self, field_name: typing_extensions.Literal["a", b"a"]) -> None: ...
+    def ClearField(self, field_name: typing.Literal["a", b"a"]) -> None: ...
 
 global___lower = lower
 
-@typing_extensions.final
+@typing.final
 class Upper(google.protobuf.message.Message):
     DESCRIPTOR: google.protobuf.descriptor.Descriptor
 
@@ -41,12 +37,12 @@ class Upper(google.protobuf.message.Message):
         *,
         Lower: global___lower | None = ...,
     ) -> None: ...
-    def HasField(self, field_name: typing_extensions.Literal["Lower", b"Lower"]) -> builtins.bool: ...
-    def ClearField(self, field_name: typing_extensions.Literal["Lower", b"Lower"]) -> None: ...
+    def HasField(self, field_name: typing.Literal["Lower", b"Lower"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing.Literal["Lower", b"Lower"]) -> None: ...
 
 global___Upper = Upper
 
-@typing_extensions.final
+@typing.final
 class lower2(google.protobuf.message.Message):
     DESCRIPTOR: google.protobuf.descriptor.Descriptor
 
@@ -58,7 +54,7 @@ class lower2(google.protobuf.message.Message):
         *,
         upper: global___Upper | None = ...,
     ) -> None: ...
-    def HasField(self, field_name: typing_extensions.Literal["upper", b"upper"]) -> builtins.bool: ...
-    def ClearField(self, field_name: typing_extensions.Literal["upper", b"upper"]) -> None: ...
+    def HasField(self, field_name: typing.Literal["upper", b"upper"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing.Literal["upper", b"upper"]) -> None: ...
 
 global___lower2 = lower2

test/generated/testproto/comment_special_chars_pb2.pyi

@@ -2,19 +2,15 @@
 @generated by mypy-protobuf.  Do not edit manually!
 isort:skip_file
 """
+
 import builtins
 import google.protobuf.descriptor
 import google.protobuf.message
-import sys
-
-if sys.version_info >= (3, 8):
-    import typing as typing_extensions
-else:
-    import typing_extensions
+import typing
 
 DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
 
-@typing_extensions.final
+@typing.final
 class Test(google.protobuf.message.Message):
     DESCRIPTOR: google.protobuf.descriptor.Descriptor
 
@@ -75,6 +71,6 @@ class Test(google.protobuf.message.Message):
         j: builtins.str = ...,
         k: builtins.str = ...,
     ) -> None: ...
-    def ClearField(self, field_name: typing_extensions.Literal["a", b"a", "b", b"b", "c", b"c", "d", b"d", "e", b"e", "f", b"f", "g", b"g", "h", b"h", "i", b"i", "j", b"j", "k", b"k"]) -> None: ...
+    def ClearField(self, field_name: typing.Literal["a", b"a", "b", b"b", "c", b"c", "d", b"d", "e", b"e", "f", b"f", "g", b"g", "h", b"h", "i", b"i", "j", b"j", "k", b"k"]) -> None: ...
 
 global___Test = Test