Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
changeKind: fix
packages:
- "@typespec/http-client-python"
---

Preserve nullability in Python type annotations for required model properties and array or dictionary elements, including nested collections. Required nullable properties remain required constructor arguments.
1 change: 1 addition & 0 deletions packages/http-client-python/emitter/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -434,6 +434,7 @@ function emitArrayOrDict(
return getSimpleTypeResult(context, {
type: kind,
elementType: getType(context, type.valueType),
...(type.valueType.kind === "nullable" ? { elementNullable: true } : {}),
});
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
import { expectDiagnostics } from "@typespec/compiler/testing";
import { expect, it } from "vitest";
import { emitCodeModel, type CodeModelType } from "./test-host.js";

interface ModelWithProperties extends CodeModelType {
properties: { clientName: string; type: CollectionType }[];
}

interface CollectionType extends CodeModelType {
elementType: CodeModelType;
elementNullable?: boolean;
}

async function emitCollections(properties: string) {
const { codeModel, diagnostics } = await emitCodeModel(`
import "@typespec/http";
using TypeSpec.Http;
@service namespace Test;
model Item { value: string; }
model Collections { ${properties} }
@post op echo(@body body: Collections): Collections;
`);
expectDiagnostics(diagnostics, []);
const model = codeModel.types.find((type) => type.name === "Collections") as ModelWithProperties;
expect(model).toBeDefined();
return Object.fromEntries(
model.properties.map((property) => [property.clientName, property.type]),
);
}

it.each(["Item[]", "Record<Item>"])(
"preserves nullable elements without mutating shared types for %s",
async (plainType) => {
const nullableType = plainType === "Item[]" ? "(Item | null)[]" : "Record<Item | null>";
const properties = await emitCollections(`
plain: ${plainType};
nullable: ${nullableType};
repeated: ${nullableType};
`);

expect(properties.plain.elementNullable).toBeUndefined();
expect(properties.nullable.elementNullable).toBe(true);
expect(properties.nullable).not.toBe(properties.plain);
expect(properties.nullable).toBe(properties.repeated);
expect(properties.nullable.elementType).toBe(properties.plain.elementType);
expect(properties.plain.elementType).not.toHaveProperty("nullable");
},
);

it("preserves nullability at each level of nested collections", async () => {
const properties = await emitCollections(`
arrays: ((string | null)[] | null)[];
dictionaries: Record<Record<string | null> | null>;
mixed: Record<(string | null)[]>;
`);

const nullableStrings = { type: "list", elementNullable: true, elementType: { type: "string" } };
expect(properties.arrays).toEqual({
type: "list",
elementNullable: true,
elementType: nullableStrings,
});
expect(properties.dictionaries).toEqual({
type: "dict",
elementNullable: true,
elementType: { type: "dict", elementNullable: true, elementType: { type: "string" } },
});
expect(properties.mixed).toEqual({ type: "dict", elementType: nullableStrings });
});
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
# --------------------------------------------------------------------------
from typing import Any, Optional, TYPE_CHECKING
from .base import BaseType
from .imports import FileImport
from .imports import FileImport, ImportType

if TYPE_CHECKING:
from .code_model import CodeModel
Expand All @@ -29,6 +29,7 @@ def __init__(
) -> None:
super().__init__(yaml_data=yaml_data, code_model=code_model)
self.element_type = element_type
self.element_nullable: bool = yaml_data.get("elementNullable", False)

@property
def encode(self) -> Optional[str]:
Expand All @@ -48,13 +49,19 @@ def type_annotation(self, **kwargs: Any) -> str:
:return: The type annotation for this schema
:rtype: str
"""
return f"dict[str, {self.element_type.type_annotation(**kwargs)}]"
element_annotation = self.element_type.type_annotation(**kwargs)
if self.element_nullable:
element_annotation = f"Optional[{element_annotation}]"
return f"dict[str, {element_annotation}]"

def description(self, *, is_operation_file: bool) -> str:
return "" if is_operation_file else self.yaml_data.get("description", "")

def docstring_text(self, **kwargs: Any) -> str:
return f"dict mapping str to {self.element_type.docstring_text(**kwargs)}"
element_text = self.element_type.docstring_text(**kwargs)
if self.element_nullable:
element_text = f"({element_text} or None)"
return f"dict mapping str to {element_text}"

@property
def xml_serialization_ctxt(self) -> Optional[str]:
Expand All @@ -66,7 +73,10 @@ def docstring_type(self, **kwargs: Any) -> str:

:param str namespace: Optional. The namespace for the models.
"""
return f"dict[str, {self.element_type.docstring_type(**kwargs)}]"
element_docstring = self.element_type.docstring_type(**kwargs)
if self.element_nullable:
element_docstring = f"Optional[{element_docstring}]"
return f"dict[str, {element_docstring}]"

def get_json_template_representation(
self,
Expand Down Expand Up @@ -114,6 +124,8 @@ def from_yaml(cls, yaml_data: dict[str, Any], code_model: "CodeModel") -> "Dicti
def imports(self, **kwargs: Any) -> FileImport:
file_import = FileImport(self.code_model)
file_import.merge(self.element_type.imports(**kwargs))
if self.element_nullable:
file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB)
return file_import

@property
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
# --------------------------------------------------------------------------
from typing import Any, Optional, Union, TYPE_CHECKING
from .base import BaseType
from .imports import FileImport
from .imports import FileImport, ImportType
from .utils import NamespaceType

if TYPE_CHECKING:
Expand All @@ -22,6 +22,7 @@ def __init__(
) -> None:
super().__init__(yaml_data=yaml_data, code_model=code_model)
self.element_type = element_type
self.element_nullable: bool = yaml_data.get("elementNullable", False)
self.max_items: Optional[int] = yaml_data.get("maxItems")
self.min_items: Optional[int] = yaml_data.get("minItems")
self.unique_items: bool = yaml_data.get("uniqueItems", False)
Expand Down Expand Up @@ -54,7 +55,10 @@ def type_annotation(self, **kwargs: Any) -> str:
list_type = "List"
else:
list_type = "list"
return f"{list_type}[{self.element_type.type_annotation(**kwargs)}]"
element_annotation = self.element_type.type_annotation(**kwargs)
if self.element_nullable:
element_annotation = f"Optional[{element_annotation}]"
return f"{list_type}[{element_annotation}]"

def description(self, *, is_operation_file: bool) -> str:
return "" if is_operation_file else self.yaml_data.get("description", "")
Expand Down Expand Up @@ -85,13 +89,19 @@ def docstring_type(self, **kwargs: Any) -> str:
if self.code_model.options["version-tolerant"] and self.element_type.xml_metadata:
# this means we're version tolerant XML, we just return the XML element
return self.element_type.docstring_type(**kwargs)
return f"list[{self.element_type.docstring_type(**kwargs)}]"
element_docstring = self.element_type.docstring_type(**kwargs)
if self.element_nullable:
element_docstring = f"Optional[{element_docstring}]"
return f"list[{element_docstring}]"

def docstring_text(self, **kwargs: Any) -> str:
if self.code_model.options["version-tolerant"] and self.element_type.xml_metadata:
# this means we're version tolerant XML, we just return the XML element
return self.element_type.docstring_text(**kwargs)
return f"list of {self.element_type.docstring_text(**kwargs)}"
element_text = self.element_type.docstring_text(**kwargs)
if self.element_nullable:
element_text = f"({element_text} or None)"
return f"list of {element_text}"

@property
def validation(self) -> Optional[dict[str, Union[bool, int, str]]]:
Expand Down Expand Up @@ -143,6 +153,8 @@ def from_yaml(cls, yaml_data: dict[str, Any], code_model: "CodeModel") -> "ListT
def imports(self, **kwargs: Any) -> FileImport:
file_import = FileImport(self.code_model)
file_import.merge(self.element_type.imports(**kwargs))
if self.element_nullable:
file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB)
return file_import

@property
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ def type_annotation(self, *, is_operation_file: bool = False, **kwargs: Any) ->
if self.nullable:
return f"Optional[{types_type_annotation}]"
return types_type_annotation
if (self.optional and self.client_default_value is None) or self.readonly:
if self.nullable or (self.optional and self.client_default_value is None) or self.readonly:
return f"Optional[{types_type_annotation}]"
return types_type_annotation

Expand Down Expand Up @@ -164,7 +164,7 @@ def imports(self, **kwargs) -> FileImport:
# In TypedDict types.py, Optional means nullable
if self.nullable:
file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB)
elif (self.optional and self.client_default_value is None) or self.readonly:
elif self.nullable or (self.optional and self.client_default_value is None) or self.readonly:
file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB)
if self.code_model.options["models-mode"] == "dpg":
if serialize_namespace_type != NamespaceType.TYPES_FILE:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1219,15 +1219,20 @@ def _deserialize_sequence(
if isinstance(obj, ET.Element):
obj = list(obj)

# Optional elements still use the array's encoding; keep their wrapper for item decoding.
array_deserializer = deserializer
if isinstance(array_deserializer, functools.partial) and array_deserializer.func is _deserialize_with_optional:
array_deserializer = array_deserializer.args[0]

# encoded string may be deserialized to sequence
if isinstance(obj, str) and isinstance(deserializer, functools.partial):
if isinstance(obj, str) and isinstance(array_deserializer, functools.partial):
# for list[str]
if _is_array_encoded_deserializer(deserializer):
return deserializer(obj)
if _is_array_encoded_deserializer(array_deserializer):
return array_deserializer(obj)

# for list[Union[...]]
if isinstance(deserializer.args[0], list):
for sub_deserializer in deserializer.args[0]:
if isinstance(array_deserializer.args[0], list):
for sub_deserializer in array_deserializer.args[0]:
if _is_array_encoded_deserializer(sub_deserializer):
return sub_deserializer(obj)

Expand All @@ -1244,6 +1249,8 @@ def _get_deserialize_callable_from_annotation( # pylint: disable=too-many-retur
annotation: typing.Any,
module: typing.Optional[str],
rf: typing.Optional["_RestField"] = None,
*,
is_field_type: bool = True,
) -> typing.Optional[typing.Callable[[typing.Any], typing.Any]]:
if not annotation:
return None
Expand Down Expand Up @@ -1281,26 +1288,30 @@ def _get_deserialize_callable_from_annotation( # pylint: disable=too-many-retur
# is it optional?
try:
if any(a is _NONE_TYPE for a in annotation.__args__): # pyright: ignore
if rf:
# Nullable elements do not make the containing XML field optional.
if rf and is_field_type:
rf._is_optional = True
if len(annotation.__args__) <= 2: # pyright: ignore
if_obj_deserializer = _get_deserialize_callable_from_annotation(
next(a for a in annotation.__args__ if a is not _NONE_TYPE), module, rf # pyright: ignore
next(a for a in annotation.__args__ if a is not _NONE_TYPE), module, rf, # pyright: ignore
is_field_type=is_field_type,
)

return functools.partial(_deserialize_with_optional, if_obj_deserializer)
# the type is Optional[Union[...]], we need to remove the None type from the Union
annotation_copy = copy.copy(annotation)
annotation_copy.__args__ = [a for a in annotation_copy.__args__ if a is not _NONE_TYPE] # pyright: ignore
return _get_deserialize_callable_from_annotation(annotation_copy, module, rf)
return _get_deserialize_callable_from_annotation(
annotation_copy, module, rf, is_field_type=is_field_type
)
except AttributeError:
pass

# is it union?
if getattr(annotation, "__origin__", None) is typing.Union:
# initial ordering is we make `string` the last deserialization option, because it is often them most generic
deserializers = [
_get_deserialize_callable_from_annotation(arg, module, rf)
_get_deserialize_callable_from_annotation(arg, module, rf, is_field_type=is_field_type)
for arg in _sorted_annotations(annotation.__args__) # pyright: ignore
]

Expand All @@ -1312,7 +1323,7 @@ def _get_deserialize_callable_from_annotation( # pylint: disable=too-many-retur
)
if annotation_name.lower() == "dict":
value_deserializer = _get_deserialize_callable_from_annotation(
annotation.__args__[1], module, rf # pyright: ignore
annotation.__args__[1], module, rf, is_field_type=False # pyright: ignore
)

return functools.partial(
Expand All @@ -1329,12 +1340,12 @@ def _get_deserialize_callable_from_annotation( # pylint: disable=too-many-retur
if annotation_name.lower() in ["list", "set", "tuple", "sequence"]:
if len(annotation.__args__) > 1: # pyright: ignore
entry_deserializers = [
_get_deserialize_callable_from_annotation(dt, module, rf)
_get_deserialize_callable_from_annotation(dt, module, rf, is_field_type=False)
for dt in annotation.__args__ # pyright: ignore
]
return functools.partial(_deserialize_multiple_sequence, entry_deserializers, module)
deserializer = _get_deserialize_callable_from_annotation(
annotation.__args__[0], module, rf # pyright: ignore
annotation.__args__[0], module, rf, is_field_type=False # pyright: ignore
)

return functools.partial(_deserialize_sequence, deserializer, module)
Expand Down
Loading