Fix: OpenAPI3 not marking part of bytes or something else as `format: binary` (#3013)

fix [#2924](https://github.com/microsoft/typespec/issues/2924)

---------

Co-authored-by: Mark Cowlishaw <markcowl@microsoft.com>

.chronus/changes/fix-openapi3-multipart-union-bytes-2024-2-12-22-52-55.md

@@ -0,0 +1,8 @@
+---
+# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking
+changeKind: fix
+packages:
+  - "@typespec/openapi3"
+---
+
+Fix: OpenAPI3 not marking part of bytes or something else as `format: binary`

packages/openapi3/src/schema-emitter.ts

@@ -339,7 +339,8 @@ export class OpenAPI3SchemaEmitter extends TypeEmitter<
     }
 
     const refSchema = this.emitter.emitTypeReference(prop.type, {
-      referenceContext: isMultipart ? { contentType: "application/json" } : {},
+      referenceContext:
+        prop.type.kind !== "Union" && isMultipart ? { contentType: "application/json" } : {},
     });
 
     if (refSchema.kind !== "code") {
@@ -471,6 +472,7 @@ export class OpenAPI3SchemaEmitter extends TypeEmitter<
     const schemaMembers: { schema: any; type: Type | null }[] = [];
     let nullable = false;
     const discriminator = getDiscriminator(program, union);
+    const isMultipart = this.#getContentType().startsWith("multipart/");
 
     for (const variant of variants) {
       if (isNullType(variant.type)) {
@@ -478,6 +480,11 @@ export class OpenAPI3SchemaEmitter extends TypeEmitter<
         continue;
       }
 
+      if (isMultipart && this.#isBytesKeptRaw(variant.type)) {
+        schemaMembers.push({ schema: { type: "string", format: "binary" }, type: variant.type });
+        continue;
+      }
+
       if (isLiteralType(variant.type)) {
         if (!literalVariantEnumByType[variant.type.kind]) {
           const enumSchema = this.emitter.emitTypeReference(variant.type);
@@ -491,7 +498,9 @@ export class OpenAPI3SchemaEmitter extends TypeEmitter<
           literalVariantEnumByType[variant.type.kind].enum.push(variant.type.value);
         }
       } else {
-        const enumSchema = this.emitter.emitTypeReference(variant.type);
+        const enumSchema = this.emitter.emitTypeReference(variant.type, {
+          referenceContext: isMultipart ? { contentType: "application/json" } : {},
+        });
         compilerAssert(enumSchema.kind === "code", "Unexpected enum schema. Should be kind: code");
         schemaMembers.push({ schema: enumSchema.value, type: variant.type });
       }

packages/openapi3/test/multipart.test.ts

@@ -39,6 +39,38 @@ describe("typespec-autorest: multipart", () => {
     });
   });
 
+  it("part of type union `bytes | {content: bytes}` produce `type: string, format: binary`", async () => {
+    const res = await openApiFor(
+      `
+      op upload(@header contentType: "multipart/form-data", profileImage: bytes | {content: bytes}): void;
+      `
+    );
+    const op = res.paths["/"].post;
+    deepStrictEqual(op.requestBody.content["multipart/form-data"], {
+      schema: {
+        type: "object",
+        properties: {
+          profileImage: {
+            anyOf: [
+              {
+                type: "string",
+                format: "binary",
+              },
+              {
+                type: "object",
+                properties: {
+                  content: { type: "string", format: "byte" },
+                },
+                required: ["content"],
+              },
+            ],
+          },
+        },
+        required: ["profileImage"],
+      },
+    });
+  });
+
   it("part of type `bytes[]` produce `type: array, items: {type: string, format: binary}`", async () => {
     const res = await openApiFor(
       `