Add validator to prevent discriminator and encodedname used together (#3763)

fix [#3507](https://github.com/microsoft/typespec/issues/3507)

.chronus/changes/discriminator-encodedname-2024-6-8-16-24-7.md

@@ -0,0 +1,8 @@
+---
+# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking
+changeKind: feature
+packages:
+  - "@typespec/compiler"
+---
+
+Add validator to prevent discriminator and encodedname used together

packages/compiler/src/core/messages.ts

@@ -876,6 +876,12 @@ const diagnostics = {
       duplicate: paramMessage`Same encoded name '${"name"}' is used for 2 members '${"mimeType"}'`,
     },
   },
+  "discriminator-encodedname": {
+    severity: "error",
+    messages: {
+      default: paramMessage`Discriminator property '${"discriminator"}' cannot be use an different encoded name.`,
+    },
+  },
 
   /**
    * Service

packages/compiler/src/core/program.ts

@@ -69,7 +69,7 @@ import {
   TypeSpecLibrary,
   TypeSpecScriptNode,
 } from "./types.js";
-
+import { validateIncompatibleEncodedNameAndDiscriminator } from "./validators/encoded-name-discriminator.js";
 export interface ProjectedProgram extends Program {
   projector: Projector;
 }
@@ -742,6 +742,7 @@ export async function compile(
   /** Run the compiler built-in validators */
   function runCompilerValidators() {
     validateEncodedNamesConflicts(program);
+    validateIncompatibleEncodedNameAndDiscriminator(program);
   }
 
   function validateRequiredImports() {

packages/compiler/src/core/validators/encoded-name-discriminator.ts

@@ -0,0 +1,37 @@
+import { getEncodedNames } from "../../lib/encoded-names.js";
+import { Discriminator, getDiscriminatedTypes } from "../intrinsic-type-state.js";
+import { reportDiagnostic } from "../messages.js";
+import type { Program } from "../program.js";
+import { Model } from "../types.js";
+
+/**
+ * Validate you do not use `@encodedName` on a discriminator property.
+ */
+export function validateIncompatibleEncodedNameAndDiscriminator(program: Program) {
+  for (const [type, discriminator] of getDiscriminatedTypes(program)) {
+    if (type.kind === "Model") {
+      validateModel(program, type, discriminator);
+      for (const child of type.derivedModels) {
+        validateModel(program, child, discriminator);
+      }
+    } else {
+      for (const variant of type.variants.values()) {
+        if (variant.type.kind === "Model") {
+          validateModel(program, variant.type, discriminator);
+        }
+      }
+    }
+  }
+}
+
+function validateModel(program: Program, type: Model, discriminator: Discriminator) {
+  const property = type.properties.get(discriminator.propertyName);
+  const names = property && getEncodedNames(program, property);
+  if (names && names?.size > 0) {
+    reportDiagnostic(program, {
+      code: "discriminator-encodedname",
+      target: type,
+      format: { discriminator: discriminator.propertyName },
+    });
+  }
+}

packages/compiler/src/lib/encoded-names.ts

@@ -39,6 +39,13 @@ export function $encodedName(
   existing.set(mimeType, name);
 }
 
+/**
+ * @internal
+ */
+export function getEncodedNames(program: Program, target: Type): Map<string, string> | undefined {
+  return program.stateMap(encodedNameKey).get(target);
+}
+
 function getEncodedName(program: Program, target: Type, mimeType: string): string | undefined {
   const mimeTypeObj = parseMimeType(mimeType);
   if (mimeTypeObj === undefined) {
@@ -47,7 +54,7 @@ function getEncodedName(program: Program, target: Type, mimeType: string): strin
   const resolvedMimeType = mimeTypeObj?.suffix
     ? `${mimeTypeObj.type}/${mimeTypeObj.suffix}`
     : mimeType;
-  return program.stateMap(encodedNameKey).get(target)?.get(resolvedMimeType);
+  return getEncodedNames(program, target)?.get(resolvedMimeType);
 }
 
 /**

packages/compiler/test/decorators/decorators.test.ts

@@ -1310,6 +1310,58 @@ describe("compiler: built-in decorators", () => {
       });
     });
 
+    describe("error when use on discriminator", () => {
+      it("emit error when use on base property", async () => {
+        const diagnostics = await runner.diagnose(`
+          @discriminator("kind")
+          model Cert {
+            @encodedName("application/json", "k")
+            kind: string;
+          }
+        `);
+
+        expectDiagnostics(diagnostics, {
+          code: "discriminator-encodedname",
+        });
+      });
+
+      it("emit error when use on derived model property", async () => {
+        const diagnostics = await runner.diagnose(`
+          @discriminator("kind")
+          model Base {
+            kind: string;
+          }
+
+          model Child extends Base {
+            @encodedName("application/json", "k")
+            kind: "child";
+          }
+        `);
+
+        expectDiagnostics(diagnostics, {
+          code: "discriminator-encodedname",
+        });
+      });
+
+      it("emit error when use on union variant property", async () => {
+        const diagnostics = await runner.diagnose(`
+          @discriminator("kind")
+          union Base {
+            child: Child
+          }
+
+          model Child {
+            @encodedName("application/json", "k")
+            kind: "child";
+          }
+        `);
+
+        expectDiagnostics(diagnostics, {
+          code: "discriminator-encodedname",
+        });
+      });
+    });
+
     it("resolve explicit encoded name", async () => {
       const { expireAt } = (await runner.compile(`
         model Cert {