[core] Improve documentation and implementation of Typekit, Mutator, …

…and Realm (microsoft#5149)

- The experimental APIs are now documented with JSDoc that explains how
to use the APIs.
- The relationship between Realm and Typekit has been clarified and
strengthened. A Typekit is now strongly bound to a realm on creation.
- When using the default typekit, a default typekit realm is created for
the current program if one does not exist already.
- RealmKit has been removed (this was previously only used to statefully
set/get the realm used by the default typekit), now `realm` is a
readonly property of the typekit.
- The default typekit `$` has been merged with a function that gets a
typekit bound to a specific realm (`$(realm)`) or bound to the default
typekit realm of a specific program (`$(program)`) making it easier to
work in specific program contexts (e.g. ProjectedProgram). The default
typekit bound to a realm is cached in the Realm itself.
- Each Program now has a "default typekit realm" that is created as
needed when a typekit for that program is requested.
- Additional typekit instances can still be created using
`createTypekit` directly, bypassing the realm-caching behavior.
- Mutators now use a typekit bound to their mutation realm, so that
types cloned by the mutator are created within the realm and not within
the typekit's default realm. (Previously, the mutator realm was unused
within the mutator itself, only passed to the mutator functions).
- The type of the mutator definition is now inferred from the definition
of `MutableType` for accuracy. The previous definition was inaccurate
and would fail to represent mutations of literal types correctly.
- Added a test that mutation of literal types works as expected.
- Worked out some module reference graph issues between defineKit and
the typekit index that could sometimes lead to typekits being imported
and invoked without the definitions of the typekit implementations being
loaded.

---------

Co-authored-by: Will Temple <[email protected]>

.chronus/changes/witemple-msft-realm-typekit-mutator-docs-2024-10-19-10-24-24.md

@@ -0,0 +1,11 @@
+---
+changeKind: feature
+packages:
+  - "@typespec/compiler"
+---
+
+Experimental: Improve Realm, Mutator, and Typekit implementations.
+
+This change strongly binds a Realm and Typekit together, and changes mutators so that new types are cloned within the
+mutator's realm. The default Typekit now creates a default typekit realm for the current program, and a Typekit can be
+easily created to work in a specific Program or Realm as needed.

packages/compiler/src/core/program.ts

@@ -1,6 +1,6 @@
 import { EmitterOptions } from "../config/types.js";
 import { createAssetEmitter } from "../emitter-framework/asset-emitter.js";
-import { setCurrentProgram } from "../experimental/typekit/define-kit.js";
+import { setCurrentProgram } from "../experimental/typekit/index.js";
 import { validateEncodedNamesConflicts } from "../lib/encoded-names.js";
 import { validatePagingOperations } from "../lib/paging.js";
 import { MANIFEST } from "../manifest.js";

packages/compiler/src/experimental/mutators.ts

@@ -1,95 +1,200 @@
 import { Program } from "../core/program.js";
 import {
   Decorator,
-  Enum,
-  EnumMember,
   FunctionParameter,
   FunctionType,
-  Interface,
   IntrinsicType,
-  Model,
-  ModelProperty,
   Namespace,
   ObjectType,
-  Operation,
   Projection,
-  Scalar,
-  ScalarConstructor,
-  StringTemplate,
-  StringTemplateSpan,
   TemplateParameter,
-  Tuple,
   Type,
-  Union,
-  UnionVariant,
 } from "../core/types.js";
 import { CustomKeyMap } from "../emitter-framework/custom-key-map.js";
 import { Realm } from "./realm.js";
 import { $ } from "./typekit/index.js";
 
-/** @experimental */
+// #region Types
+
+/**
+ * A description of how a specific kind of type should be mutated.
+ *
+ * This can either be an object specifying an optional `filter` function and one of `mutate` or `replace`, or simply a
+ * function that mutates the type.
+ *
+ * If a function is provided, it is equivalent to providing an object with a `mutate` function and no `filter` function.
+ *
+ * @experimental
+ */
 export type MutatorRecord<T extends Type> =
-  | {
-      filter?: MutatorFilterFn<T>;
-      mutate: MutatorFn<T>;
-    }
-  | {
-      filter?: MutatorFilterFn<T>;
-      replace: MutatorReplaceFn<T>;
-    }
+  | MutatorReplaceRecord<T>
+  | MutatorMutateRecord<T>
   | MutatorFn<T>;
 
-/** @experimental */
-export interface MutatorFn<T extends Type> {
-  (sourceType: T, clone: T, program: Program, realm: Realm): void;
+/**
+ * Common functionality for mutator records.
+ *
+ * @experimental
+ */
+export interface MutatorRecordCommon<T extends Type> {
+  /**
+   * A filter function that determines if the mutator should be applied to the type.
+   */
+  filter?: MutatorFilterFn<T>;
 }
 
-/** @experimental */
-export interface MutatorFilterFn<T extends Type> {
-  (sourceType: T, program: Program, realm: Realm): boolean | MutatorFlow;
+/**
+ * A mutator that replaces a type's clone with a new type instance.
+ *
+ * @experimental
+ */
+export interface MutatorReplaceRecord<T extends Type> extends MutatorRecordCommon<T> {
+  /**
+   * A mutator function that returns a new type instance to replace the cloned type instance.
+   */
+  replace: MutatorReplaceFn<T>;
 }
 
-/** @experimental */
-export interface MutatorReplaceFn<T extends Type> {
-  (sourceType: T, clone: T, program: Program, realm: Realm): Type;
+/**
+ * A mutator that changes the clone of a type in place.
+ *
+ * @experimental
+ */
+export interface MutatorMutateRecord<T extends Type> extends MutatorRecordCommon<T> {
+  /**
+   * A mutator function that edits the clone of the type in place.
+   */
+  mutate: MutatorFn<T>;
 }
 
-/** @experimental */
-export interface Mutator {
+/**
+ * Edits the clone of the type in place. This function _SHOULD NOT_ modify the source type.
+ *
+ * @see {@link mutateSubgraph}
+ *
+ * @param sourceType - The source type.
+ * @param clone - The clone of the source type to mutate.
+ * @param program - The program in which the `sourceType` occurs.
+ * @param realm - The realm in which the `clone` resides.
+ *
+ * @experimental
+ */
+export type MutatorFn<T extends Type> = (
+  sourceType: T,
+  clone: T,
+  program: Program,
+  realm: Realm,
+) => void;
+
+/**
+ * Determines if the mutator should be applied to the type.
+ *
+ * This function may either return a boolean or {@link MutatorFlow} flags:
+ *
+ * - If `true`, the mutator will be applied and will recur (equivalent to `MutatorFlags.MutateAndRecur`).
+ * - If `false`, the mutator will not be applied and will recur (equivalent to `MutatorFlags.DoNotMutate`).
+ *
+ * This predicate runs before the type is cloned.
+ *
+ * @param sourceType - The source type.
+ * @param program - The program in which the `sourceType` occurs.
+ * @param realm - The realm where the `sourceType` will be cloned, if this type is mutated.
+ *
+ * @returns a boolean or {@link MutatorFlow} flags.
+ *
+ * @experimental
+ */
+export type MutatorFilterFn<T extends Type> = (
+  sourceType: T,
+  program: Program,
+  realm: Realm,
+) => boolean | MutatorFlow;
+
+/**
+ * A function that replaces a mutable type with a new type instance.
+ *
+ * Returning `clone` from this function is equivalent to providing a `mutate` function instead of a `replace` function.
+ *
+ * This function runs after the `sourceType` is cloned within the realm.
+ *
+ * @param sourceType - The source type.
+ * @param clone - The clone of the source type to mutate.
+ * @param program - The program in which the `sourceType` occurs.
+ * @param realm - The realm in which the `clone` resides.
+ *
+ * @returns a new type instance to replace the cloned type instance.
+ *
+ * @experimental
+ */
+export type MutatorReplaceFn<T extends Type> = (
+  sourceType: T,
+  clone: T,
+  program: Program,
+  realm: Realm,
+) => Type;
+
+/**
+ * Mutators describe procedures for mutating types in the type graph.
+ *
+ * Each entry in the mutator describes how to mutate a specific type of node.
+ *
+ * See {@link mutateSubgraph}.
+ *
+ * @experimental
+ */
+export type Mutator = {
+  /**
+   * The name of this mutator.
+   */
   name: string;
-  Model?: MutatorRecord<Model>;
-  ModelProperty?: MutatorRecord<ModelProperty>;
-  Scalar?: MutatorRecord<Scalar>;
-  Enum?: MutatorRecord<Enum>;
-  EnumMember?: MutatorRecord<EnumMember>;
-  Union?: MutatorRecord<Union>;
-  UnionVariant?: MutatorRecord<UnionVariant>;
-  Tuple?: MutatorRecord<Tuple>;
-  Operation?: MutatorRecord<Operation>;
-  Interface?: MutatorRecord<Interface>;
-  String?: MutatorRecord<Scalar>;
-  Number?: MutatorRecord<Scalar>;
-  Boolean?: MutatorRecord<Scalar>;
-  ScalarConstructor?: MutatorRecord<ScalarConstructor>;
-  StringTemplate?: MutatorRecord<StringTemplate>;
-  StringTemplateSpan?: MutatorRecord<StringTemplateSpan>;
-}
+} & {
+  /**
+   * Describes how to mutate a type with the given node kind.
+   */
+  [Kind in MutableType["kind"]]?: MutatorRecord<Extract<MutableType, { kind: Kind }>>;
+};
 
 /**
- * @experimental - This is a type that extends Mutator with a Namespace property.
+ * A mutator that can additionally mutate namespaces.
+ *
+ * @experimental
  */
 export type MutatorWithNamespace = Mutator & {
   Namespace: MutatorRecord<Namespace>;
 };
 
-/** @experimental */
+/**
+ * Flow control for mutators.
+ *
+ * When filtering types in a mutator, the filter function may return MutatorFlow flags to control how mutation should
+ * proceed.
+ *
+ * @see {@link MutatorFilterFn}
+ *
+ * @experimental
+ */
 export enum MutatorFlow {
-  MutateAndRecurse = 0,
+  /**
+   * Mutate the type and recur, further mutating the type's children. This is the default behavior.
+   */
+  MutateAndRecur = 0,
+  /**
+   * If this flag is set, the type will not be mutated.
+   */
   DoNotMutate = 1 << 0,
-  DoNotRecurse = 1 << 1,
+  /**
+   * If this flag is set, the mutator will not proceed recursively into the children of the type.
+   */
+  DoNotRecur = 1 << 1,
 }
 
-/** @experimental */
+/**
+ * A type that can be mutated.
+ *
+ * @see {@link mutateSubgraph}
+ *
+ * @experimental
+ */
 export type MutableType = Exclude<
   Type,
   | TemplateParameter
@@ -104,6 +209,8 @@ export type MutableType = Exclude<
 
 /**
  * Determines if a type is mutable.
+ *
+ * @experimental
  */
 export function isMutableType(type: Type): type is MutableType {
   switch (type.kind) {
@@ -114,14 +221,25 @@ export function isMutableType(type: Type): type is MutableType {
     case "FunctionParameter":
     case "Object":
     case "Projection":
+    case "Namespace":
       return false;
     default:
+      void (type satisfies MutableType);
       return true;
   }
 }
 
-/** @experimental */
+/**
+ * A mutable type, inclusive of namespaces.
+ *
+ * @experimental
+ */
 export type MutableTypeWithNamespace = MutableType | Namespace;
+
+// #endregion
+
+// #region Mutator Application
+
 const typeId = CustomKeyMap.objectKeyer();
 const mutatorId = CustomKeyMap.objectKeyer();
 const seen = new CustomKeyMap<[MutableType, Set<Mutator> | Mutator[]], Type>(([type, mutators]) => {
@@ -132,10 +250,19 @@ const seen = new CustomKeyMap<[MutableType, Set<Mutator> | Mutator[]], Type>(([t
 });
 
 /**
- * Mutate the type graph with some namespace mutation.
- * **Warning** this will most likely end up mutating the entire TypeGraph
- * as every type relate to namespace in some way or another
- * causing parent navigation which in turn would mutate everything in that namespace.
+ * Mutate the type graph, allowing namespaces to be mutated.
+ *
+ * **Warning**: This function will likely mutate the entire type graph. Most TypeSpec types relate to namespaces
+ * in some way (e.g. through namespace parent links, or the `namespace` property of a Model).
+ *
+ * @param program - The program in which the `type` occurs.
+ * @param mutators - An array of mutators to apply to the type graph rooted at `type`.
+ * @param type - The type to mutate.
+ *
+ * @returns an object containing the mutated `type` and a nullable `Realm` in which the mutated type resides.
+ *
+ * @see {@link mutateSubgraph}
+ *
  * @experimental
  */
 export function mutateSubgraphWithNamespace<T extends MutableTypeWithNamespace>(
@@ -146,7 +273,31 @@ export function mutateSubgraphWithNamespace<T extends MutableTypeWithNamespace>(
   return mutateSubgraph(program, mutators, type as any);
 }
 
-/** @experimental */
+/**
+ * Mutate the type graph.
+ *
+ * Mutators clone the input `type`, creating a new type instance that is mutated in place.
+ *
+ * The mutator returns the mutated type and optionally a `realm` in which the mutated clone resides.
+ *
+ * @see {@link Mutator}
+ * @see {@link Realm}
+ *
+ * **Warning**: Mutators _SHOULD NOT_ modify the source type. Modifications to the source type
+ * will be visible to other emitters or libraries that view the original source type, and will
+ * be sensitive to the order in which the mutator was applied. Only edit the `clone` type.
+ * Furthermore, mutators must take care not to modify elements of the source and clone types
+ * that are shared between the two types, such as the properties of any parent references
+ * or the `decorators` of the type without taking care to clone them first.
+ *
+ * @param program - The program in which the `type` occurs.
+ * @param mutators - An array of mutators to apply to the type graph rooted at `type`.
+ * @param type - The type to mutate.
+ *
+ * @returns an object containing the mutated `type` and a nullable `Realm` in which the mutated type resides.
+ *
+ * @experimental
+ */
 export function mutateSubgraph<T extends MutableType>(
   program: Program,
   mutators: Mutator[],
@@ -212,7 +363,7 @@ export function mutateSubgraph<T extends MutableType>(
             recurse = true;
           } else {
             mutate = (filterResult & MutatorFlow.DoNotMutate) === 0;
-            recurse = (filterResult & MutatorFlow.DoNotRecurse) === 0;
+            recurse = (filterResult & MutatorFlow.DoNotRecur) === 0;
           }
         } else {
           mutate = true;
@@ -276,12 +427,12 @@ export function mutateSubgraph<T extends MutableType>(
       visitSubgraph();
     }
 
-    $.type.finishType(clone!);
+    $(realm).type.finishType(clone!);
 
     return clone!;
 
     function initializeClone() {
-      clone = $.type.clone(type);
+      clone = $(realm).type.clone(type);
       seen.set([type, activeMutators], clone);
       seen.set([type, mutatorsToApply], clone);
     }
@@ -337,3 +488,5 @@ export function mutateSubgraph<T extends MutableType>(
     }
   }
 }
+
+// #endregion