diff --git a/.chronus/changes/azhang_RemoveAzureResource-2024-6-3-12-45-15.md b/.chronus/changes/azhang_RemoveAzureResource-2024-6-3-12-45-15.md
new file mode 100644
index 0000000000..e1855108c4
--- /dev/null
+++ b/.chronus/changes/azhang_RemoveAzureResource-2024-6-3-12-45-15.md
@@ -0,0 +1,8 @@
+---
+changeKind: feature
+packages:
+  - "@azure-tools/typespec-autorest"
+  - "@azure-tools/typespec-azure-resource-manager"
+---
+
+Removed direct reference to OpenAPI extension `x-ms-azure-resource` in ARM library and replaced with `@Azure.ResourceManager.Private.azureResourceBase` decorator. It is only used internally on base resource types. `autorest` emitter has been updated to check the decorator and still emit `x-ms-azure-resource` extension in swagger. 
diff --git a/packages/samples/common-types/src/types.tsp b/packages/samples/common-types/src/types.tsp
index 853ab46349..a5f7a68350 100644
--- a/packages/samples/common-types/src/types.tsp
+++ b/packages/samples/common-types/src/types.tsp
@@ -20,3 +20,6 @@ op registerParams(
 ): void;
 
 @@extension(ErrorDetail.details, "x-ms-identifiers", ["message", "target"]);
+
+@@extension(Resource, "x-ms-azure-resource", true);
+@@extension(ResourceModelWithAllowedPropertySet, "x-ms-azure-resource", true);
diff --git a/packages/typespec-autorest/src/openapi.ts b/packages/typespec-autorest/src/openapi.ts
index df257b3d38..52c1d06557 100644
--- a/packages/typespec-autorest/src/openapi.ts
+++ b/packages/typespec-autorest/src/openapi.ts
@@ -13,6 +13,7 @@ import {
 import {
   getArmCommonTypeOpenAPIRef,
   isArmCommonType,
+  isAzureResource,
 } from "@azure-tools/typespec-azure-resource-manager";
 import { shouldFlattenProperty } from "@azure-tools/typespec-client-generator-core";
 import {
@@ -1854,6 +1855,9 @@ export async function getOpenAPIForService(
   function attachExtensions(type: Type, emitObject: any) {
     // Attach any OpenAPI extensions
     const extensions = getExtensions(program, type);
+    if (isAzureResource(program, type as Model)) {
+      emitObject["x-ms-azure-resource"] = true;
+    }
     if (getAsEmbeddingVector(program, type as Model) !== undefined) {
       emitObject["x-ms-embedding-vector"] = true;
     }
diff --git a/packages/typespec-autorest/test/arm/resources.test.ts b/packages/typespec-autorest/test/arm/resources.test.ts
index e71ade9be9..78783de801 100644
--- a/packages/typespec-autorest/test/arm/resources.test.ts
+++ b/packages/typespec-autorest/test/arm/resources.test.ts
@@ -307,3 +307,18 @@ it("emits a scalar string with decorator parameter for resource", async () => {
     type: "string",
   });
 });
+
+it("emits x-ms-azure-resource for resource with @azureResourceBase", async () => {
+  const openApi = await openApiFor(`
+    @armProviderNamespace
+    @useDependency(Azure.ResourceManager.Versions.v1_0_Preview_1)
+    namespace Microsoft.Contoso;
+
+    @doc("Widget resource")
+    @Azure.ResourceManager.Private.azureResourceBase
+    model Widget {
+       name: string;
+    }
+`);
+  ok(openApi.definitions.Widget["x-ms-azure-resource"]);
+});
diff --git a/packages/typespec-azure-resource-manager/CHANGELOG.md b/packages/typespec-azure-resource-manager/CHANGELOG.md
index fa32d04822..35e4ac2d81 100644
--- a/packages/typespec-azure-resource-manager/CHANGELOG.md
+++ b/packages/typespec-azure-resource-manager/CHANGELOG.md
@@ -19,14 +19,12 @@
 - [#811](https://github.com/Azure/typespec-azure/pull/811) Remove dependency on `typespec-autorest` emitter
 - [#432](https://github.com/Azure/typespec-azure/pull/432) Add support for values
 
-
 ## 0.42.1
 
 ### Bug Fixes
 
 - [#868](https://github.com/Azure/typespec-azure/pull/868) Changing back `ManagedServiceIdentity.userAssignedIdentities` back to `Record<UserAssignedIdentity`. Adding ARM common-type references for on all Managed Identity models.
 
-
 ## 0.42.0
 
 ### Bug Fixes
@@ -50,7 +48,7 @@
 ### Deprecations
 
 - [#762](https://github.com/Azure/typespec-azure/pull/762) Renaming internal TypeSpec ARM foundation model names to be consistent with ARM common-type definitions.
-However, these are `Azure.Resource.Manager.Foundations` models that would not normally be used directly in service specs.
+  However, these are `Azure.Resource.Manager.Foundations` models that would not normally be used directly in service specs.
 
 - Deprecate `Foundations.ArmResource`. `Foundations.Resource` should be used instead.
 
@@ -68,6 +66,7 @@ However, these are `Azure.Resource.Manager.Foundations` models that would not no
 - Deprecate `ManagedServiceIdentity`. `ManagedServiceIdentityProperty` should be used instead.
 
   Example:
+
   ```diff
   -...ManagedServiceIdentity;
   +...ManagedServiceIdentityProperty;
@@ -76,6 +75,7 @@ However, these are `Azure.Resource.Manager.Foundations` models that would not no
 - Deprecate `ManagedSystemAssignedIdentity`. `ManagedSystemAssignedIdentityProperty` should be used instead.
 
   Example:
+
   ```diff
   -...ManagedSystemAssignedIdentity;
   +...ManagedSystemAssignedIdentityProperty;
@@ -84,6 +84,7 @@ However, these are `Azure.Resource.Manager.Foundations` models that would not no
 - Deprecate `EntityTag`. `EntityTagProperty` should be used instead.
 
   Example:
+
   ```diff
   -...EntityTag;
   +...EntityTagProperty;
@@ -92,6 +93,7 @@ However, these are `Azure.Resource.Manager.Foundations` models that would not no
 - Deprecate `ResourceKind`. `ResourceKindProperty` should be used instead.
 
   Example:
+
   ```diff
   -...ResourceKind;
   +...ResourceKindProperty;
@@ -100,6 +102,7 @@ However, these are `Azure.Resource.Manager.Foundations` models that would not no
 - Deprecate `ResourcePlan`. `ResourcePlanProperty` should be used instead.
 
   Example:
+
   ```diff
   -...ResourcePlan;
   +...ResourcePlanProperty;
@@ -108,6 +111,7 @@ However, these are `Azure.Resource.Manager.Foundations` models that would not no
 - Deprecate `ResourceSku`. `ResourceSkuProperty` should be used instead.
 
   Example:
+
   ```diff
   -...ResourceSku;
   +...ResourceSkuProperty;
@@ -116,6 +120,7 @@ However, these are `Azure.Resource.Manager.Foundations` models that would not no
 - Deprecate `ManagedBy`. `ManagedByProperty` should be used instead.
 
   Example:
+
   ```diff
   -...ManagedBy;
   +...ManagedByProperty;
@@ -125,7 +130,6 @@ However, these are `Azure.Resource.Manager.Foundations` models that would not no
 
 - [#521](https://github.com/Azure/typespec-azure/pull/521) Switching ProvisioningState from enum to Open union
 
-
 ## 0.41.0
 
 ### Bug Fixes
@@ -148,7 +152,6 @@ However, these are `Azure.Resource.Manager.Foundations` models that would not no
 
 - [#407](https://github.com/Azure/typespec-azure/pull/407) Deprecate `ResourceIdentifier` in favor of new `Azure.Core.armResourceIdentifier`
 
-
 ## 0.40.0
 
 ### Bug Fixes
@@ -167,7 +170,6 @@ However, these are `Azure.Resource.Manager.Foundations` models that would not no
 - [#298](https://github.com/Azure/typespec-azure/pull/298) ARM: add `arm-delete-response-codes` rule.
 - [#317](https://github.com/Azure/typespec-azure/pull/317) Add ruleset `canonical-versioning` to Azure Resource Manager library.
 
-
 ## 0.39.1
 
 ### Patch Changes
@@ -181,7 +183,6 @@ However, these are `Azure.Resource.Manager.Foundations` models that would not no
 - ebfe639: Fix Location resource issue, add mechanism for additional path segments
 - 148eee4: Update references to ARM, Add template customization parameters, add migration docs
 
-
 ## 0.38.0
 
 Wed, 24 Jan 2024 05:47:19 GMT
diff --git a/packages/typespec-azure-resource-manager/lib/common-types/types-ref.tsp b/packages/typespec-azure-resource-manager/lib/common-types/types-ref.tsp
index 09cadfb7ba..eb04399e3d 100644
--- a/packages/typespec-azure-resource-manager/lib/common-types/types-ref.tsp
+++ b/packages/typespec-azure-resource-manager/lib/common-types/types-ref.tsp
@@ -7,6 +7,9 @@ using Azure.ResourceManager.CommonTypes.Private;
 
 namespace Azure.ResourceManager.CommonTypes;
 
+@@azureResourceBase(Resource);
+@@azureResourceBase(ResourceModelWithAllowedPropertySet);
+
 /** Common resource fields that are returned in the response for all Azure Resource Manager resources. */
 @@armCommonDefinition(Resource, "Resource", Azure.ResourceManager.CommonTypes.Versions.v3);
 @@armCommonDefinition(Resource, "Resource", Azure.ResourceManager.CommonTypes.Versions.v4);
diff --git a/packages/typespec-azure-resource-manager/lib/common-types/types.tsp b/packages/typespec-azure-resource-manager/lib/common-types/types.tsp
index 6889a3ac3f..109a64091d 100644
--- a/packages/typespec-azure-resource-manager/lib/common-types/types.tsp
+++ b/packages/typespec-azure-resource-manager/lib/common-types/types.tsp
@@ -7,7 +7,6 @@ using Azure.Core;
 namespace Azure.ResourceManager.CommonTypes;
 
 /** Common fields that are returned in the response for all Azure Resource Manager resources */
-@extension("x-ms-azure-resource", true)
 model Resource {
   /** Fully qualified resource ID for the resource. Ex - /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName} */
   @typeChangedFrom(Versions.v4, string)
@@ -60,7 +59,6 @@ model ExtensionResource extends Resource {}
 /**
  * The resource model definition containing the full set of allowed properties for a resource. Except properties bag, there cannot be a top level property outside of this set.
  */
-@extension("x-ms-azure-resource", true)
 model ResourceModelWithAllowedPropertySet extends TrackedResource {
   /** The fully qualified resource ID of the resource that manages this resource. Indicates if this resource is managed by another Azure resource.
    * If this is present, complete mode deployment will not delete the resource if it is removed from the template since it is managed by another resource. */
diff --git a/packages/typespec-azure-resource-manager/lib/private.decorators.tsp b/packages/typespec-azure-resource-manager/lib/private.decorators.tsp
index 7993ea6bbb..036396a14c 100644
--- a/packages/typespec-azure-resource-manager/lib/private.decorators.tsp
+++ b/packages/typespec-azure-resource-manager/lib/private.decorators.tsp
@@ -66,3 +66,11 @@ extern dec defaultResourceKeySegmentName(
  * Note, this is intended for internal use only for now.
  */
 extern dec enforceConstraint(target: Operation | Model, sourceType: Model, constraintType: Model);
+
+/**
+ * This decorator is used to identify Azure Resource Manager resource. In generated
+ * swagger definition, it will be marked with `x-ms-azure-resource`.
+ *
+ * It is *not* meant to be used directly by a spec author,
+ */
+extern dec azureResourceBase(target: Model);
diff --git a/packages/typespec-azure-resource-manager/src/index.ts b/packages/typespec-azure-resource-manager/src/index.ts
index 1109e9bf1d..7d458b5e47 100644
--- a/packages/typespec-azure-resource-manager/src/index.ts
+++ b/packages/typespec-azure-resource-manager/src/index.ts
@@ -12,6 +12,8 @@ export {
   type ArmCommonTypeVersions,
 } from "./common-types.js";
 
+export { isAzureResource } from "./private.decorators.js";
+
 export * from "./namespace.js";
 export * from "./operations.js";
 export * from "./resource.js";
diff --git a/packages/typespec-azure-resource-manager/src/private.decorators.ts b/packages/typespec-azure-resource-manager/src/private.decorators.ts
index 39afe7bc27..c4baaa398f 100644
--- a/packages/typespec-azure-resource-manager/src/private.decorators.ts
+++ b/packages/typespec-azure-resource-manager/src/private.decorators.ts
@@ -350,3 +350,12 @@ function hasProperty(program: Program, model: Model): boolean {
   if (model.baseModel) return hasProperty(program, model.baseModel);
   return false;
 }
+
+export function $azureResourceBase(context: DecoratorContext, resourceType: Model) {
+  context.program.stateMap(ArmStateKeys.azureResourceBase).set(resourceType, true);
+}
+
+export function isAzureResource(program: Program, resourceType: Model): boolean {
+  const isResourceBase = program.stateMap(ArmStateKeys.azureResourceBase).get(resourceType);
+  return isResourceBase ?? false;
+}
diff --git a/packages/typespec-azure-resource-manager/src/state.ts b/packages/typespec-azure-resource-manager/src/state.ts
index c26affb22b..e8d5d68922 100644
--- a/packages/typespec-azure-resource-manager/src/state.ts
+++ b/packages/typespec-azure-resource-manager/src/state.ts
@@ -22,6 +22,7 @@ export const ArmStateKeys = {
   armBuiltInResource: azureResourceManagerCreateStateSymbol("armExternalResource"),
 
   // private.decorator.ts
+  azureResourceBase: azureResourceManagerCreateStateSymbol("azureResourceBase"),
   armCommonDefinitions: azureResourceManagerCreateStateSymbol("armCommonDefinitions"),
   armCommonParameters: azureResourceManagerCreateStateSymbol("armCommonParameters"),
   armCommonTypesVersions: azureResourceManagerCreateStateSymbol("armCommonTypesVersions"),