From 90165cb758df215a932c8c2c916d629d5e404130 Mon Sep 17 00:00:00 2001 From: fundale <69996669+fundale@users.noreply.github.com> Date: Fri, 6 Sep 2024 23:57:07 -0500 Subject: [PATCH 1/2] Add AudioReactiveBlendshapes.cs and related files A small utility script based off `AudioReactiveLight.cs` for blendshapes --- .../Scripts/AudioReactiveBlendshapes.asset | 532 ++++++++++++++++++ .../AudioReactiveBlendshapes.asset.meta | 8 + .../Scripts/AudioReactiveBlendshapes.cs | 47 ++ .../Scripts/AudioReactiveBlendshapes.cs.meta | 11 + 4 files changed, 598 insertions(+) create mode 100644 Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.asset create mode 100644 Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.asset.meta create mode 100644 Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.cs create mode 100644 Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.cs.meta diff --git a/Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.asset b/Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.asset new file mode 100644 index 00000000..e9acec8f --- /dev/null +++ b/Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.asset @@ -0,0 +1,532 @@ +%YAML 1.1 +%TAG !u! tag:unity3d.com,2011: +--- !u!114 &11400000 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: c333ccfdd0cbdbc4ca30cef2dd6e6b9b, type: 3} + m_Name: AudioReactiveBlendshapes + m_EditorClassIdentifier: + serializedUdonProgramAsset: {fileID: 11400000, guid: 7107e0ae49ffa160ea90396c83adf8bd, type: 2} + udonAssembly: + assemblyError: + sourceCsScript: {fileID: 11500000, guid: 3b1feb6f66f3cdbe4aa71d91a7d6dd7b, type: 3} + scriptVersion: 2 + compiledVersion: 2 + behaviourSyncMode: 0 + hasInteractEvent: 0 + scriptID: 5021526840855900124 + serializationData: + SerializedFormat: 2 + SerializedBytes: + ReferencedUnityObjects: [] + SerializedBytesString: + Prefab: {fileID: 0} + PrefabModificationsReferencedUnityObjects: [] + PrefabModifications: [] + SerializationNodes: + - Name: fieldDefinitions + Entry: 7 + Data: 0|System.Collections.Generic.Dictionary`2[[System.String, mscorlib],[UdonSharp.Compiler.FieldDefinition, + UdonSharp.Editor]], mscorlib + - Name: comparer + Entry: 7 + Data: 1|System.Collections.Generic.GenericEqualityComparer`1[[System.String, + mscorlib]], mscorlib + - Name: + Entry: 8 + Data: + - Name: + Entry: 12 + Data: 9 + - Name: + Entry: 7 + Data: + - Name: $k + Entry: 1 + Data: audioLink + - Name: $v + Entry: 7 + Data: 2|UdonSharp.Compiler.FieldDefinition, UdonSharp.Editor + - Name: k__BackingField + Entry: 1 + Data: audioLink + - Name: k__BackingField + Entry: 7 + Data: 3|System.RuntimeType, mscorlib + - Name: + Entry: 1 + Data: AudioLink.AudioLink, AudioLink + - Name: + Entry: 8 + Data: + - Name: k__BackingField + Entry: 7 + Data: 4|System.RuntimeType, mscorlib + - Name: + Entry: 1 + Data: VRC.Udon.UdonBehaviour, VRC.Udon + - Name: + Entry: 8 + Data: + - Name: k__BackingField + Entry: 7 + Data: System.Nullable`1[[UdonSharp.UdonSyncMode, UdonSharp.Runtime]], mscorlib + - Name: + Entry: 6 + Data: + - Name: + Entry: 8 + Data: + - Name: k__BackingField + Entry: 5 + Data: true + - Name: _fieldAttributes + Entry: 7 + Data: 5|System.Collections.Generic.List`1[[System.Attribute, mscorlib]], mscorlib + - Name: + Entry: 12 + Data: 0 + - Name: + Entry: 13 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 7 + Data: + - Name: $k + Entry: 1 + Data: band + - Name: $v + Entry: 7 + Data: 6|UdonSharp.Compiler.FieldDefinition, UdonSharp.Editor + - Name: k__BackingField + Entry: 1 + Data: band + - Name: k__BackingField + Entry: 7 + Data: 7|System.RuntimeType, mscorlib + - Name: + Entry: 1 + Data: System.Int32, mscorlib + - Name: + Entry: 8 + Data: + - Name: k__BackingField + Entry: 9 + Data: 7 + - Name: k__BackingField + Entry: 7 + Data: System.Nullable`1[[UdonSharp.UdonSyncMode, UdonSharp.Runtime]], mscorlib + - Name: + Entry: 6 + Data: + - Name: + Entry: 8 + Data: + - Name: k__BackingField + Entry: 5 + Data: true + - Name: _fieldAttributes + Entry: 7 + Data: 8|System.Collections.Generic.List`1[[System.Attribute, mscorlib]], mscorlib + - Name: + Entry: 12 + Data: 0 + - Name: + Entry: 13 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 7 + Data: + - Name: $k + Entry: 1 + Data: delay + - Name: $v + Entry: 7 + Data: 9|UdonSharp.Compiler.FieldDefinition, UdonSharp.Editor + - Name: k__BackingField + Entry: 1 + Data: delay + - Name: k__BackingField + Entry: 9 + Data: 7 + - Name: k__BackingField + Entry: 9 + Data: 7 + - Name: k__BackingField + Entry: 7 + Data: System.Nullable`1[[UdonSharp.UdonSyncMode, UdonSharp.Runtime]], mscorlib + - Name: + Entry: 6 + Data: + - Name: + Entry: 8 + Data: + - Name: k__BackingField + Entry: 5 + Data: true + - Name: _fieldAttributes + Entry: 7 + Data: 10|System.Collections.Generic.List`1[[System.Attribute, mscorlib]], mscorlib + - Name: + Entry: 12 + Data: 1 + - Name: + Entry: 7 + Data: 11|UnityEngine.RangeAttribute, UnityEngine.CoreModule + - Name: min + Entry: 4 + Data: 0 + - Name: max + Entry: 4 + Data: 127 + - Name: + Entry: 8 + Data: + - Name: + Entry: 13 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 7 + Data: + - Name: $k + Entry: 1 + Data: blendshapeIDs + - Name: $v + Entry: 7 + Data: 12|UdonSharp.Compiler.FieldDefinition, UdonSharp.Editor + - Name: k__BackingField + Entry: 1 + Data: blendshapeIDs + - Name: k__BackingField + Entry: 7 + Data: 13|System.RuntimeType, mscorlib + - Name: + Entry: 1 + Data: System.Int32[], mscorlib + - Name: + Entry: 8 + Data: + - Name: k__BackingField + Entry: 9 + Data: 13 + - Name: k__BackingField + Entry: 7 + Data: System.Nullable`1[[UdonSharp.UdonSyncMode, UdonSharp.Runtime]], mscorlib + - Name: + Entry: 6 + Data: + - Name: + Entry: 8 + Data: + - Name: k__BackingField + Entry: 5 + Data: true + - Name: _fieldAttributes + Entry: 7 + Data: 14|System.Collections.Generic.List`1[[System.Attribute, mscorlib]], mscorlib + - Name: + Entry: 12 + Data: 0 + - Name: + Entry: 13 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 7 + Data: + - Name: $k + Entry: 1 + Data: blendshapeWeights + - Name: $v + Entry: 7 + Data: 15|UdonSharp.Compiler.FieldDefinition, UdonSharp.Editor + - Name: k__BackingField + Entry: 1 + Data: blendshapeWeights + - Name: k__BackingField + Entry: 7 + Data: 16|System.RuntimeType, mscorlib + - Name: + Entry: 1 + Data: System.Single[], mscorlib + - Name: + Entry: 8 + Data: + - Name: k__BackingField + Entry: 9 + Data: 16 + - Name: k__BackingField + Entry: 7 + Data: System.Nullable`1[[UdonSharp.UdonSyncMode, UdonSharp.Runtime]], mscorlib + - Name: + Entry: 6 + Data: + - Name: + Entry: 8 + Data: + - Name: k__BackingField + Entry: 5 + Data: true + - Name: _fieldAttributes + Entry: 7 + Data: 17|System.Collections.Generic.List`1[[System.Attribute, mscorlib]], mscorlib + - Name: + Entry: 12 + Data: 0 + - Name: + Entry: 13 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 7 + Data: + - Name: $k + Entry: 1 + Data: blendshapeMinWeights + - Name: $v + Entry: 7 + Data: 18|UdonSharp.Compiler.FieldDefinition, UdonSharp.Editor + - Name: k__BackingField + Entry: 1 + Data: blendshapeMinWeights + - Name: k__BackingField + Entry: 9 + Data: 16 + - Name: k__BackingField + Entry: 9 + Data: 16 + - Name: k__BackingField + Entry: 7 + Data: System.Nullable`1[[UdonSharp.UdonSyncMode, UdonSharp.Runtime]], mscorlib + - Name: + Entry: 6 + Data: + - Name: + Entry: 8 + Data: + - Name: k__BackingField + Entry: 5 + Data: true + - Name: _fieldAttributes + Entry: 7 + Data: 19|System.Collections.Generic.List`1[[System.Attribute, mscorlib]], mscorlib + - Name: + Entry: 12 + Data: 0 + - Name: + Entry: 13 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 7 + Data: + - Name: $k + Entry: 1 + Data: _skinnedMeshRenderer + - Name: $v + Entry: 7 + Data: 20|UdonSharp.Compiler.FieldDefinition, UdonSharp.Editor + - Name: k__BackingField + Entry: 1 + Data: _skinnedMeshRenderer + - Name: k__BackingField + Entry: 7 + Data: 21|System.RuntimeType, mscorlib + - Name: + Entry: 1 + Data: UnityEngine.SkinnedMeshRenderer, UnityEngine.CoreModule + - Name: + Entry: 8 + Data: + - Name: k__BackingField + Entry: 9 + Data: 21 + - Name: k__BackingField + Entry: 7 + Data: System.Nullable`1[[UdonSharp.UdonSyncMode, UdonSharp.Runtime]], mscorlib + - Name: + Entry: 6 + Data: + - Name: + Entry: 8 + Data: + - Name: k__BackingField + Entry: 5 + Data: false + - Name: _fieldAttributes + Entry: 7 + Data: 22|System.Collections.Generic.List`1[[System.Attribute, mscorlib]], mscorlib + - Name: + Entry: 12 + Data: 0 + - Name: + Entry: 13 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 7 + Data: + - Name: $k + Entry: 1 + Data: _dataIndex + - Name: $v + Entry: 7 + Data: 23|UdonSharp.Compiler.FieldDefinition, UdonSharp.Editor + - Name: k__BackingField + Entry: 1 + Data: _dataIndex + - Name: k__BackingField + Entry: 9 + Data: 7 + - Name: k__BackingField + Entry: 9 + Data: 7 + - Name: k__BackingField + Entry: 7 + Data: System.Nullable`1[[UdonSharp.UdonSyncMode, UdonSharp.Runtime]], mscorlib + - Name: + Entry: 6 + Data: + - Name: + Entry: 8 + Data: + - Name: k__BackingField + Entry: 5 + Data: false + - Name: _fieldAttributes + Entry: 7 + Data: 24|System.Collections.Generic.List`1[[System.Attribute, mscorlib]], mscorlib + - Name: + Entry: 12 + Data: 0 + - Name: + Entry: 13 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 7 + Data: + - Name: $k + Entry: 1 + Data: _maxBlendshapes + - Name: $v + Entry: 7 + Data: 25|UdonSharp.Compiler.FieldDefinition, UdonSharp.Editor + - Name: k__BackingField + Entry: 1 + Data: _maxBlendshapes + - Name: k__BackingField + Entry: 9 + Data: 7 + - Name: k__BackingField + Entry: 9 + Data: 7 + - Name: k__BackingField + Entry: 7 + Data: System.Nullable`1[[UdonSharp.UdonSyncMode, UdonSharp.Runtime]], mscorlib + - Name: + Entry: 6 + Data: + - Name: + Entry: 8 + Data: + - Name: k__BackingField + Entry: 5 + Data: false + - Name: _fieldAttributes + Entry: 7 + Data: 26|System.Collections.Generic.List`1[[System.Attribute, mscorlib]], mscorlib + - Name: + Entry: 12 + Data: 0 + - Name: + Entry: 13 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 8 + Data: + - Name: + Entry: 13 + Data: + - Name: + Entry: 8 + Data: diff --git a/Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.asset.meta b/Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.asset.meta new file mode 100644 index 00000000..e3e552e0 --- /dev/null +++ b/Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.asset.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 420c34b4ed4823a52bfa768e4fd9a5d0 +NativeFormatImporter: + externalObjects: {} + mainObjectFileID: 11400000 + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.cs b/Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.cs new file mode 100644 index 00000000..5e14edab --- /dev/null +++ b/Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.cs @@ -0,0 +1,47 @@ +using UnityEngine; + +namespace AudioLink +{ +#if UDONSHARP + using UdonSharp; + using UnityEngine.Rendering.PostProcessing; + + public class AudioReactiveBlendshapes : UdonSharpBehaviour +#else + public class AudioReactiveBlendshapes : MonoBehaviour +#endif + { + public AudioLink audioLink; + public int band; + [Range(0, 127)] + public int delay; + public int[] blendshapeIDs; + public float[] blendshapeWeights; + public float[] blendshapeMinWeights; + + private SkinnedMeshRenderer _skinnedMeshRenderer; + private int _dataIndex; + private int _maxBlendshapes; + + void Start() + { + _skinnedMeshRenderer = transform.GetComponent(); + _dataIndex = (band * 128) + delay; + int[] maxBlendshapes = new int[3] { blendshapeIDs.Length, blendshapeWeights.Length, blendshapeMinWeights.Length }; + _maxBlendshapes = Mathf.Min(maxBlendshapes); + + } + + void Update() + { + if (audioLink.AudioDataIsAvailable()) + { + // Convert to grayscale + float amplitude = Vector3.Dot(audioLink.GetDataAtPixel(delay, band), new Vector3(0.299f, 0.587f, 0.114f)) * 100f; + for (int indx = 0; indx < _maxBlendshapes; indx++) + _skinnedMeshRenderer.SetBlendShapeWeight(blendshapeIDs[indx], Mathf.LerpUnclamped(blendshapeMinWeights[indx], blendshapeWeights[indx], amplitude)); + } + } + + } +} \ No newline at end of file diff --git a/Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.cs.meta b/Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.cs.meta new file mode 100644 index 00000000..74d5589f --- /dev/null +++ b/Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 3b1feb6f66f3cdbe4aa71d91a7d6dd7b +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: From e1901928c6df945a912e3a630b98c84241eebaf7 Mon Sep 17 00:00:00 2001 From: fundale <69996669+fundale@users.noreply.github.com> Date: Sat, 7 Sep 2024 12:00:01 -0500 Subject: [PATCH 2/2] Update AudioReactiveBlendshapes.cs Require SkinnedMeshRenderer, remove unused _dataIndex, skip Vector.Dot and instead access pixel's red channel --- .../Runtime/Scripts/AudioReactiveBlendshapes.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.cs b/Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.cs index 5e14edab..ddfce43e 100644 --- a/Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.cs +++ b/Packages/com.llealloo.audiolink/Runtime/Scripts/AudioReactiveBlendshapes.cs @@ -6,8 +6,10 @@ namespace AudioLink using UdonSharp; using UnityEngine.Rendering.PostProcessing; + [RequireComponent(typeof(SkinnedMeshRenderer))] public class AudioReactiveBlendshapes : UdonSharpBehaviour #else + [RequireComponent(typeof(SkinnedMeshRenderer))] public class AudioReactiveBlendshapes : MonoBehaviour #endif { @@ -20,13 +22,11 @@ public class AudioReactiveBlendshapes : MonoBehaviour public float[] blendshapeMinWeights; private SkinnedMeshRenderer _skinnedMeshRenderer; - private int _dataIndex; private int _maxBlendshapes; void Start() { _skinnedMeshRenderer = transform.GetComponent(); - _dataIndex = (band * 128) + delay; int[] maxBlendshapes = new int[3] { blendshapeIDs.Length, blendshapeWeights.Length, blendshapeMinWeights.Length }; _maxBlendshapes = Mathf.Min(maxBlendshapes); @@ -36,12 +36,11 @@ void Update() { if (audioLink.AudioDataIsAvailable()) { - // Convert to grayscale - float amplitude = Vector3.Dot(audioLink.GetDataAtPixel(delay, band), new Vector3(0.299f, 0.587f, 0.114f)) * 100f; + float amplitude = audioLink.GetDataAtPixel(delay, band).x * 100f; for (int indx = 0; indx < _maxBlendshapes; indx++) _skinnedMeshRenderer.SetBlendShapeWeight(blendshapeIDs[indx], Mathf.LerpUnclamped(blendshapeMinWeights[indx], blendshapeWeights[indx], amplitude)); } } } -} \ No newline at end of file +}