diff --git a/src/components/Avatar/Avatar.test.tsx b/src/components/Avatar/Avatar.test.tsx index 603ed77c..369633c9 100644 --- a/src/components/Avatar/Avatar.test.tsx +++ b/src/components/Avatar/Avatar.test.tsx @@ -11,6 +11,8 @@ it('renders defualt Avatar (blob) unchanged', () => { {}} instruct={false} avatar3dVisible={true} setAvatar3dVisible={() => {}} @@ -34,6 +36,8 @@ it('renders Avatar with blob and avatar in blob unchanged', () => { }} tenant={tenant} instruct={false} + isTotem={false} + setEnablePositionControls={() => {}} avatar3dVisible={true} setAvatar3dVisible={() => {}} hasUserActivatedSpeak={false} @@ -50,6 +54,8 @@ it('renders Avatar with custom glb model unchanged', () => { {}} integrationConfig={{ ...integrationConfig, avatar: 'customglb', @@ -74,6 +80,8 @@ it('renders Avatar with rpm 3d avatar unchanged', () => { {}} integrationConfig={{ ...integrationConfig, avatar: 'readyplayerme', diff --git a/src/components/Avatar/Avatar.tsx b/src/components/Avatar/Avatar.tsx index 54e9beb8..148c01ba 100644 --- a/src/components/Avatar/Avatar.tsx +++ b/src/components/Avatar/Avatar.tsx @@ -17,6 +17,8 @@ import Edit from '../icons/Edit'; import cx from 'classnames'; import ContainerAvatarView from './AvatarView'; import { useViseme } from '../../context/visemeContext'; +import PositionControls from './AvatarView/AvatarComponent/positionControls/positionControls'; +import { getLocalConfig } from '../../helpers/configuration'; export interface Props { memori: Memori; @@ -34,6 +36,10 @@ export interface Props { animation?: string; isZoomed?: boolean; chatProps?: any; + enablePositionControls?: boolean; + setEnablePositionControls: (value: boolean) => void; + avatarType?: 'blob' | 'avatar3d' | undefined; + isTotem?: boolean; } const Avatar: React.FC = ({ @@ -52,8 +58,11 @@ const Avatar: React.FC = ({ animation, isZoomed = false, chatProps, + avatarType, + enablePositionControls, + setEnablePositionControls, + isTotem = false, }) => { - const { t } = useTranslation(); const [isClient, setIsClient] = useState(false); @@ -87,7 +96,8 @@ const Avatar: React.FC = ({ integrationConfig?.avatar === 'readyplayerme-full' || integrationConfig?.avatar === 'customglb' || integrationConfig?.avatar === 'customrpm') && - integrationConfig?.avatarURL + integrationConfig?.avatarURL && + (avatarType && avatarType !== 'blob') ) { return ( <> @@ -115,11 +125,10 @@ const Avatar: React.FC = ({ const renderAvatarContent = () => { if (!isClient) return null; - if ( integrationConfig?.avatar === 'readyplayerme' || - integrationConfig?.avatar === 'readyplayerme-full' || - integrationConfig?.avatar === 'customrpm' + integrationConfig?.avatar === 'readyplayerme-full' || + integrationConfig?.avatar === 'customrpm' ) { return ( = ({ } > = ({ style={getAvatarStyle()} stopProcessing={stopProcessing} resetVisemeQueue={resetVisemeQueue} - isZoomed={isZoomed} + isZoomed={isZoomed} + isTotem={isTotem} chatEmission={chatProps?.dialogState?.emission} + setEnablePositionControls={setEnablePositionControls} /> ); @@ -183,10 +195,10 @@ const Avatar: React.FC = ({ const getAvatarStyle = () => { if (integrationConfig?.avatar === 'readyplayerme') { return { - width: '300px', - height: '300px', + width: '100%', + height: '100%', backgroundColor: 'none', - borderRadius: '100%', + // borderRadius: '100%', boxShadow: 'none', }; } diff --git a/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx b/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx index 934fe1b8..a5d89650 100644 --- a/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx +++ b/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx @@ -1,7 +1,10 @@ import React, { useState, useEffect, useCallback } from 'react'; import AnimationControlPanel from './components/controls'; -import FullbodyAvatar from './components/fullbodyAvatar'; +import { FullbodyAvatar } from './components/FullbodyAvatar/FullbodyAvatar'; import HalfBodyAvatar from './components/halfbodyAvatar'; +import PositionControls from './positionControls/positionControls'; +import { PerspectiveCamera, Vector3 } from 'three'; +import { getLocalConfig } from '../../../../helpers/configuration'; interface Props { showControls: boolean; @@ -14,11 +17,14 @@ interface Props { speaking: boolean; isZoomed: boolean; chatEmission: any; + avatarHeight?: number; + avatarDepth?: number; stopProcessing: () => void; resetVisemeQueue: () => void; updateCurrentViseme: ( currentTime: number ) => { name: string; weight: number } | null; + setCameraZ: (value: number) => void; } interface BaseAction { @@ -65,8 +71,11 @@ export const AvatarView: React.FC = ({ halfBody, loading, isZoomed, + avatarHeight, + avatarDepth, updateCurrentViseme, resetVisemeQueue, + setCameraZ, }) => { const [currentBaseAction, setCurrentBaseAction] = useState({ action: animation || 'Idle1', @@ -87,8 +96,11 @@ export const AvatarView: React.FC = ({ // Set the morph target influences for the given emotions const setEmotionMorphTargetInfluences = useCallback((action: string) => { - - if(action === 'Loading1' || action === 'Loading2' || action === 'Loading3') { + if ( + action === 'Loading1' || + action === 'Loading2' || + action === 'Loading3' + ) { return; } @@ -99,9 +111,6 @@ export const AvatarView: React.FC = ({ Tristezza: { Tristezza: 1 }, Timore: { Timore: 1 }, }; - - - // Set all emotions to 0 const defaultEmotions = Object.keys(emotionMap).reduce((acc, key) => { @@ -190,13 +199,6 @@ export const AvatarView: React.FC = ({ } }, [loading]); - // useEffect(() => { - // if (speaking && currentBaseAction.action !== 'Idle1') { - // const animation = `Idle1`; - // onBaseActionChange(animation); - // } - // }, [speaking]); - return ( <> {showControls && ( @@ -214,13 +216,12 @@ export const AvatarView: React.FC = ({ {halfBody ? ( ) : ( = ({ setMorphTargetDictionary={setMorphTargetDictionary} setMorphTargetInfluences={setMorphTargetInfluences} emotionMorphTargets={emotionMorphTargets} + halfBody={halfBody} + onCameraZChange={setCameraZ} + avatarHeight={avatarHeight || 50} + avatarDepth={avatarDepth || -50} /> )} diff --git a/src/components/Avatar/AvatarView/AvatarComponent/components/FullbodyAvatar/AnimationController.ts b/src/components/Avatar/AvatarView/AvatarComponent/components/FullbodyAvatar/AnimationController.ts new file mode 100644 index 00000000..e7347f4b --- /dev/null +++ b/src/components/Avatar/AvatarView/AvatarComponent/components/FullbodyAvatar/AnimationController.ts @@ -0,0 +1,260 @@ +import { AnimationState, AnimationConfig } from './types'; +import { AnimationAction, AnimationMixer, LoopOnce } from 'three'; +import { DEFAULT_CONFIG } from '../constants'; + +/** + * Controller class for managing avatar animations and transitions between states + */ +export class AnimationController { + // Current animation state (LOADING, EMOTION, IDLE) + private currentState: AnimationState = AnimationState.LOADING; + // Currently playing animation action + private currentAction: AnimationAction | null = null; + // Three.js animation mixer + private mixer: AnimationMixer; + // Map of available animation actions + private actions: Record; + // Animation configuration settings + private config: AnimationConfig; + // Index of last played idle animation + private lastIdleIndex: number = -1; + // Flag to prevent overlapping transitions + private isTransitioning: boolean = false; + // Counter for number of times current idle has looped + private currentIdleLoopCount: number = 0; + // Maximum number of idle loops before forcing change + private readonly MAX_IDLE_LOOPS = 5; + // Timestamp of last animation frame + private lastAnimationTime: number = 0; + // Flag to check if chat has already started + private isChatAlreadyStarted: boolean = false; + + constructor( + mixer: AnimationMixer, + actions: Record, + config: AnimationConfig = DEFAULT_CONFIG + ) { + // console.log('Initializing AnimationController'); + this.mixer = mixer; + this.actions = actions; + this.config = config; + } + + /** + * Checks if current idle animation has completed a loop + */ + private checkForLoop() { + if (!this.currentAction || this.currentState !== AnimationState.IDLE) + return; + + const clip = this.currentAction.getClip(); + const currentTime = this.currentAction.time; + + // If the current time is less than the last time we recorded, + // it means the animation has looped + if (currentTime < this.lastAnimationTime) { + this.currentIdleLoopCount++; + // console.log( + // `[AnimationController] Loop detected! Count: ${this.currentIdleLoopCount}` + // ); + + // Force idle change after MAX_IDLE_LOOPS + if (this.currentIdleLoopCount >= this.MAX_IDLE_LOOPS) { + // console.log( + // '[AnimationController] Max loops reached, changing idle animation' + // ); + this.forceIdleChange(); + } + } + + this.lastAnimationTime = currentTime; + } + + /** + * Forces transition to a new idle animation + */ + private forceIdleChange() { + // console.log('[AnimationController] Forcing idle change'); + this.currentIdleLoopCount = 0; + this.lastAnimationTime = 0; + this.transitionTo(AnimationState.IDLE); + } + + /** + * Selects next random idle animation that differs from last played + */ + private getNextIdleAnimation(): AnimationAction { + let nextIndex; + do { + nextIndex = Math.floor(Math.random() * this.config.idleCount) + 1; + } while (nextIndex === this.lastIdleIndex); + + // console.log( + // '[AnimationController] isChatAlreadyStarted', + // this.isChatAlreadyStarted + // ); + + if (this.isChatAlreadyStarted && nextIndex === 3) { + // If chat has already started and the last idle was Idle4, use Idle3 instead + nextIndex = this.lastIdleIndex !== 4 ? 4 : 2; + } + + // console.log( + // `[AnimationController] Selected idle animation: Idle${nextIndex}` + // ); + this.lastIdleIndex = nextIndex; + const idleAction = this.actions[`Idle${nextIndex}`]; + + if (!idleAction) { + throw new Error(`Idle animation ${nextIndex} not found`); + } + + return idleAction; + } + + /** + * Transitions to a new animation state + */ + transitionTo(state: AnimationState, emotionName?: string) { + if (this.isTransitioning) { + // console.log( + // '[AnimationController] Transition already in progress, skipping' + // ); + return; + } + + // console.log( + // `[AnimationController] Transitioning to ${state}${ + // emotionName ? ` (${emotionName})` : '' + // }` + // ); + this.isTransitioning = true; + + try { + let nextAction: AnimationAction | null = null; + + switch (state) { + case AnimationState.LOADING: + nextAction = this.actions[emotionName || 'Loading1']; + this.currentIdleLoopCount = 0; + this.lastAnimationTime = 0; + break; + case AnimationState.EMOTION: + nextAction = this.actions[emotionName || 'Timore1']; + this.currentIdleLoopCount = 0; + this.lastAnimationTime = 0; + break; + case AnimationState.IDLE: + nextAction = this.getNextIdleAnimation(); + // Only reset loop count if we're coming from a different idle animation + if (this.currentState !== AnimationState.IDLE) { + this.currentIdleLoopCount = 0; + this.lastAnimationTime = 0; + } + break; + } + + if (!nextAction) { + throw new Error(`No animation found for state: ${state}`); + } + + // Fade out current animation + if (this.currentAction) { + this.currentAction.fadeOut(this.config.fadeOutDuration); + } + + // Setup next animation + nextAction.reset().fadeIn(this.config.fadeInDuration).play(); + + // Configure animation properties + nextAction.timeScale = this.config.timeScale; + if (state !== AnimationState.IDLE) { + nextAction.setLoop(LoopOnce, 1); + nextAction.clampWhenFinished = true; + } else { + nextAction.setLoop(Infinity, Infinity); + } + + this.currentAction = nextAction; + this.currentState = state; + // console.log('[AnimationController] Transition completed successfully'); + } catch (error) { + console.error( + '[AnimationController] Error during animation transition:', + error + ); + if (state !== AnimationState.IDLE) { + this.transitionTo(AnimationState.IDLE); + } + } finally { + this.isTransitioning = false; + } + } + + /** + * Updates animation state on each frame + */ + update(delta: number) { + if (!this.currentAction) return; + + // Check for loop completion in idle animations + this.checkForLoop(); + + // Check if emotion/loading animation is finished + if ( + this.currentState !== AnimationState.IDLE && + this.currentAction.time >= this.currentAction.getClip().duration * 0.9 + ) { + // console.log( + // '[AnimationController] Non-idle animation completed, transitioning to idle' + // ); + this.transitionTo(AnimationState.IDLE); + } + + this.mixer.update(delta); + } + + /** + * Returns current animation state + */ + getCurrentState(): AnimationState { + return this.currentState; + } + + /** + * Returns number of times current idle has looped + */ + getLoopCount(): number { + return this.currentIdleLoopCount; + } + + /** + * Updates animation playback speed + */ + setTimeScale(timeScale: number) { + // console.log(`[AnimationController] Setting time scale to ${timeScale}`); + this.config.timeScale = timeScale; + if (this.currentAction) { + this.currentAction.timeScale = timeScale; + } + } + + updateIsChatAlreadyStarted(isChatAlreadyStarted: boolean) { + this.isChatAlreadyStarted = isChatAlreadyStarted; + } + + /** + * Returns debug information about current animation state + */ + getDebugInfo() { + return { + currentState: this.currentState, + currentIdleIndex: this.lastIdleIndex, + loopCount: this.currentIdleLoopCount, + currentTime: this.currentAction?.time || 0, + lastTime: this.lastAnimationTime, + isTransitioning: this.isTransitioning, + isChatAlreadyStarted: this.isChatAlreadyStarted, + }; + } +} diff --git a/src/components/Avatar/AvatarView/AvatarComponent/components/FullbodyAvatar/fullbodyAvatar.tsx b/src/components/Avatar/AvatarView/AvatarComponent/components/FullbodyAvatar/fullbodyAvatar.tsx new file mode 100644 index 00000000..b6c52859 --- /dev/null +++ b/src/components/Avatar/AvatarView/AvatarComponent/components/FullbodyAvatar/fullbodyAvatar.tsx @@ -0,0 +1,199 @@ +import { useEffect, useRef, useMemo } from 'react'; +import { + AnimationMixer, + SkinnedMesh, + Object3D, + AnimationAction, +} from 'three'; +import { useAnimations, useGLTF } from '@react-three/drei'; +import { useFrame } from '@react-three/fiber'; +import { AnimationState, FullbodyAvatarProps } from './types'; +import { AnimationController } from './AnimationController'; +import { MorphTargetController } from '../MorphTargetController'; +import { AvatarPositionController } from '../PositionController'; +import { + AVATAR_POSITION, + AVATAR_ROTATION, + AVATAR_POSITION_ZOOMED, + ANIMATION_URLS, + DEFAULT_CONFIG, + SCALE_LERP_FACTOR, +} from '../constants'; + +export function FullbodyAvatar({ + url, + sex, + currentBaseAction, + timeScale, + isZoomed, + eyeBlink, + updateCurrentViseme, + setMorphTargetDictionary, + setMorphTargetInfluences, + emotionMorphTargets, + avatarHeight = 50, + avatarDepth = 0, + onCameraZChange, +}: FullbodyAvatarProps) { + const { scene } = useGLTF(url); + const { animations } = useGLTF(ANIMATION_URLS[sex]); + const { actions } = useAnimations(animations, scene); + + const animationControllerRef = useRef(); + const morphTargetControllerRef = useRef(); + const positionControllerRef = useRef(); + + const blinkStateRef = useRef({ + isBlinking: false, + lastBlinkTime: 0, + nextBlinkTime: 0, + blinkStartTime: 0, + }); + + // Initialize controllers + useEffect(() => { + if (!positionControllerRef.current) { + positionControllerRef.current = new AvatarPositionController(AVATAR_POSITION); + } + + if (!actions || !scene) return; + + const mixer = new AnimationMixer(scene); + animationControllerRef.current = new AnimationController( + mixer, + actions as Record, + { ...DEFAULT_CONFIG } + ); + + if (headMesh) { + morphTargetControllerRef.current = new MorphTargetController(headMesh); + + if (headMesh.morphTargetDictionary && headMesh.morphTargetInfluences) { + setMorphTargetDictionary(headMesh.morphTargetDictionary); + const initialInfluences = Object.keys(headMesh.morphTargetDictionary) + .reduce((acc, key) => ({ ...acc, [key]: 0 }), {}); + setMorphTargetInfluences(initialInfluences); + } + } + }, [actions, scene]); + useEffect(() => { + if (positionControllerRef.current) { + positionControllerRef.current.updateHeight(avatarHeight, false); + } + }, [avatarHeight]); + + useEffect(() => { + if (positionControllerRef.current && onCameraZChange) { + const newCameraZ = positionControllerRef.current.updateDepth(avatarDepth, false); + onCameraZChange(newCameraZ); + } + }, [avatarDepth, onCameraZChange]); + + // Find head mesh + const headMesh = useMemo(() => { + let foundMesh: SkinnedMesh | undefined; + scene?.traverse((object: Object3D) => { + if ( + object instanceof SkinnedMesh && + (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar') + ) { + foundMesh = object; + } + }); + return foundMesh; + }, [scene]); + + // Initialize controllers + useEffect(() => { + if (!actions || !headMesh) return; + + const mixer = new AnimationMixer(scene); + animationControllerRef.current = new AnimationController( + mixer, + actions as Record, + { ...DEFAULT_CONFIG } + ); + + morphTargetControllerRef.current = new MorphTargetController(headMesh); + + // Initialize morph target dictionary and influences + if (headMesh.morphTargetDictionary && headMesh.morphTargetInfluences) { + setMorphTargetDictionary(headMesh.morphTargetDictionary); + const initialInfluences = Object.keys( + headMesh.morphTargetDictionary + ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {}); + setMorphTargetInfluences(initialInfluences); + } + }, [ + actions, + headMesh, + scene, + setMorphTargetDictionary, + setMorphTargetInfluences, + timeScale, + ]); + + // Handle animation state changes + useEffect(() => { + if (!animationControllerRef.current) return; + + if (currentBaseAction.action.startsWith('Loading')) { + animationControllerRef.current.updateIsChatAlreadyStarted(true); + animationControllerRef.current.transitionTo( + AnimationState.LOADING, + currentBaseAction.action + ); + } else if (currentBaseAction.action.startsWith('Idle')) { + animationControllerRef.current.transitionTo(AnimationState.IDLE); + } else { + animationControllerRef.current.transitionTo( + AnimationState.EMOTION, + currentBaseAction.action + ); + } + }, [currentBaseAction]); + + // Update timeScale when it changes + useEffect(() => { + animationControllerRef.current?.setTimeScale(timeScale); + }, [timeScale]); + + + // Animation and scaling update loop + useFrame((state, delta) => { + const currentTime = state.clock.elapsedTime * 1000; + + // Update animations + animationControllerRef.current?.update(delta); + + // Update morph targets + if (morphTargetControllerRef.current) { + const currentViseme = updateCurrentViseme(currentTime / 1000); + morphTargetControllerRef.current.updateMorphTargets( + currentTime, + emotionMorphTargets, + currentViseme, + eyeBlink || false, + blinkStateRef.current + ); + } + + // Update scale with smooth transition + if (scene && positionControllerRef.current) { + const newScale = positionControllerRef.current.updateScale(SCALE_LERP_FACTOR); + scene.scale.copy(newScale); + } + }); + + // Get current position from controller + const position = positionControllerRef.current?.getPosition() || AVATAR_POSITION; + + return ( + + + + ); +} \ No newline at end of file diff --git a/src/components/Avatar/AvatarView/AvatarComponent/components/FullbodyAvatar/types.ts b/src/components/Avatar/AvatarView/AvatarComponent/components/FullbodyAvatar/types.ts new file mode 100644 index 00000000..759b8c95 --- /dev/null +++ b/src/components/Avatar/AvatarView/AvatarComponent/components/FullbodyAvatar/types.ts @@ -0,0 +1,44 @@ +export enum AnimationState { + LOADING = 'LOADING', + EMOTION = 'EMOTION', + IDLE = 'IDLE', +} + +export interface AnimationConfig { + fadeInDuration: number; + fadeOutDuration: number; + idleCount: number; + timeScale: number; +} + +export interface FullbodyAvatarProps { + url: string; + sex: 'MALE' | 'FEMALE'; + onLoaded?: () => void; + currentBaseAction: { + action: string; + weight: number; + }; + timeScale: number; + onCameraZChange: (value: number) => void; + isZoomed?: boolean; + eyeBlink?: boolean; + avatarDepth?: number; + avatarHeight?: number; + stopProcessing: () => void; + resetVisemeQueue: () => void; + updateCurrentViseme: ( + currentTime: number + ) => { name: string; weight: number } | null; + smoothMorphTarget?: boolean; + morphTargetSmoothing?: number; + morphTargetInfluences: Record; + setMorphTargetDictionary: ( + morphTargetDictionary: Record + ) => void; + setMorphTargetInfluences: ( + morphTargetInfluences: Record + ) => void; + emotionMorphTargets: Record; + halfBody: boolean; +} diff --git a/src/components/Avatar/AvatarView/AvatarComponent/components/MorphTargetController.ts b/src/components/Avatar/AvatarView/AvatarComponent/components/MorphTargetController.ts new file mode 100644 index 00000000..483c6547 --- /dev/null +++ b/src/components/Avatar/AvatarView/AvatarComponent/components/MorphTargetController.ts @@ -0,0 +1,123 @@ +import { SkinnedMesh } from 'three'; +import { MathUtils } from 'three'; +import { EMOTION_SMOOTHING, VISEME_SMOOTHING, BLINK_CONFIG } from './constants'; + +export class MorphTargetController { + private headMesh: SkinnedMesh; + private currentEmotionValues: Record = {}; + private previousEmotionKeys: Set = new Set(); + + constructor(headMesh: SkinnedMesh) { + this.headMesh = headMesh; + } + + updateMorphTargets( + currentTime: number, + emotionMorphTargets: Record, + currentViseme: { name: string; weight: number } | null, + eyeBlink: boolean, + blinkState: { + isBlinking: boolean; + lastBlinkTime: number; + nextBlinkTime: number; + blinkStartTime: number; + } + ) { + if ( + !this.headMesh.morphTargetDictionary || + !this.headMesh.morphTargetInfluences + ) { + return; + } + + const blinkValue = this.calculateBlinkValue( + currentTime, + blinkState, + eyeBlink + ); + const currentEmotionKeys = new Set(Object.keys(emotionMorphTargets)); + + Object.entries(this.headMesh.morphTargetDictionary).forEach( + ([key, index]) => { + if (typeof index !== 'number') return; + + let targetValue = 0; + + // Handle emotion morphs + if (currentEmotionKeys.has(key)) { + const targetEmotionValue = emotionMorphTargets[key]; + const currentEmotionValue = this.currentEmotionValues[key] || 0; + const newEmotionValue = MathUtils.lerp( + currentEmotionValue, + targetEmotionValue * 2.5, + EMOTION_SMOOTHING + ); + this.currentEmotionValues[key] = newEmotionValue; + targetValue += newEmotionValue; + } + + // Handle viseme + if (currentViseme && key === currentViseme.name) { + targetValue += currentViseme.weight; + } + + // Handle blinking + if (key === 'eyesClosed' && eyeBlink) { + targetValue += blinkValue; + } + + // Apply final value + targetValue = MathUtils.clamp(targetValue, 0, 1); + if (this.headMesh.morphTargetInfluences) { + this.headMesh.morphTargetInfluences[index] = MathUtils.lerp( + this.headMesh.morphTargetInfluences[index] || 0, + targetValue, + VISEME_SMOOTHING + ); + } + } + ); + + this.previousEmotionKeys = currentEmotionKeys; + } + + private calculateBlinkValue( + currentTime: number, + blinkState: { + isBlinking: boolean; + lastBlinkTime: number; + nextBlinkTime: number; + blinkStartTime: number; + }, + eyeBlink: boolean + ): number { + if (!eyeBlink) return 0; + + let blinkValue = 0; + + if (currentTime >= blinkState.nextBlinkTime && !blinkState.isBlinking) { + blinkState.isBlinking = true; + blinkState.blinkStartTime = currentTime; + blinkState.lastBlinkTime = currentTime; + blinkState.nextBlinkTime = + currentTime + + Math.random() * (BLINK_CONFIG.maxInterval - BLINK_CONFIG.minInterval) + + BLINK_CONFIG.minInterval; + } + + if (blinkState.isBlinking) { + const blinkProgress = + (currentTime - blinkState.blinkStartTime) / BLINK_CONFIG.blinkDuration; + if (blinkProgress <= 0.5) { + blinkValue = blinkProgress * 2; + } else if (blinkProgress <= 1) { + blinkValue = 2 - blinkProgress * 2; + } else { + blinkState.isBlinking = false; + blinkValue = 0; + } + } + + return blinkValue; + } +} diff --git a/src/components/Avatar/AvatarView/AvatarComponent/components/PositionController.ts b/src/components/Avatar/AvatarView/AvatarComponent/components/PositionController.ts new file mode 100644 index 00000000..0d83cd68 --- /dev/null +++ b/src/components/Avatar/AvatarView/AvatarComponent/components/PositionController.ts @@ -0,0 +1,83 @@ +import { Vector3, MathUtils } from 'three'; +import { AVATAR_POSITION, AVATAR_POSITION_ZOOMED } from './constants'; + +export class AvatarPositionController { + private currentScale: Vector3; + private targetScale: Vector3; + private currentPosition: Vector3; + private basePosition: Vector3; + private defaultPosition: Vector3; + private zoomedPosition: Vector3; + private initialCameraPosition: Vector3; + + constructor( + defaultPosition: Vector3 = AVATAR_POSITION.clone(), + zoomedPosition: Vector3 = AVATAR_POSITION_ZOOMED.clone(), + initialCameraZ: number = 0.6 + ) { + this.defaultPosition = defaultPosition; + this.zoomedPosition = zoomedPosition; + this.currentScale = new Vector3(1, 1, 1); + this.targetScale = new Vector3(1, 1, 1); + this.currentPosition = defaultPosition.clone(); + this.basePosition = defaultPosition.clone(); + this.initialCameraPosition = new Vector3(0, 0, initialCameraZ); + } + + // Map height slider value (0 to 100) to scale (0.8 to 1.2) + private mapHeightToScale(sliderValue: number, isHalfBody: boolean): number { + // Convert slider value to scale factor + if (isHalfBody) { + return MathUtils.lerp(1.4, 1.8, sliderValue / 100); + } else { + return MathUtils.lerp(0.5, 1.5, sliderValue / 100); + } + } + + // Map depth slider value (-100 to 100) to camera Z position + private mapDepthToCamera(depthValue: number, isHalfBody: boolean): number { + const baseZ = this.initialCameraPosition.z; + if (isHalfBody) { + return MathUtils.lerp(baseZ - 0, baseZ + 1.5, (depthValue + 100) / 200); + } else { + return MathUtils.lerp(baseZ - 0, baseZ + 3, (depthValue + 100) / 200); + } + } + + // Update height from GUI control (0-100) + updateHeight(heightValue: number, isHalfBody: boolean): void { + const heightScale = this.mapHeightToScale(heightValue, isHalfBody); + this.targetScale.set(heightScale, heightScale, heightScale); + } + + // Update depth and return new camera Z position + updateDepth(depthValue: number, isHalfBody: boolean): number { + return this.mapDepthToCamera(depthValue, isHalfBody); + } + + // Update base position for zooming + updateBasePosition(isZoomed: boolean): void { + const newPosition = isZoomed ? this.zoomedPosition : this.defaultPosition; + this.basePosition.copy(newPosition); + this.currentPosition.copy(newPosition); + } + + // Smoothly interpolate current scale to target scale + updateScale(lerpFactor: number): Vector3 { + this.currentScale.lerp(this.targetScale, lerpFactor); + return this.currentScale; + } + + // Get current position + getPosition(): Vector3 { + return this.currentPosition; + } + + // Reset to default values + reset(): void { + this.currentScale.set(1, 1, 1); + this.targetScale.set(1, 1, 1); + this.currentPosition.copy(this.defaultPosition); + this.basePosition.copy(this.defaultPosition); + } +} \ No newline at end of file diff --git a/src/components/Avatar/AvatarView/AvatarComponent/components/constants.ts b/src/components/Avatar/AvatarView/AvatarComponent/components/constants.ts new file mode 100644 index 00000000..8e94c4db --- /dev/null +++ b/src/components/Avatar/AvatarView/AvatarComponent/components/constants.ts @@ -0,0 +1,29 @@ +import { Vector3, Euler } from 'three'; +import { AnimationConfig } from './FullbodyAvatar/types'; + +export const AVATAR_POSITION = new Vector3(0, -1, 0); +export const AVATAR_ROTATION = new Euler(0.175, 0, 0); +export const AVATAR_POSITION_ZOOMED = new Vector3(0, -1.45, 0); +export const SCALE_LERP_FACTOR = 0.1; + +export const ANIMATION_URLS = { + MALE: 'https://assets.memori.ai/api/v2/asset/2c5e88a4-cf62-408b-9ef0-518b099dfcb2.glb', + FEMALE: + 'https://assets.memori.ai/api/v2/asset/8d1a5853-f05a-4a34-9f99-6eff64986081.glb', +}; + +export const BLINK_CONFIG = { + minInterval: 1000, + maxInterval: 5000, + blinkDuration: 150, +}; + +export const DEFAULT_CONFIG: AnimationConfig = { + fadeInDuration: 0.8, + fadeOutDuration: 0.8, + idleCount: 5, + timeScale: 1.0, +}; + +export const EMOTION_SMOOTHING = 0.3; +export const VISEME_SMOOTHING = 0.5; diff --git a/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx b/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx deleted file mode 100644 index 7c5eb7fd..00000000 --- a/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx +++ /dev/null @@ -1,281 +0,0 @@ -import React, { useEffect, useRef, useMemo, useCallback } from 'react'; -import { - Vector3, - Euler, - AnimationMixer, - SkinnedMesh, - Object3D, - MathUtils, - AnimationAction, - LoopOnce, -} from 'three'; -import { useAnimations, useGLTF } from '@react-three/drei'; -import { useGraph, useFrame } from '@react-three/fiber'; - -interface FullbodyAvatarProps { - url: string; - sex: 'MALE' | 'FEMALE'; - onLoaded?: () => void; - currentBaseAction: { - action: string; - weight: number; - }; - timeScale: number; - isZoomed?: boolean; - eyeBlink?: boolean; - stopProcessing: () => void; - resetVisemeQueue: () => void; - updateCurrentViseme: ( - currentTime: number - ) => { name: string; weight: number } | null; - smoothMorphTarget?: boolean; - morphTargetSmoothing?: number; - morphTargetInfluences: Record; - setMorphTargetDictionary: ( - morphTargetDictionary: Record - ) => void; - setMorphTargetInfluences: ( - morphTargetInfluences: Record - ) => void; - emotionMorphTargets: Record; -} - -const AVATAR_POSITION = new Vector3(0, -1, 0); -const AVATAR_ROTATION = new Euler(0.175, 0, 0); -const AVATAR_POSITION_ZOOMED = new Vector3(0, -1.45, 0); - -const ANIMATION_URLS = { - MALE: 'https://assets.memori.ai/api/v2/asset/2c5e88a4-cf62-408b-9ef0-518b099dfcb2.glb', - FEMALE: - 'https://assets.memori.ai/api/v2/asset/8d1a5853-f05a-4a34-9f99-6eff64986081.glb', -}; - -const BLINK_CONFIG = { - minInterval: 1000, - maxInterval: 5000, - blinkDuration: 150, -}; - -const EMOTION_SMOOTHING = 0.3; -const VISME_SMOOTHING = 0.5; - -export default function FullbodyAvatar({ - url, - sex, - currentBaseAction, - timeScale, - isZoomed, - eyeBlink, - updateCurrentViseme, - setMorphTargetDictionary, - setMorphTargetInfluences, - emotionMorphTargets, -}: FullbodyAvatarProps) { - const { scene } = useGLTF(url); - const { animations } = useGLTF(ANIMATION_URLS[sex]); - const { actions } = useAnimations(animations, scene); - - const mixerRef = useRef(); - const headMeshRef = useRef(); - const currentActionRef = useRef(null); - const isTransitioningToIdleRef = useRef(false); - - const lastBlinkTimeRef = useRef(0); - const nextBlinkTimeRef = useRef(0); - const isBlinkingRef = useRef(false); - const blinkStartTimeRef = useRef(0); - - const currentEmotionRef = useRef>({}); - const previousEmotionKeysRef = useRef>(new Set()); - - // Memoize the scene traversal - const headMesh = useMemo(() => { - let foundMesh: SkinnedMesh | undefined; - scene.traverse((object: Object3D) => { - if ( - object instanceof SkinnedMesh && - (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar') - ) { - foundMesh = object; - } - }); - return foundMesh; - }, [scene]); - - useEffect(() => { - if (headMesh) { - headMeshRef.current = headMesh; - if (headMesh.morphTargetDictionary && headMesh.morphTargetInfluences) { - setMorphTargetDictionary(headMesh.morphTargetDictionary); - const initialInfluences = Object.keys( - headMesh.morphTargetDictionary - ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {}); - setMorphTargetInfluences(initialInfluences); - } - } - mixerRef.current = new AnimationMixer(scene); - }, [headMesh, scene, setMorphTargetDictionary, setMorphTargetInfluences]); - - // Memoize the animation change handler - const handleAnimationChange = useCallback(() => { - if (!actions || !currentBaseAction.action) return; - - const newAction = actions[currentBaseAction.action]; - if (!newAction) { - console.warn( - `Animation "${currentBaseAction.action}" not found in actions.` - ); - return; - } - - const fadeOutDuration = 0.8; - const fadeInDuration = 0.8; - - if (currentActionRef.current) { - currentActionRef.current.fadeOut(fadeOutDuration); - } - - newAction.reset().fadeIn(fadeInDuration).play(); - currentActionRef.current = newAction; - newAction.timeScale = timeScale; - - if ( - currentBaseAction.action.startsWith('Gioia') || - currentBaseAction.action.startsWith('Rabbia') || - currentBaseAction.action.startsWith('Sorpresa') || - currentBaseAction.action.startsWith('Timore') || - currentBaseAction.action.startsWith('Tristezza') - ) { - newAction.setLoop(LoopOnce, 1); - newAction.clampWhenFinished = true; - isTransitioningToIdleRef.current = true; - } - }, [actions, currentBaseAction, timeScale]); - - useEffect(() => { - handleAnimationChange(); - }, [handleAnimationChange]); - - // Optimize the frame update function - const updateFrame = useCallback( - (currentTime: number) => { - if ( - !headMeshRef.current || - !headMeshRef.current.morphTargetDictionary || - !headMeshRef.current.morphTargetInfluences - ) - return; - - let blinkValue = 0; - if (eyeBlink) { - if (currentTime >= nextBlinkTimeRef.current && !isBlinkingRef.current) { - isBlinkingRef.current = true; - blinkStartTimeRef.current = currentTime; - lastBlinkTimeRef.current = currentTime; - nextBlinkTimeRef.current = - currentTime + - Math.random() * - (BLINK_CONFIG.maxInterval - BLINK_CONFIG.minInterval) + - BLINK_CONFIG.minInterval; - } - - if (isBlinkingRef.current) { - const blinkProgress = - (currentTime - blinkStartTimeRef.current) / - BLINK_CONFIG.blinkDuration; - if (blinkProgress <= 0.5) { - blinkValue = blinkProgress * 2; - } else if (blinkProgress <= 1) { - blinkValue = 2 - blinkProgress * 2; - } else { - isBlinkingRef.current = false; - blinkValue = 0; - } - } - } - - // Update current viseme - const currentViseme = updateCurrentViseme(currentTime / 1000); - const currentEmotionKeys = new Set(Object.keys(emotionMorphTargets)); - - // Update morph target influences - Object.entries(headMeshRef.current.morphTargetDictionary).forEach( - ([key, index]) => { - if (typeof index === 'number') { - let targetValue = 0; - - if (currentEmotionKeys.has(key)) { - const targetEmotionValue = emotionMorphTargets[key]; - const currentEmotionValue = currentEmotionRef.current[key] || 0; - const newEmotionValue = MathUtils.lerp( - currentEmotionValue, - targetEmotionValue * 2.5, - EMOTION_SMOOTHING - ); - currentEmotionRef.current[key] = newEmotionValue; - targetValue += newEmotionValue; - } - - if (currentViseme && key === currentViseme.name) { - targetValue += currentViseme.weight; - } - - if (key === 'eyesClosed' && eyeBlink) { - targetValue += blinkValue; - } - - targetValue = MathUtils.clamp(targetValue, 0, 1); - if ( - headMeshRef.current && - headMeshRef.current.morphTargetInfluences - ) { - headMeshRef.current.morphTargetInfluences[index] = MathUtils.lerp( - headMeshRef.current.morphTargetInfluences[index], - targetValue, - VISME_SMOOTHING - ); - } - } - } - ); - - // Update previous emotion keys - previousEmotionKeysRef.current = currentEmotionKeys; - - // Transition to idle - if (isTransitioningToIdleRef.current && currentActionRef.current) { - if ( - currentActionRef.current.time >= - currentActionRef.current.getClip().duration - ) { - const idleNumber = Math.floor(Math.random() * 5) + 1; - const idleAction = - actions[`Idle${idleNumber === 3 ? 4 : idleNumber}`]; - - if (idleAction) { - currentActionRef.current.fadeOut(0.5); - idleAction.reset().fadeIn(0.5).play(); - currentActionRef.current = idleAction; - isTransitioningToIdleRef.current = false; - } - } - } - - mixerRef.current?.update(0.01); - }, - [actions, emotionMorphTargets, eyeBlink, updateCurrentViseme] - ); - - useFrame(state => { - updateFrame(state.clock.elapsedTime * 1000); - }); - - return ( - - - - ); -} diff --git a/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx b/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx index 0a013e3b..cb97968c 100644 --- a/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx +++ b/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx @@ -1,168 +1,148 @@ import React, { useEffect, useMemo, useRef } from 'react'; -import { Object3D, SkinnedMesh, Vector3 } from 'three'; +import { MathUtils, Object3D, SkinnedMesh, Vector3 } from 'three'; import { useGLTF } from '@react-three/drei'; +import { useGraph, useFrame, useThree } from '@react-three/fiber'; import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils'; -import { useGraph, dispose, useFrame } from '@react-three/fiber'; -import { useAvatarBlink } from '../../utils/useEyeBlink'; -import useHeadMovement from '../../utils/useHeadMovement'; +import { MorphTargetController } from './MorphTargetController'; +import { AvatarPositionController } from './PositionController'; +import { + AVATAR_POSITION, + SCALE_LERP_FACTOR, + AVATAR_POSITION_ZOOMED, +} from './constants'; import { hideHands } from '../../utils/utils'; -import { AnimationMixer, MathUtils } from 'three'; interface HalfBodyAvatarProps { url: string; setMorphTargetInfluences: (morphTargetInfluences: any) => void; - headMovement?: boolean; - speaking?: boolean; - onLoaded?: () => void; setMorphTargetDictionary: (morphTargetDictionary: any) => void; - eyeBlink?: boolean; - morphTargetInfluences: any; updateCurrentViseme: (currentTime: number) => any; + eyeBlink?: boolean; + isZoomed?: boolean; + heightValue?: number; // 0-100 slider value + avatarHeight: number; + avatarDepth: number; morphTargetSmoothing?: number; + onLoaded?: () => void; + onCameraZChange: (value: number) => void; } -const AVATAR_POSITION = new Vector3(0, -0.6, 0); -// Blink configuration -const BLINK_CONFIG = { - minInterval: 1000, - maxInterval: 5000, - blinkDuration: 150, -}; - export default function HalfBodyAvatar({ url, setMorphTargetInfluences, setMorphTargetDictionary, - eyeBlink, - onLoaded, - morphTargetSmoothing = 0.5, updateCurrentViseme, + eyeBlink = false, + isZoomed = false, + avatarHeight = 50, + avatarDepth = 0, + morphTargetSmoothing = 0.5, + onLoaded, + onCameraZChange, }: HalfBodyAvatarProps) { const { scene } = useGLTF(url); const { nodes, materials } = useGraph(scene); - const mixer = useRef(new AnimationMixer(scene)); - const avatarMeshRef = useRef(null); - - // Blink state - const lastBlinkTime = useRef(0); - const nextBlinkTime = useRef(0); - const isBlinking = useRef(false); - const blinkStartTime = useRef(0); + const { camera } = useThree(); + + const morphTargetControllerRef = useRef(); + const positionControllerRef = useRef(); + const targetCameraZRef = useRef(camera.position.z); + + const blinkStateRef = useRef({ + isBlinking: false, + lastBlinkTime: 0, + nextBlinkTime: 0, + blinkStartTime: 0, + }); - const headMeshRef = useRef(); + // Find head mesh + const headMesh = useMemo(() => { + let foundMesh: SkinnedMesh | undefined; + scene?.traverse((object: Object3D) => { + if ( + object instanceof SkinnedMesh && + (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar') + ) { + foundMesh = object; + } + }); + return foundMesh; + }, [scene]); + // Initialize controllers useEffect(() => { - correctMaterials(materials); + if (!positionControllerRef.current) { + positionControllerRef.current = new AvatarPositionController( + AVATAR_POSITION, + AVATAR_POSITION_ZOOMED, + ); + } - scene.traverse((object: Object3D) => { - if (object instanceof SkinnedMesh) { - if (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar') { - headMeshRef.current = object; - if (object.morphTargetDictionary && object.morphTargetInfluences) { - setMorphTargetDictionary(object.morphTargetDictionary); + if (headMesh) { + morphTargetControllerRef.current = new MorphTargetController(headMesh); - const initialInfluences = Object.keys( - object.morphTargetDictionary - ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {}); - setMorphTargetInfluences(initialInfluences); - } - } + if (headMesh.morphTargetDictionary && headMesh.morphTargetInfluences) { + setMorphTargetDictionary(headMesh.morphTargetDictionary); + const initialInfluences = Object.keys(headMesh.morphTargetDictionary) + .reduce((acc, key) => ({ ...acc, [key]: 0 }), {}); + setMorphTargetInfluences(initialInfluences); } - }); + } + correctMaterials(materials); onLoaded?.(); - + hideHands(nodes); + return () => { Object.values(materials).forEach(material => material.dispose()); Object.values(nodes) .filter(isSkinnedMesh) .forEach(mesh => mesh.geometry.dispose()); }; - }, [materials, nodes, url, onLoaded, scene]); - useFrame(state => { + }, [materials, nodes, url, onLoaded, scene, headMesh]); - if ( - headMeshRef.current && - headMeshRef.current.morphTargetDictionary && - headMeshRef.current.morphTargetInfluences - ) { - const currentTime = state.clock.getElapsedTime() * 1000; // Convert to milliseconds + useEffect(() => { + if (positionControllerRef.current) { + positionControllerRef.current.updateHeight(avatarHeight, true); + } + }, [avatarHeight]); - // Handle blinking - let blinkValue = 0; - if (eyeBlink) { - if (currentTime >= nextBlinkTime.current && !isBlinking.current) { - isBlinking.current = true; - blinkStartTime.current = currentTime; - lastBlinkTime.current = currentTime; - nextBlinkTime.current = - currentTime + - Math.random() * - (BLINK_CONFIG.maxInterval - BLINK_CONFIG.minInterval) + - BLINK_CONFIG.minInterval; - } + useEffect(() => { + if (positionControllerRef.current && onCameraZChange) { + const newCameraZ = positionControllerRef.current.updateDepth(avatarDepth, true); + onCameraZChange(newCameraZ); + } + }, [avatarDepth, onCameraZChange]); - if (isBlinking.current) { - const blinkProgress = - (currentTime - blinkStartTime.current) / BLINK_CONFIG.blinkDuration; - if (blinkProgress <= 0.5) { - // Eyes closing - blinkValue = blinkProgress * 2; - } else if (blinkProgress <= 1) { - // Eyes opening - blinkValue = 2 - blinkProgress * 2; - } else { - // Blink finished - isBlinking.current = false; - blinkValue = 0; - } - } - } + // Animation and morphing update loop + useFrame((state) => { + const currentTime = state.clock.elapsedTime * 1000; + // Update morph targets + if (morphTargetControllerRef.current) { const currentViseme = updateCurrentViseme(currentTime / 1000); - - // Update morph targets - Object.entries(headMeshRef.current.morphTargetDictionary).forEach( - ([key, index]) => { - if (typeof index === 'number') { - let targetValue = 0; - - // Handle visemes (additive layer) - if (currentViseme && key === currentViseme.name) { - targetValue += currentViseme.weight * 1.3; // Amplify the effect - } - - // Handle blinking (additive layer, only for 'eyesClosed') - if (key === 'eyesClosed' && eyeBlink) { - targetValue += blinkValue; - } - - // Clamp the final value between 0 and 1 - targetValue = MathUtils.clamp(targetValue, 0, 1); - - // Apply smoothing - if ( - headMeshRef.current && - headMeshRef.current.morphTargetInfluences - ) { - headMeshRef.current.morphTargetInfluences[index] = MathUtils.lerp( - headMeshRef.current.morphTargetInfluences[index], - targetValue, - morphTargetSmoothing - ); - } - } - } + morphTargetControllerRef.current.updateMorphTargets( + currentTime, + {}, + currentViseme, + eyeBlink, + blinkStateRef.current, ); + } - // Update the animation mixer - mixer.current.update(0.01); // Fixed delta time for consistent animation speed + // Update scale with smooth transition + if (scene && positionControllerRef.current) { + const newScale = positionControllerRef.current.updateScale(SCALE_LERP_FACTOR); + scene.scale.copy(newScale); } }); + // Get current position from controller + const position = positionControllerRef.current?.getPosition() || AVATAR_POSITION; + return ( - + ); -} +} \ No newline at end of file diff --git a/src/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.css b/src/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.css new file mode 100644 index 00000000..c5ebcda8 --- /dev/null +++ b/src/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.css @@ -0,0 +1,65 @@ +.memori--position-controls{ + position: fixed; + z-index: 1000; + top: 60px; + left: 15px; + display: flex; + min-width: 400px; + max-width: 400px; + height: auto !important; + flex-direction: column; + padding: calc(var(--memori-inner-content-pad) / 4) calc(var(--memori-inner-content-pad) / 2); + border-radius: 10px; + margin-left: auto; + -webkit-backdrop-filter: blur(10px); + backdrop-filter: blur(10px); + background-color: var(--memori-inner-bg, #fff); + gap: 0.5rem; + text-align: left; +} + +.memori--position-controls-helper-text{ + color: #3b3e46; + font-size: 0.875rem; +} + +.memori--slider-container { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.memori--preset-button{ + padding: 0.5rem; + border: 1px solid #ccc; +} + +.memori--preset-button:hover{ + background-color: #f0f0f0; +} + +.memori--preset-buttons{ + display: flex; + gap: 0.5rem; +} + +.memori--slider-label{ + margin: 0px; + color: #3b3e46; + font-size: 0.875rem; +} + +.memori--position-controls-close{ + position: absolute; + top: 0; + right: 0; +} + +.memori--position-controls-close-button{ + padding: 0.5rem; +} + +.memori--position-controls-close-button:hover, .memori--position-controls-close-button:active, .memori--position-controls-close-button:focus { + border: none !important; + box-shadow: none !important; +} \ No newline at end of file diff --git a/src/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.tsx b/src/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.tsx new file mode 100644 index 00000000..314c7951 --- /dev/null +++ b/src/components/Avatar/AvatarView/AvatarComponent/positionControls/positionControls.tsx @@ -0,0 +1,193 @@ +import { useEffect, useRef } from 'react'; +import { setLocalConfig } from '../../../../../helpers/configuration'; +import { useTranslation } from 'react-i18next'; +import Slider from '../../../../../components/ui/Slider'; +import './positionControls.css'; +import Button from '../../../../ui/Button'; +import Close from '../../../../icons/Close'; + +interface PositionControlsProps { + avatarHeight: number; + avatarDepth: number; + setAvatarHeight: (value: number) => void; + setAvatarDepth: (value: number) => void; + isZoomed?: boolean; + setEnablePositionControls: (value: boolean) => void; +} + +export const normalPosition = { height: 75, depth: -45 }; +export const zoomedPosition = { height: 65, depth: -80 }; +export const farPosition = { height: 100, depth: 50 }; + +// eslint-disable-next-line no-undef +const PositionControls: React.FC = ({ + avatarHeight, + avatarDepth, + setAvatarHeight, + setAvatarDepth, + isZoomed = false, + setEnablePositionControls, +}: PositionControlsProps) => { + const settingsRef = useRef>({ + height: avatarHeight, + depth: avatarDepth, + zoomed: false, + normal: false, + far: false, + }); + const { t } = useTranslation(); + + // Update settings when values change externally + useEffect(() => { + settingsRef.current.height = avatarHeight; + settingsRef.current.depth = avatarDepth; + }, [avatarHeight, avatarDepth]); + + // Keyboard controls for depth + useEffect(() => { + const handleKeyDown = (event: KeyboardEvent) => { + if (event.key === '-' || event.key === '_' && settingsRef.current.depth < 100) { + const newValue = Math.min(settingsRef.current.depth + 10, 100); + setAvatarDepth(newValue); + setLocalConfig('avatarDepth', newValue); + } else if ( + (event.key === '+' || event.key === '=') && + settingsRef.current.depth > -100 + ) { + const newValue = Math.max(settingsRef.current.depth - 10, -100); + setAvatarDepth(newValue); + setLocalConfig('avatarDepth', newValue); + } + }; + + //add event listeners for plus and minus + window.addEventListener('keydown', handleKeyDown); + return () => { + window.removeEventListener('keydown', handleKeyDown); + }; + }, [setAvatarDepth]); + + useEffect(() => { + const handleArrowUp = (event: KeyboardEvent) => { + if (event.key === 'ArrowUp' && settingsRef.current.height < 100) { + const newValue = settingsRef.current.height + 5; + setAvatarHeight(newValue); + setLocalConfig('avatarHeight', newValue); + } + }; + + const handleArrowDown = (event: KeyboardEvent) => { + if (event.key === 'ArrowDown' && settingsRef.current.height > 0) { + const newValue = settingsRef.current.height - 5; + setAvatarHeight(newValue); + setLocalConfig('avatarHeight', newValue); + } + }; + + window.addEventListener('keydown', handleArrowUp); + window.addEventListener('keydown', handleArrowDown); + + return () => { + window.removeEventListener('keydown', handleArrowUp); + window.removeEventListener('keydown', handleArrowDown); + }; + }, [setAvatarHeight]); + + return ( +
+
+
+
+

+ {t('write_and_speak.suggestions')} +

+
+
+ {t('write_and_speak.height')}} + step={1} + onChange={(value: number) => { + setAvatarHeight(value); + setLocalConfig('avatarHeight', value); + }} + /> +
+
+ {t('write_and_speak.depth')}} + onChange={(value: number) => { + setAvatarDepth(value); + setLocalConfig('avatarDepth', value); + }} + /> +
+
+ + + +
+
+ ); +}; + +export default PositionControls; diff --git a/src/components/Avatar/AvatarView/index.tsx b/src/components/Avatar/AvatarView/index.tsx index cd9bac69..a395251a 100644 --- a/src/components/Avatar/AvatarView/index.tsx +++ b/src/components/Avatar/AvatarView/index.tsx @@ -1,11 +1,18 @@ -import { CSSProperties } from 'react'; +import { CSSProperties, useEffect, useRef, useState } from 'react'; import React, { Suspense } from 'react'; import { Canvas } from '@react-three/fiber'; -import { OrbitControls, SpotLight, Environment } from '@react-three/drei'; +import { + OrbitControls, + SpotLight, + Environment, + PerspectiveCamera, +} from '@react-three/drei'; import { isAndroid, isiOS } from '../../../helpers/utils'; -import {AvatarView} from './AvatarComponent/avatarComponent'; +import { AvatarView } from './AvatarComponent/avatarComponent'; import Loader from './AvatarComponent/components/loader'; - +import { Vector3 } from 'three'; +import PositionControls from './AvatarComponent/positionControls/positionControls'; +import { getLocalConfig } from '../../../helpers/configuration'; export interface Props { url: string; sex: 'MALE' | 'FEMALE'; @@ -22,10 +29,15 @@ export interface Props { showControls?: boolean; isZoomed?: boolean; chatEmission?: any; + enablePositionControls?: boolean; + setEnablePositionControls: (value: boolean) => void; + isTotem?: boolean; setMeshRef?: any; stopProcessing: () => void; resetVisemeQueue: () => void; - updateCurrentViseme: (currentTime: number) => { name: string; weight: number } | null; + updateCurrentViseme: ( + currentTime: number + ) => { name: string; weight: number } | null; } const defaultStyles = { @@ -43,21 +55,6 @@ const defaultStyles = { }, }; -/* Animation Control Panel */ -const getCameraSettings = (halfBody: boolean, isZoomed?: boolean) => - halfBody - ? { - fov: 40, - position: [0, 0, 0.6], - } - : !halfBody && isZoomed - ? { - // Zoomed in - fov: 44, - position: [0, 0, 1.25], - } - : { fov: 40, position: [0, 0.0000175, 3] }; - const getLightingComponent = () => isAndroid() || isiOS() ? ( ) : ( ); - +const getCameraSettings = (halfBody: boolean, isZoomed: boolean) => { + const baseZ = halfBody ? 0.6 : 3; + + return { + fov: isZoomed ? 44 : 40, + position: [0, 0, baseZ], + target: [0, 0, 0], + }; +}; export default function ContainerAvatarView({ url, @@ -92,34 +97,89 @@ export default function ContainerAvatarView({ stopProcessing, resetVisemeQueue, updateCurrentViseme, + enablePositionControls, + setEnablePositionControls, + isTotem = false, }: Props) { + const [cameraZ, setCameraZ] = useState( + () => getCameraSettings(halfBody, isZoomed || false).position[2] + ); + + const getAvatarHeight = () => { + console.log('avatarHeight', getLocalConfig('avatarHeight', 50), isTotem); + if (isTotem) { + return getLocalConfig('avatarHeight', 50); + } else if (halfBody) { + return 100; + } else { + return isZoomed ? 20 : 55; + } + }; + + const getAvatarDepth = () => { + if (isTotem) { + console.log('avatarDepth', getLocalConfig('avatarDepth', 50)); + return getLocalConfig('avatarDepth', 50); + } else if (halfBody) { + return 50 + } else { + return isZoomed ? -80 : 70; + } + }; + + const [avatarHeight, setAvatarHeight] = useState(getAvatarHeight()); + const [avatarDepth, setAvatarDepth] = useState(getAvatarDepth()); + return ( - - }> - {getLightingComponent()} - {rotateAvatar && } - + + + + {rotateAvatar && ( + + )} + + }> + {getLightingComponent()} + + + + + {enablePositionControls && ( + - - + )} + ); } diff --git a/src/components/Avatar/AvatarView/utils/hideHands.ts b/src/components/Avatar/AvatarView/utils/hideHands.ts new file mode 100644 index 00000000..213db0a1 --- /dev/null +++ b/src/components/Avatar/AvatarView/utils/hideHands.ts @@ -0,0 +1,11 @@ +import { Nodes } from "./utils"; + +export const hideHands = (nodes: Nodes) => { + if (nodes.Wolf3D_Hands) { + nodes.Wolf3D_Hands.visible = false; + } + if (nodes.RightHand && nodes.LeftHand) { + nodes.RightHand.position.set(0, -2, 0); + nodes.LeftHand.position.set(0, -2, 0); + } +}; diff --git a/src/components/MemoriWidget/MemoriWidget.tsx b/src/components/MemoriWidget/MemoriWidget.tsx index d815c685..caf2b5e6 100644 --- a/src/components/MemoriWidget/MemoriWidget.tsx +++ b/src/components/MemoriWidget/MemoriWidget.tsx @@ -544,6 +544,9 @@ const MemoriWidget = ({ const [controlsPosition, setControlsPosition] = useState<'center' | 'bottom'>( 'center' ); + + const [enablePositionControls, setEnablePositionControls] = useState(false); + const [avatarType, setAvatarType] = useState<'blob' | 'avatar3d' | undefined>(undefined); const [hideEmissions, setHideEmissions] = useState(false); const { @@ -598,6 +601,7 @@ const MemoriWidget = ({ setControlsPosition( getLocalConfig('controlsPosition', defaultControlsPosition) ); + setAvatarType(getLocalConfig('avatarType', 'avatar3d')); setHideEmissions(getLocalConfig('hideEmissions', false)); if (!additionalInfo?.loginToken && !authToken) { @@ -3079,6 +3083,9 @@ const MemoriWidget = ({ loading: !!memoriTyping, baseUrl, apiUrl, + enablePositionControls, + setEnablePositionControls, + avatarType, }; const startPanelProps: StartPanelProps = { @@ -3378,6 +3385,11 @@ const MemoriWidget = ({ setControlsPosition={setControlsPosition} hideEmissions={hideEmissions} setHideEmissions={setHideEmissions} + avatarType={avatarType} + setAvatarType={setAvatarType} + enablePositionControls={enablePositionControls} + setEnablePositionControls={setEnablePositionControls} + isAvatar3d={!!integrationConfig?.avatarURL} additionalSettings={additionalSettings} /> )} diff --git a/src/components/SettingsDrawer/SettingsDrawer.css b/src/components/SettingsDrawer/SettingsDrawer.css index 1a66b9fc..4face8c1 100644 --- a/src/components/SettingsDrawer/SettingsDrawer.css +++ b/src/components/SettingsDrawer/SettingsDrawer.css @@ -1,7 +1,7 @@ .memori-settings-drawer .memori-settings-drawer--field { display: flex; flex-direction: column; - margin: 0.5rem 0 1.5rem; + margin: 0.5rem 0; } .memori-settings-drawer .memori-settings-drawer--field .memori-select--value { @@ -9,9 +9,11 @@ } .memori-settings-drawer--controlsposition-radio, +.memori-settings-drawer--avatarType-radio, .memori-settings-drawer--microphoneMode-radio { display: flex; flex-wrap: wrap; align-items: center; margin: 0.5rem 0; -} \ No newline at end of file + gap: 0.2rem; +} diff --git a/src/components/SettingsDrawer/SettingsDrawer.test.tsx b/src/components/SettingsDrawer/SettingsDrawer.test.tsx index a0420626..b4aa8258 100644 --- a/src/components/SettingsDrawer/SettingsDrawer.test.tsx +++ b/src/components/SettingsDrawer/SettingsDrawer.test.tsx @@ -25,6 +25,12 @@ it('renders SettingsDrawer unchanged', () => { setControlsPosition={jest.fn()} hideEmissions={false} setHideEmissions={jest.fn()} + avatarType="avatar3d" + setAvatarType={jest.fn()} + avatarHeight={150} + setAvatarHeight={jest.fn()} + avatarDepth={150} + setAvatarDepth={jest.fn()} /> ); expect(container).toMatchSnapshot(); @@ -43,6 +49,12 @@ it('renders SettingsDrawer open unchanged', () => { setControlsPosition={jest.fn()} hideEmissions={false} setHideEmissions={jest.fn()} + avatarType="avatar3d" + setAvatarType={jest.fn()} + avatarHeight={150} + setAvatarHeight={jest.fn()} + avatarDepth={150} + setAvatarDepth={jest.fn()} /> ); expect(container).toMatchSnapshot(); @@ -61,6 +73,12 @@ it('renders SettingsDrawer open with continuous speech enabled unchanged', () => setControlsPosition={jest.fn()} hideEmissions={false} setHideEmissions={jest.fn()} + avatarType="avatar3d" + setAvatarType={jest.fn()} + avatarHeight={150} + setAvatarHeight={jest.fn()} + avatarDepth={150} + setAvatarDepth={jest.fn()} /> ); expect(container).toMatchSnapshot(); @@ -79,6 +97,12 @@ it('renders SettingsDrawer open with non-default continuous speech timeout uncha setControlsPosition={jest.fn()} hideEmissions={false} setHideEmissions={jest.fn()} + avatarType="avatar3d" + setAvatarType={jest.fn()} + avatarHeight={150} + setAvatarHeight={jest.fn()} + avatarDepth={150} + setAvatarDepth={jest.fn()} /> ); expect(container).toMatchSnapshot(); @@ -98,6 +122,12 @@ it('renders SettingsDrawer for totem layout open unchanged', () => { setControlsPosition={jest.fn()} hideEmissions={false} setHideEmissions={jest.fn()} + avatarType="avatar3d" + setAvatarType={jest.fn()} + avatarHeight={150} + setAvatarHeight={jest.fn()} + avatarDepth={150} + setAvatarDepth={jest.fn()} /> ); expect(container).toMatchSnapshot(); @@ -117,6 +147,12 @@ it('renders SettingsDrawer for totem layout open with controls at center unchang setControlsPosition={jest.fn()} hideEmissions={false} setHideEmissions={jest.fn()} + avatarType="avatar3d" + setAvatarType={jest.fn()} + avatarHeight={150} + setAvatarHeight={jest.fn()} + avatarDepth={150} + setAvatarDepth={jest.fn()} /> ); expect(container).toMatchSnapshot(); @@ -136,6 +172,12 @@ it('renders SettingsDrawer for totem layout with continuous speech and hide emis setControlsPosition={jest.fn()} hideEmissions={true} setHideEmissions={jest.fn()} + avatarType="avatar3d" + setAvatarType={jest.fn()} + avatarHeight={150} + setAvatarHeight={jest.fn()} + avatarDepth={150} + setAvatarDepth={jest.fn()} /> ); expect(container).toMatchSnapshot(); @@ -172,6 +214,12 @@ it('renders SettingsDrawer with additional custom settings unchanged', () => { hideEmissions={false} setHideEmissions={jest.fn()} additionalSettings={} + avatarType="avatar3d" + setAvatarType={jest.fn()} + avatarHeight={150} + setAvatarHeight={jest.fn()} + avatarDepth={150} + setAvatarDepth={jest.fn()} /> ); expect(container).toMatchSnapshot(); diff --git a/src/components/SettingsDrawer/SettingsDrawer.tsx b/src/components/SettingsDrawer/SettingsDrawer.tsx index 5392d0e8..e308244e 100644 --- a/src/components/SettingsDrawer/SettingsDrawer.tsx +++ b/src/components/SettingsDrawer/SettingsDrawer.tsx @@ -3,10 +3,11 @@ import { useTranslation } from 'react-i18next'; import Checkbox from '../ui/Checkbox'; import Select from '../ui/Select'; import { setLocalConfig } from '../../helpers/configuration'; -import { RadioGroup } from '@headlessui/react'; +import { RadioGroup, Switch } from '@headlessui/react'; import Button from '../ui/Button'; import { Props as WidgetProps } from '../MemoriWidget/MemoriWidget'; - +import { useState } from 'react'; +import Slider from '../ui/Slider'; export interface Props { open: boolean; layout?: WidgetProps['layout']; @@ -20,6 +21,11 @@ export interface Props { hideEmissions?: boolean; setHideEmissions: (value: boolean) => void; additionalSettings?: WidgetProps['additionalSettings']; + avatarType?: 'blob' | 'avatar3d'; + setAvatarType: (value: 'blob' | 'avatar3d') => void; + enablePositionControls?: boolean; + setEnablePositionControls: (value: boolean) => void; + isAvatar3d?: boolean; } const silenceSeconds = [2, 3, 5, 10, 15, 20, 30, 60]; @@ -37,6 +43,11 @@ const SettingsDrawer = ({ hideEmissions, setHideEmissions, additionalSettings, + avatarType, + setAvatarType, + enablePositionControls, + setEnablePositionControls, + isAvatar3d, }: Props) => { const { t } = useTranslation(); @@ -58,7 +69,7 @@ const SettingsDrawer = ({ value={microphoneMode} defaultValue={microphoneMode} className="memori-settings-drawer--microphoneMode-radio" - onChange={value => { + onChange={(value: any) => { let micMode = value === 'CONTINUOUS' ? 'CONTINUOUS' : 'HOLD_TO_TALK'; @@ -117,7 +128,7 @@ const SettingsDrawer = ({ value={controlsPosition} defaultValue={controlsPosition} className="memori-settings-drawer--controlsposition-radio" - onChange={value => { + onChange={(value: any) => { setControlsPosition(value); setLocalConfig('controlsPosition', value); }} @@ -144,6 +155,67 @@ const SettingsDrawer = ({ + + {isAvatar3d && ( + <> +
+ + + { + setAvatarType && setAvatarType(value); + setLocalConfig('avatarType', value); + }} + > + + {({ checked }) => ( + + )} + + + {({ checked }) => ( + + )} + + +
+ +
+ { + setEnablePositionControls(e.target.checked); + }} + /> +
+ + )} +
= ({
- {Avatar && avatarProps && } + {Avatar && avatarProps && }
diff --git a/src/components/layouts/totem.css b/src/components/layouts/totem.css index 75cf4d3e..879b688a 100644 --- a/src/components/layouts/totem.css +++ b/src/components/layouts/totem.css @@ -95,28 +95,28 @@ bottom: auto; } -.memori-totem-layout--avatar .memori--avatar-wrapper>div { - overflow: visible !important; +/* .memori-totem-layout--avatar .memori--avatar-wrapper>div { + overflow: visible !important; */ /* width: 100% !important; height: 100% !important; */ - width: auto !important; + /* width: auto !important; height: 90vh !important; max-height: 90vh; border-radius: 0; transform: scale(1.7) translate(0px, 10vh); -} +} */ -.memori-totem-layout--avatar .memori--avatar-wrapper canvas { +/* .memori-totem-layout--avatar .memori--avatar-wrapper canvas { width: auto !important; max-width: 100%; height: 100% !important; max-height: 100%; -} +} */ -.memori-totem-layout--controls { +/* .memori-totem-layout--controls { position: relative; z-index: 5; -} +} */ .memori-totem-layout--controls .memori--start-panel, .memori-totem-layout--controls .memori-chat--history, diff --git a/src/components/ui/Button.css b/src/components/ui/Button.css index d9d60add..a93d4b38 100644 --- a/src/components/ui/Button.css +++ b/src/components/ui/Button.css @@ -12,6 +12,10 @@ font-size: 14px; transition: all 0.3s cubic-bezier(0.645, 0.045, 0.355, 1); } +.memori-button--active { + background: var(--memori-primary) !important; + color: var(--memori-primary-text) !important; +} .memori-button:hover { box-shadow: 0 2px 0 rgba(0, 0, 0, 0.02), 0 4px 8px rgba(0, 0, 0, 0.04); diff --git a/src/components/ui/Button.tsx b/src/components/ui/Button.tsx index 7fdce78b..893a6630 100644 --- a/src/components/ui/Button.tsx +++ b/src/components/ui/Button.tsx @@ -21,6 +21,7 @@ export interface Props { className?: string; title?: string; id?: string; + isActive?: boolean; htmlType?: 'button' | 'submit' | 'reset'; onClick?: (event: React.MouseEvent) => void; onMouseDown?: ( @@ -60,6 +61,7 @@ const Button: FC = ({ onTouchStart, onTouchEnd, children, + isActive, }) => (