Increase Pokémon sprite animation size

By default, Pokémon sprite animations have maximum sizes that limit how creative you can be with them. It's possible to overcome these limits and have animations use up to 255 tiles.

(The code for this feature was adapted from Pokémon Prism.)

Contents

1. Understanding the problem
2. Allocate space in SRAM for animation graphics
3. Change how animations are loaded
4. Don't use tile $7F

1. Understanding the problem

Pokémon sprite animations are composed of frames; for example, Haunter's 5-frame animation:

Sometimes you may try to redesign an animation; for example, this one by SoupPotato:

But some tiles are incorrect when it plays:

This is because sprite animations have size limits:

• A 5x5-tile sprite (40x40 pixels) uses 5×5=25 tiles for its first frame, and can use another 25 for its animation tiles, for a total of 50 tiles.
• A 6x6-tile sprite (48x48 pixels) uses 6×6=36 tiles for its first frame, and can use another 36 for its animation tiles, for a total of 72 tiles.
• A 7x7-tile sprite (56x56 pixels) uses 7×7=49 tiles for its first frame, and can use another 49 for its animation tiles, for a total of 98 tiles.

What are "animation tiles"? Well, all graphics are composed of 8x8-pixel tiles, and only a few of them change from one frame to the next. So animations are stored in a compressed form: every tile of the first frame is kept, but then there's only one copy of each unique tile for all its animation frames. For example, Haunter's animation gets stored like this:

That takes up 56 tiles: 36 for the first frame, and 20 for the animation tiles, which is within the 72-tile limit. But here's the edited animation:

That's 78 tiles. The last 6 don't even get loaded, so they show up incorrectly when the animation plays, which is why the earlier screenshot shows white tiles where Haunter's hand is.

To see how this might be solved, let's look at BGB's VRAM viewer:

The top-left segment of VRAM (video memory) stores tiles for move animations. The middle-left is for text characters. The bottom-left is for the enemy's front sprite, the player's back sprite, and the battle interface. Finally, the bottom-right is for the enemy's animation: its entire first frame, and then the unique animation tiles. The top-right and middle-right segments are unused.

By carefully changing how the animation graphics are loaded and played, we can use the entire bottom-left and middle-left segments to allow 255-tile animations.

2. Allocate space in SRAM for animation graphics

Edit ram/wram.asm:

 SECTION "Scratch RAM", WRAMX
 
 UNION
 wScratchTilemap:: ds BG_MAP_WIDTH * BG_MAP_HEIGHT
 wScratchAttrmap:: ds BG_MAP_WIDTH * BG_MAP_HEIGHT
 
 NEXTU
-wDecompressScratch:: ds $80 tiles
-wDecompressEnemyFrontpic:: ds $80 tiles
+wDecompressScratch:: ds $100 tiles
 
 NEXTU
 ; unidentified uses
 w6_d000:: ds $1000
 ENDU

Edit ram/sram.asm:

 SECTION "Scratch", SRAM

+UNION
 sScratch:: ds $60 tiles
+
+NEXTU
+sEnemyFrontpicTileCount:: db
+sPaddedEnemyFrontpic:: ds 7 * 7 tiles
+ENDU

TODO: Explain changes.

3. Change how animations are loaded

Edit engine/gfx/load_pics.asm:

 GetMonFrontpic:
 	ld a, [wCurPartySpecies]
 	ld [wCurSpecies], a
 	call IsAPokemon
 	ret c
 	ldh a, [rSVBK]
 	push af
 	call _GetFrontpic
 	pop af
 	ldh [rSVBK], a
-	ret
+	jp CloseSRAM

 GetAnimatedFrontpic:
 	ld a, [wCurPartySpecies]
 	ld [wCurSpecies], a
 	call IsAPokemon
 	ret c
 	ldh a, [rSVBK]
 	push af
 	xor a
 	ldh [hBGMapMode], a
 	call _GetFrontpic
+	ld a, BANK(vTiles3)
+	ldh [rVBK], a
 	call GetAnimatedEnemyFrontpic
+	xor a
+	ldh [rVBK], a
 	pop af
 	ldh [rSVBK], a
-	ret
+	jp CloseSRAM

 _GetFrontpic:
+	ld a, BANK(sEnemyFrontpicTileCount)
+	call OpenSRAM
 	push de
 	call GetBaseData
 	ld a, [wBasePicSize]
 	and $f
 	ld b, a
 	push bc
 	call GetFrontpicPointer
-	ld a, BANK(wDecompressEnemyFrontpic)
+	ld a, BANK(wDecompressScratch)
 	ldh [rSVBK], a
 	ld a, b
-	ld de, wDecompressEnemyFrontpic
+	ld de, wDecompressScratch
 	call FarDecompress
+	; Save decompressed size
+	swap e
+	swap d
+	ld a, d
+	and $f0
+	or e
+	ld [sEnemyFrontpicTileCount], a
 	pop bc
-	ld hl, wDecompressScratch
-	ld de, wDecompressEnemyFrontpic
+	ld hl, sPaddedEnemyFrontpic
+	ld de, wDecompressScratch
 	call PadFrontpic
 	pop hl
 	push hl
-	ld de, wDecompressScratch
+	ld de, sPaddedEnemyFrontpic
 	ld c, 7 * 7
 	ldh a, [hROMBank]
 	ld b, a
 	call Get2bpp
 	pop hl
 	ret

 GetAnimatedEnemyFrontpic:
-	ld a, BANK(vTiles3)
-	ldh [rVBK], a
 	push hl
-	ld de, wDecompressScratch
+	ld de, sPaddedEnemyFrontpic
 	ld c, 7 * 7
 	ldh a, [hROMBank]
 	ld b, a
 	call Get2bpp
 	pop hl
 	ld de, 7 * 7 tiles
 	add hl, de
 	push hl
 	ld a, BANK(wBasePicSize)
 	ld hl, wBasePicSize
 	call GetFarWRAMByte
 	pop hl
 	and $f
-	ld de, wDecompressEnemyFrontpic + 5 * 5 tiles
+	ld de, wDecompressScratch + 5 * 5 tiles
 	ld c, 5 * 5
 	cp 5
 	jr z, .got_dims
-	ld de, wDecompressEnemyFrontpic + 6 * 6 tiles
+	ld de, wDecompressScratch + 6 * 6 tiles
 	ld c, 6 * 6
 	cp 6
 	jr z, .got_dims
-	ld de, wDecompressEnemyFrontpic + 7 * 7 tiles
+	ld de, wDecompressScratch + 7 * 7 tiles
 	ld c, 7 * 7
 .got_dims
+	; Get animation size (total tiles - base sprite size)
+	ld a, [sEnemyFrontpicTileCount]
+	sub c
+	ret z ; Return if there are no animation tiles
+	ld c, a
 	push hl
 	push bc
 	call LoadFrontpicTiles
 	pop bc
 	pop hl
 	ld de, wDecompressScratch
 	ldh a, [hROMBank]
 	ld b, a
+	; If we can load it in a single pass, just do it
+	ld a, c
+	sub 128 - 7 * 7
+	jr c, .finish
+	; Otherwise, load the first part...
+	inc a
+	ld [sEnemyFrontpicTileCount], a
+	ld c, 127 - 7 * 7
 	call Get2bpp
-	xor a
-	ldh [rVBK], a
-	ret
+	; ...then load the rest into vTiles4
+	ld de, wDecompressScratch + (127 - 7 * 7) tiles
+	ld hl, vTiles4
+	ldh a, [hROMBank]
+	ld b, a
+	ld a, [sEnemyFrontpicTileCount]
+	ld c, a
+.finish
+	jp Get2bpp

 LoadFrontpicTiles:
 	ld hl, wDecompressScratch
 	swap c
 	ld a, c
 	and $f
 	ld b, a
 	ld a, c
 	and $f0
 	ld c, a
 	push bc
 	call LoadOrientedFrontpic
 	pop bc
+	ld a, c
+	and a
+	jr z, .handle_loop
+	inc b
+	jr .handle_loop
 .loop
 	push bc
 	ld c, 0
 	call LoadOrientedFrontpic
 	pop bc
+.handle_loop
 	dec b
 	jr nz, .loop
 	ret

TODO: Explain changes.

4. Don't use tile $7F

Edit engine/gfx/pic_animation.asm:

 PokeAnim_ConvertAndApplyBitmask:
 	...
 
 .ApplyFrame:
 	push af
 	call .GetCoord
 	pop af
 	push hl
 	call .GetTilemap
 	ld hl, wPokeAnimGraphicStartTile
 	add [hl]
 	pop hl
 	ld [hl], a
+	cp $7f
+	ret c
+	inc [hl]
 	ret

TODO: Explain changes.

Now we're done!