Skip to content

Commit

Permalink
chore(migrate): instrument with some debug logging
Browse files Browse the repository at this point in the history
  • Loading branch information
bjoerge committed Jan 30, 2024
1 parent 2dba206 commit 79c5e12
Show file tree
Hide file tree
Showing 4 changed files with 24 additions and 1 deletion.
1 change: 1 addition & 0 deletions packages/@sanity/migrate/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
"@sanity/types": "^3.23.4",
"@sanity/util": "3.26.1",
"arrify": "^2.0.1",
"debug": "^4.3.4",
"fast-fifo": "^1.3.2",
"groq-js": "^1.4.1",
"p-map": "^7.0.1"
Expand Down
3 changes: 3 additions & 0 deletions packages/@sanity/migrate/src/debug.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
import createDebug from 'debug'

export default createDebug('sanity:migrate')
15 changes: 14 additions & 1 deletion packages/@sanity/migrate/src/fs-webstream/bufferThroughFile.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import {FileHandle, open} from 'node:fs/promises'
import baseDebug from '../debug'

const debug = baseDebug.extend('bufferThroughFile')

const CHUNK_SIZE = 1024

Expand Down Expand Up @@ -30,6 +33,7 @@ export function bufferThroughFile(
let bufferDone = false

signal?.addEventListener('abort', async () => {
debug('Aborting bufferThroughFile')
await Promise.all([
writeHandle && writeHandle.close(),
readHandle && (await readHandle).close(),
Expand Down Expand Up @@ -69,6 +73,7 @@ export function bufferThroughFile(
totalBytesRead,
)
if (bytesRead === 0 && !bufferDone && !signal?.aborted) {
debug('Not enough data in buffer file, waiting for more data to be written')
// we're waiting for more data to be written to the buffer file, try again
return tryReadFromBuffer(handle)
}
Expand All @@ -80,17 +85,22 @@ export function bufferThroughFile(
function init(): Promise<void> {
if (!ready) {
ready = (async () => {
debug('Initializing bufferThroughFile')
writeHandle = await open(filename, 'w')
// start pumping data from the source stream to the buffer file
// note, don't await this, as it will block the ReadableStream.start() method
pump(source.getReader())
debug('Start buffering source stream to file')
pump(source.getReader()).then(() => {
debug('Buffering source stream to buffer file')
})
})()
}
return ready
}

function getReadHandle(): Promise<FileHandle> {
if (!readHandle) {
debug('Opening read handle on %s', filename)
readHandle = open(filename, 'r')
}
return readHandle
Expand All @@ -104,6 +114,7 @@ export function bufferThroughFile(
if (readerCount === 0 && readHandle) {
const handle = readHandle
readHandle = null
debug('Closing read handle on %s', filename)
await (await handle).close()
}
}
Expand All @@ -116,6 +127,7 @@ export function bufferThroughFile(
if (signal?.aborted) {
throw new Error('Cannot create new buffered readers on aborted stream')
}
debug('Reader started reading from file handle')
onReaderStart()
await init()
await getReadHandle()
Expand All @@ -126,6 +138,7 @@ export function bufferThroughFile(
}
const {bytesRead, buffer} = await readChunk(await readHandle)
if (bytesRead === 0 && bufferDone) {
debug('Reader done reading from file handle')
await onReaderEnd()
controller.close()
} else {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import type {FileHandle} from 'node:fs/promises'
import {open} from 'node:fs/promises'
import baseDebug from '../debug'

const debug = baseDebug.extend('readFileAsWebStream')

export function readFileAsWebStream(filename: string): ReadableStream<Uint8Array> {
const CHUNK_SIZE = 1024
Expand All @@ -9,6 +12,7 @@ export function readFileAsWebStream(filename: string): ReadableStream<Uint8Array

return new ReadableStream({
async start() {
debug('Starting readable stream from', filename)
fileHandle = await open(filename, 'r')
},
async pull(controller) {
Expand All @@ -17,6 +21,7 @@ export function readFileAsWebStream(filename: string): ReadableStream<Uint8Array
const {bytesRead} = await fileHandle.read(buffer, 0, CHUNK_SIZE, position)
if (bytesRead === 0) {
await fileHandle.close()
debug('Closing readable stream from', filename)
controller.close()
} else {
position += bytesRead
Expand All @@ -25,6 +30,7 @@ export function readFileAsWebStream(filename: string): ReadableStream<Uint8Array
},

cancel() {
debug('Cancelling readable stream from', filename)
return fileHandle.close()
},
})
Expand Down

0 comments on commit 79c5e12

Please sign in to comment.