Skip to content

Commit

Permalink
f add --ms_delay flag.
Browse files Browse the repository at this point in the history
use `ms_delay` to pause between spanner transaction `--readchunk`s. This allows
some primative throttling for feeding spanner data.
Reminder: `readchunk` sets the max number of items to try to write per chunk
to spanner in any given transaction, default value 1000.
  • Loading branch information
jrconlin committed Apr 2, 2020
1 parent 08a646a commit 55edc74
Showing 1 changed file with 11 additions and 2 deletions.
13 changes: 11 additions & 2 deletions tools/user_migration/migrate_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -416,7 +416,7 @@ def spanner_transact_bso(transaction, data, fxa_kid, fxa_uid, args):
logging.info("Skipped {} of {} rows for {}".format(
abort_count, col_count, abort_col
))
for bunch in divvy(data, args.readchunk or 1000):
for bunch in divvy(data, args.chunk or 1000):
# Occasionally, there is a batch fail because a
# user collection is not found before a bso is written.
# to solve that, divide the UC updates from the
Expand All @@ -436,6 +436,10 @@ def spanner_transact_bso(transaction, data, fxa_kid, fxa_uid, args):
fxa_uid,
args,
)
if args.ms_delay > 0:
logging.debug(
"Sleeping for {} seconds".format(args.ms_delay * .01))
time.sleep(args.ms_delay * .01)

except AlreadyExists:
logging.warn(
Expand Down Expand Up @@ -584,7 +588,8 @@ def get_args():
help="skip user_collections table"
)
parser.add_argument(
'--readchunk',
'--write_chunk',
dest="chunk",
default=1000,
help="how many rows per transaction for spanner (default: 1000)"
)
Expand Down Expand Up @@ -617,6 +622,10 @@ def get_args():
'--sort_users', action="store_true",
help="Sort the user"
)
parser.add_argument(
'--ms_delay', type=int, default=0,
help="inject a sleep between writes to spanner as a throttle"
)

return parser.parse_args()

Expand Down

0 comments on commit 55edc74

Please sign in to comment.