test-cluster.sh

#!/bin/sh

EXPECTED="sum of 32 histograms: [13987347, 444344, 464807, 494154, 527579, 555977, 593278, 644168, 690073, 770120, 848826, 952755, 1068455, 1250493, 1478754, 1788388, 2219113, 2838449, 3688682, 4971856, 6958251, 9388021, 12139686, 14918153, 17122337, 18691241, 19450632, 18456306, 15773278, 13290033, 11084996, 9294330, 7862234, 6677863, 5596762, 4811285, 4223898, 3767657, 3392701, 3085925, 2825714, 2606245, 2415371, 2249726, 2101740, 1976889, 1864298, 1763432, 1673587, 1592520, 1524162, 1454708, 1395702, 1337306, 1286334, 1240016, 1198095, 1157505, 1114038, 1078578, 1041854, 1010797, 980160, 949193, 921719, 894498, 868124, 844549, 821385, 797778, 777626, 756585, 739546, 719303, 702629, 686788, 672178, 656845, 642054, 630727, 617809, 606337, 588144, 592751, 577131, 568410, 559833, 551149, 546390, 537658, 532490, 527737, 522543, 517403, 514780, 510708, 509402, 504667, 504347, 501452, 501022, 498993, 496618, 499080, 494893, 496271, 496981, 501359, 498638, 499648, 502304, 503822, 506880, 505076, 508585, 510595, 513209, 517282, 515593, 519653, 522123, 522967, 527403, 526449, 529533, 531740, 533610, 535644, 537753, 541651, 542642, 544699, 549667, 545781, 551688, 553182, 555719, 559029, 561574, 563335, 568614, 568994, 571846, 573982, 576911, 577061, 580854, 583890, 584138, 585757, 585585, 587234, 586720, 586443, 584638, 585328, 583351, 574726, 585724, 576711, 573396, 570039, 567335, 562765, 560190, 556672, 551271, 548326, 541820, 537532, 532744, 526700, 518283, 513899, 504854, 497246, 489277, 482811, 472478, 461958, 452171, 443635, 433375, 419722, 410872, 396276, 386211, 371959, 361459, 347577, 334223, 320171, 307197, 293054, 278741, 265837, 252728, 239052, 227954, 211128, 199779, 188097, 175408, 164005, 152456, 142572, 131995, 116189, 118536, 103910, 96096, 87228, 80807, 73587, 67132, 60995, 55463, 50232, 45688, 40837, 36909, 32988, 28985, 26859, 23144, 21387, 18833, 17185, 15714, 13929, 12538, 11589, 10763, 10003, 9350, 8651, 8117, 7627, 7360, 6768, 6534, 6198, 5990, 5682, 5378, 4994, 4925, 4632, 4439, 4299, 154313, 0, 0, 0, 0, 1601317]"

APPLICATION_ID=$(sh run-cluster 2>&1 | tee /dev/stderr | grep 'Submitted application' | awk '{print $NF}')
ACTUAL=$(yarn logs -applicationId $APPLICATION_ID | grep 'sum of')

if [ "$EXPECTED" != "$ACTUAL" ]; then
  echo -e "expected: $EXPECTED,\nactual: $ACTUAL"
  exit 1
fi