[BUG] test_cache_reverse_order failed

[jlowe]

A recent nightly test run had a failure in test_cache_reverse_order:

[2022-03-23T12:31:02.486Z] =================================== FAILURES ===================================
[2022-03-23T12:31:02.486Z] �[31m�[1m_ test_cache_reverse_order[{'spark.sql.inMemoryColumnarStorage.enableVectorizedReader': 'false'}] _�[0m
[2022-03-23T12:31:02.486Z] 
[2022-03-23T12:31:02.486Z] enable_vectorized_conf = {'spark.sql.inMemoryColumnarStorage.enableVectorizedReader': 'false'}
[2022-03-23T12:31:02.486Z] 
[2022-03-23T12:31:02.486Z]     @pytest.mark.parametrize('enable_vectorized_conf', enable_vectorized_confs, ids=idfn)
[2022-03-23T12:31:02.486Z]     @allow_non_gpu('CollectLimitExec')
[2022-03-23T12:31:02.486Z]     def test_cache_reverse_order(enable_vectorized_conf):
[2022-03-23T12:31:02.486Z]         col0 = StructGen([['child0', StructGen([['child1', byte_gen]])]])
[2022-03-23T12:31:02.486Z]         col1 = StructGen([['child0', byte_gen]])
[2022-03-23T12:31:02.486Z]         def partial_return():
[2022-03-23T12:31:02.486Z]             def partial_return_cache(spark):
[2022-03-23T12:31:02.486Z]                 return two_col_df(spark, col0, col1).select(f.col("a"), f.col("b")).cache().limit(50).select(f.col("b"), f.col("a"))
[2022-03-23T12:31:02.486Z]             return partial_return_cache
[2022-03-23T12:31:02.486Z] >       assert_gpu_and_cpu_are_equal_collect(partial_return(), conf=enable_vectorized_conf)
[2022-03-23T12:31:02.486Z] 
[2022-03-23T12:31:02.486Z] �[1m�[31m../../src/main/python/cache_test.py�[0m:127: 
[2022-03-23T12:31:02.486Z] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
[2022-03-23T12:31:02.486Z] �[1m�[31m../../src/main/python/asserts.py�[0m:508: in assert_gpu_and_cpu_are_equal_collect
[2022-03-23T12:31:02.486Z]     _assert_gpu_and_cpu_are_equal(func, 'COLLECT', conf=conf, is_cpu_first=is_cpu_first)
[2022-03-23T12:31:02.486Z] �[1m�[31m../../src/main/python/asserts.py�[0m:439: in _assert_gpu_and_cpu_are_equal
[2022-03-23T12:31:02.486Z]     assert_equal(from_cpu, from_gpu)
[2022-03-23T12:31:02.486Z] �[1m�[31m../../src/main/python/asserts.py�[0m:106: in assert_equal
[2022-03-23T12:31:02.486Z]     _assert_equal(cpu, gpu, float_check=get_float_check(), path=[])
[2022-03-23T12:31:02.486Z] �[1m�[31m../../src/main/python/asserts.py�[0m:42: in _assert_equal
[2022-03-23T12:31:02.486Z]     _assert_equal(cpu[index], gpu[index], float_check, path + [index])
[2022-03-23T12:31:02.486Z] �[1m�[31m../../src/main/python/asserts.py�[0m:35: in _assert_equal
[2022-03-23T12:31:02.486Z]     _assert_equal(cpu[field], gpu[field], float_check, path + [field])
[2022-03-23T12:31:02.486Z] �[1m�[31m../../src/main/python/asserts.py�[0m:35: in _assert_equal
[2022-03-23T12:31:02.486Z]     _assert_equal(cpu[field], gpu[field], float_check, path + [field])
[2022-03-23T12:31:02.486Z] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
[2022-03-23T12:31:02.486Z] 
[2022-03-23T12:31:02.486Z] cpu = 126, gpu = -92
[2022-03-23T12:31:02.486Z] float_check = <function get_float_check.<locals>.<lambda> at 0x7f79d3ef83a0>
[2022-03-23T12:31:02.486Z] path = [0, 'b', 'child0']
[2022-03-23T12:31:02.486Z] 
[2022-03-23T12:31:02.486Z]     def _assert_equal(cpu, gpu, float_check, path):
[2022-03-23T12:31:02.486Z]         t = type(cpu)
[2022-03-23T12:31:02.486Z]         if (t is Row):
[2022-03-23T12:31:02.486Z]             assert len(cpu) == len(gpu), "CPU and GPU row have different lengths at {} CPU: {} GPU: {}".format(path, len(cpu), len(gpu))
[2022-03-23T12:31:02.487Z]             if hasattr(cpu, "__fields__") and hasattr(gpu, "__fields__"):
[2022-03-23T12:31:02.487Z]                 assert cpu.__fields__ == gpu.__fields__, "CPU and GPU row have different fields at {} CPU: {} GPU: {}".format(path, cpu.__fields__, gpu.__fields__)
[2022-03-23T12:31:02.487Z]                 for field in cpu.__fields__:
[2022-03-23T12:31:02.487Z]                     _assert_equal(cpu[field], gpu[field], float_check, path + [field])
[2022-03-23T12:31:02.487Z]             else:
[2022-03-23T12:31:02.487Z]                 for index in range(len(cpu)):
[2022-03-23T12:31:02.487Z]                     _assert_equal(cpu[index], gpu[index], float_check, path + [index])
[2022-03-23T12:31:02.487Z]         elif (t is list):
[2022-03-23T12:31:02.487Z]             assert len(cpu) == len(gpu), "CPU and GPU list have different lengths at {} CPU: {} GPU: {}".format(path, len(cpu), len(gpu))
[2022-03-23T12:31:02.487Z]             for index in range(len(cpu)):
[2022-03-23T12:31:02.487Z]                 _assert_equal(cpu[index], gpu[index], float_check, path + [index])
[2022-03-23T12:31:02.487Z]         elif (t is tuple):
[2022-03-23T12:31:02.487Z]             assert len(cpu) == len(gpu), "CPU and GPU list have different lengths at {} CPU: {} GPU: {}".format(path, len(cpu), len(gpu))
[2022-03-23T12:31:02.487Z]             for index in range(len(cpu)):
[2022-03-23T12:31:02.487Z]                 _assert_equal(cpu[index], gpu[index], float_check, path + [index])
[2022-03-23T12:31:02.487Z]         elif (t is pytypes.GeneratorType):
[2022-03-23T12:31:02.487Z]             index = 0
[2022-03-23T12:31:02.487Z]             # generator has no zip :( so we have to do this the hard way
[2022-03-23T12:31:02.487Z]             done = False
[2022-03-23T12:31:02.487Z]             while not done:
[2022-03-23T12:31:02.487Z]                 sub_cpu = None
[2022-03-23T12:31:02.487Z]                 sub_gpu = None
[2022-03-23T12:31:02.487Z]                 try:
[2022-03-23T12:31:02.487Z]                     sub_cpu = next(cpu)
[2022-03-23T12:31:02.487Z]                 except StopIteration:
[2022-03-23T12:31:02.487Z]                     done = True
[2022-03-23T12:31:02.487Z]     
[2022-03-23T12:31:02.487Z]                 try:
[2022-03-23T12:31:02.487Z]                     sub_gpu = next(gpu)
[2022-03-23T12:31:02.487Z]                 except StopIteration:
[2022-03-23T12:31:02.487Z]                     done = True
[2022-03-23T12:31:02.487Z]     
[2022-03-23T12:31:02.487Z]                 if done:
[2022-03-23T12:31:02.487Z]                     assert sub_cpu == sub_gpu and sub_cpu == None, "CPU and GPU generators have different lengths at {}".format(path)
[2022-03-23T12:31:02.487Z]                 else:
[2022-03-23T12:31:02.487Z]                     _assert_equal(sub_cpu, sub_gpu, float_check, path + [index])
[2022-03-23T12:31:02.487Z]     
[2022-03-23T12:31:02.487Z]                 index = index + 1
[2022-03-23T12:31:02.487Z]         elif (t is dict):
[2022-03-23T12:31:02.487Z]             # The order of key/values is not guaranteed in python dicts, nor are they guaranteed by Spark
[2022-03-23T12:31:02.487Z]             # so sort the items to do our best with ignoring the order of dicts
[2022-03-23T12:31:02.487Z]             cpu_items = list(cpu.items()).sort(key=_RowCmp)
[2022-03-23T12:31:02.487Z]             gpu_items = list(gpu.items()).sort(key=_RowCmp)
[2022-03-23T12:31:02.487Z]             _assert_equal(cpu_items, gpu_items, float_check, path + ["map"])
[2022-03-23T12:31:02.487Z]         elif (t is int):
[2022-03-23T12:31:02.487Z] >           assert cpu == gpu, "GPU and CPU int values are different at {}".format(path)
[2022-03-23T12:31:02.487Z] �[1m�[31mE           AssertionError: GPU and CPU int values are different at [0, 'b', 'child0']�[0m
[2022-03-23T12:31:02.487Z] 
[2022-03-23T12:31:02.487Z] �[1m�[31m../../src/main/python/asserts.py�[0m:77: AssertionError
[2022-03-23T12:31:02.487Z] ----------------------------- Captured stdout call -----------------------------
[2022-03-23T12:31:02.487Z] ### CPU RUN ###
[2022-03-23T12:31:02.487Z] ### GPU RUN ###
[2022-03-23T12:31:02.487Z] ### COLLECT: GPU TOOK 0.2261505126953125 CPU TOOK 0.19403505325317383 ###
[2022-03-23T12:31:02.487Z] CPU OUTPUT: [Row(b=Row(child0=126), a=Row(child0=Row(child1=-89))), Row(b=Row(child0=52), a=Row(child0=Row(child1=-61))), Row(b=Row(child0=55), a=Row(child0=Row(child1=-73))), Row(b=Row(child0=125), a=Row(child0=Row(child1=None))), Row(b=Row(child0=-31), a=Row(child0=Row(child1=-88))), Row(b=Row(child0=-1), a=Row(child0=Row(child1=-98))), Row(b=Row(child0=28), a=Row(child0=Row(child1=-13))), Row(b=Row(child0=-97), a=Row(child0=Row(child1=-51))), Row(b=Row(child0=88), a=Row(child0=Row(child1=39))), Row(b=Row(child0=40), a=Row(child0=Row(child1=-56))), Row(b=Row(child0=108), a=Row(child0=Row(child1=-124))), Row(b=Row(child0=-53), a=Row(child0=Row(child1=71))), Row(b=Row(child0=-42), a=Row(child0=Row(child1=-38))), Row(b=Row(child0=-75), a=Row(child0=Row(child1=-88))), Row(b=Row(child0=64), a=Row(child0=Row(child1=-108))), Row(b=Row(child0=106), a=Row(child0=Row(child1=-83))), Row(b=Row(child0=73), a=Row(child0=Row(child1=79))), Row(b=None, a=Row(child0=Row(child1=112))), Row(b=Row(child0=78), a=Row(child0=Row(child1=-109))), Row(b=Row(child0=20), a=Row(child0=Row(child1=-111))), Row(b=Row(child0=46), a=None), Row(b=Row(child0=-4), a=Row(child0=Row(child1=-69))), Row(b=Row(child0=-50), a=Row(child0=Row(child1=52))), Row(b=Row(child0=57), a=Row(child0=Row(child1=-102))), Row(b=Row(child0=None), a=Row(child0=Row(child1=78))), Row(b=Row(child0=117), a=Row(child0=Row(child1=-64))), Row(b=Row(child0=89), a=Row(child0=Row(child1=3))), Row(b=Row(child0=-72), a=Row(child0=Row(child1=-96))), Row(b=Row(child0=17), a=Row(child0=Row(child1=68))), Row(b=Row(child0=34), a=Row(child0=Row(child1=15))), Row(b=Row(child0=-56), a=None), Row(b=Row(child0=82), a=Row(child0=Row(child1=68))), Row(b=Row(child0=None), a=Row(child0=Row(child1=-120))), Row(b=Row(child0=-5), a=Row(child0=Row(child1=126))), Row(b=Row(child0=102), a=Row(child0=Row(child1=-89))), Row(b=None, a=Row(child0=Row(child1=-77))), Row(b=Row(child0=54), a=Row(child0=Row(child1=74))), Row(b=Row(child0=75), a=Row(child0=Row(child1=-123))), Row(b=Row(child0=110), a=Row(child0=Row(child1=71))), Row(b=Row(child0=114), a=Row(child0=Row(child1=14))), Row(b=Row(child0=-72), a=Row(child0=Row(child1=49))), Row(b=Row(child0=12), a=Row(child0=Row(child1=-116))), Row(b=Row(child0=-60), a=Row(child0=Row(child1=100))), Row(b=Row(child0=24), a=Row(child0=Row(child1=-53))), Row(b=Row(child0=-26), a=Row(child0=Row(child1=117))), Row(b=Row(child0=41), a=Row(child0=Row(child1=-57))), Row(b=Row(child0=-8), a=Row(child0=Row(child1=77))), Row(b=Row(child0=115), a=Row(child0=Row(child1=-57))), Row(b=Row(child0=3), a=Row(child0=Row(child1=-102))), Row(b=Row(child0=None), a=Row(child0=Row(child1=-76)))]
[2022-03-23T12:31:02.487Z] GPU OUTPUT: [Row(b=Row(child0=-92), a=Row(child0=Row(child1=29))), Row(b=Row(child0=114), a=Row(child0=Row(child1=-73))), Row(b=Row(child0=90), a=Row(child0=Row(child1=-65))), Row(b=Row(child0=64), a=Row(child0=Row(child1=-49))), Row(b=Row(child0=-27), a=Row(child0=None)), Row(b=Row(child0=122), a=Row(child0=Row(child1=73))), Row(b=Row(child0=None), a=Row(child0=Row(child1=-86))), Row(b=Row(child0=-114), a=Row(child0=Row(child1=55))), Row(b=Row(child0=-107), a=Row(child0=Row(child1=-68))), Row(b=Row(child0=-75), a=None), Row(b=Row(child0=121), a=Row(child0=Row(child1=-51))), Row(b=Row(child0=127), a=Row(child0=Row(child1=125))), Row(b=Row(child0=-16), a=None), Row(b=Row(child0=-53), a=Row(child0=Row(child1=34))), Row(b=Row(child0=112), a=Row(child0=Row(child1=66))), Row(b=Row(child0=25), a=Row(child0=Row(child1=-108))), Row(b=Row(child0=107), a=Row(child0=Row(child1=-46))), Row(b=Row(child0=-111), a=None), Row(b=Row(child0=-23), a=Row(child0=Row(child1=-89))), Row(b=Row(child0=None), a=Row(child0=Row(child1=118))), Row(b=Row(child0=-35), a=Row(child0=Row(child1=-118))), Row(b=Row(child0=27), a=Row(child0=Row(child1=124))), Row(b=Row(child0=18), a=Row(child0=Row(child1=-45))), Row(b=Row(child0=-87), a=Row(child0=Row(child1=118))), Row(b=Row(child0=69), a=Row(child0=Row(child1=None))), Row(b=Row(child0=37), a=Row(child0=Row(child1=-118))), Row(b=Row(child0=-72), a=Row(child0=Row(child1=68))), Row(b=Row(child0=None), a=Row(child0=Row(child1=44))), Row(b=Row(child0=-66), a=Row(child0=Row(child1=70))), Row(b=Row(child0=105), a=Row(child0=Row(child1=72))), Row(b=Row(child0=2), a=Row(child0=Row(child1=None))), Row(b=Row(child0=87), a=Row(child0=Row(child1=-18))), Row(b=Row(child0=84), a=Row(child0=Row(child1=88))), Row(b=Row(child0=85), a=Row(child0=Row(child1=-113))), Row(b=Row(child0=-92), a=Row(child0=Row(child1=None))), Row(b=Row(child0=None), a=Row(child0=Row(child1=-37))), Row(b=Row(child0=-65), a=Row(child0=Row(child1=-22))), Row(b=Row(child0=-71), a=Row(child0=Row(child1=-9))), Row(b=None, a=Row(child0=Row(child1=-114))), Row(b=Row(child0=None), a=Row(child0=Row(child1=39))), Row(b=Row(child0=33), a=Row(child0=Row(child1=-53))), Row(b=Row(child0=-61), a=Row(child0=Row(child1=62))), Row(b=Row(child0=-74), a=Row(child0=Row(child1=-126))), Row(b=Row(child0=-42), a=Row(child0=Row(child1=-72))), Row(b=Row(child0=-73), a=Row(child0=Row(child1=57))), Row(b=Row(child0=-47), a=Row(child0=Row(child1=121))), Row(b=Row(child0=-43), a=Row(child0=Row(child1=-42))), Row(b=Row(child0=7), a=Row(child0=Row(child1=-22))), Row(b=Row(child0=-84), a=Row(child0=Row(child1=97))), Row(b=Row(child0=9), a=Row(child0=Row(child1=105)))]