Skip to content

Commit

Permalink
Update Spark to 3.3.1
Browse files Browse the repository at this point in the history
## Description

Spark 3.3.1 released. This PR upgrade spark to Spark 3.3.1.

Existing test.

## Does this PR introduce _any_ user-facing changes?

No.

Closes #1382

Signed-off-by: Shixiong Zhu <zsxwing@gmail.com>
GitOrigin-RevId: 75413de8536a3a083ec59a67f4715bd269e9415b
  • Loading branch information
wangyum authored and scottsand-db committed Nov 9, 2022
1 parent 518827a commit 529313d
Show file tree
Hide file tree
Showing 3 changed files with 4 additions and 4 deletions.
2 changes: 1 addition & 1 deletion build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import java.nio.file.Files
import TestParallelization._

val sparkVersion = "3.3.0"
val sparkVersion = "3.3.1"
val scala212 = "2.12.15"
val scala213 = "2.13.5"
val default_scala_version = scala212
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -933,7 +933,8 @@ abstract class MergeIntoSuiteBase

append(Seq((100, 100), (3, 5)).toDF("key2", "value"))
// cache is in effect, as the above change is not reflected
checkAnswer(spark.table(s"delta.`$tempPath`"), Row(2, 2) :: Row(1, 4) :: Nil)
checkAnswer(spark.table(s"delta.`$tempPath`"),
Row(2, 2) :: Row(1, 4) :: Row(100, 100) :: Row(3, 5) :: Nil)

executeMerge(
target = s"delta.`$tempPath` as trgNew",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,7 @@ class StatsCollectionSuite
val statsDf = statsDF(deltaLog)
assert(statsDf.where('numRecords.isNotNull).count() > 0)
// scalastyle:off line.size.limit
// TODO(delta-lake-oss): Fix test on migrating to Spark 3.4
val expectedStats = Seq(Row(4, Row(10, 20), Row(17, 27)), Row(2, Row(11, 21), Row(19, 29)), Row(4, Row(12, 22), Row(18, 28)))
val expectedStats = Seq(Row(3, Row(10, 20), Row(19, 29)), Row(4, Row(12, 22), Row(17, 27)), Row(3, Row(11, 21), Row(18, 28)))
// scalastyle:on line.size.limit
checkAnswer(statsDf, expectedStats)
}
Expand Down

0 comments on commit 529313d

Please # to comment.