Skip to content

Commit 27a42c7

Browse files
JoshRosenJoshRosen
committed
[SPARK-10359] Enumerate dependencies in a file and diff against it for new pull requests
This patch adds a new build check which enumerates Spark's resolved runtime classpath and saves it to a file, then diffs against that file to detect whether pull requests have introduced dependency changes. The aim of this check is to make it simpler to reason about whether pull request which modify the build have introduced new dependencies or changed transitive dependencies in a way that affects the final classpath. This supplants the checks added in SPARK-4123 / apache#5093, which are currently disabled due to bugs. This patch is based on pwendell's work in apache#8531. Closes apache#8531. Author: Josh Rosen <joshrosen@databricks.com> Author: Patrick Wendell <patrick@databricks.com> Closes apache#10461 from JoshRosen/SPARK-10359.
1 parent d1ca634 commit 27a42c7

10 files changed

+512
-120
lines changed

‎.rat-excludes

+1
Original file line numberDiff line numberDiff line change
@@ -85,3 +85,4 @@ org.apache.spark.sql.sources.DataSourceRegister
8585
org.apache.spark.scheduler.SparkHistoryListenerFactory
8686
.*parquet
8787
LZ4BlockInputStream.java
88+
spark-deps-.*

‎dev/deps/spark-deps-hadoop-2.3

+184
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
JavaEWAH-0.3.2.jar
2+
RoaringBitmap-0.5.11.jar
3+
ST4-4.0.4.jar
4+
activation-1.1.1.jar
5+
akka-actor_2.10-2.3.11.jar
6+
akka-remote_2.10-2.3.11.jar
7+
akka-slf4j_2.10-2.3.11.jar
8+
antlr-2.7.7.jar
9+
antlr-runtime-3.4.jar
10+
aopalliance-1.0.jar
11+
apache-log4j-extras-1.2.17.jar
12+
arpack_combined_all-0.1.jar
13+
asm-3.1.jar
14+
asm-commons-3.1.jar
15+
asm-tree-3.1.jar
16+
avro-1.7.7.jar
17+
avro-ipc-1.7.7-tests.jar
18+
avro-ipc-1.7.7.jar
19+
avro-mapred-1.7.7-hadoop2.jar
20+
base64-2.3.8.jar
21+
bcprov-jdk15on-1.51.jar
22+
bonecp-0.8.0.RELEASE.jar
23+
breeze-macros_2.10-0.11.2.jar
24+
breeze_2.10-0.11.2.jar
25+
calcite-avatica-1.2.0-incubating.jar
26+
calcite-core-1.2.0-incubating.jar
27+
calcite-linq4j-1.2.0-incubating.jar
28+
chill-java-0.5.0.jar
29+
chill_2.10-0.5.0.jar
30+
commons-beanutils-1.7.0.jar
31+
commons-beanutils-core-1.8.0.jar
32+
commons-cli-1.2.jar
33+
commons-codec-1.10.jar
34+
commons-collections-3.2.2.jar
35+
commons-compiler-2.7.6.jar
36+
commons-compress-1.4.1.jar
37+
commons-configuration-1.6.jar
38+
commons-dbcp-1.4.jar
39+
commons-digester-1.8.jar
40+
commons-httpclient-3.1.jar
41+
commons-io-2.4.jar
42+
commons-lang-2.6.jar
43+
commons-lang3-3.3.2.jar
44+
commons-logging-1.1.3.jar
45+
commons-math3-3.4.1.jar
46+
commons-net-2.2.jar
47+
commons-pool-1.5.4.jar
48+
compress-lzf-1.0.3.jar
49+
config-1.2.1.jar
50+
core-1.1.2.jar
51+
curator-client-2.4.0.jar
52+
curator-framework-2.4.0.jar
53+
curator-recipes-2.4.0.jar
54+
datanucleus-api-jdo-3.2.6.jar
55+
datanucleus-core-3.2.10.jar
56+
datanucleus-rdbms-3.2.9.jar
57+
derby-10.10.1.1.jar
58+
eigenbase-properties-1.1.5.jar
59+
geronimo-annotation_1.0_spec-1.1.1.jar
60+
geronimo-jaspic_1.0_spec-1.0.jar
61+
geronimo-jta_1.1_spec-1.1.1.jar
62+
groovy-all-2.1.6.jar
63+
guice-3.0.jar
64+
guice-servlet-3.0.jar
65+
hadoop-annotations-2.3.0.jar
66+
hadoop-auth-2.3.0.jar
67+
hadoop-client-2.3.0.jar
68+
hadoop-common-2.3.0.jar
69+
hadoop-hdfs-2.3.0.jar
70+
hadoop-mapreduce-client-app-2.3.0.jar
71+
hadoop-mapreduce-client-common-2.3.0.jar
72+
hadoop-mapreduce-client-core-2.3.0.jar
73+
hadoop-mapreduce-client-jobclient-2.3.0.jar
74+
hadoop-mapreduce-client-shuffle-2.3.0.jar
75+
hadoop-yarn-api-2.3.0.jar
76+
hadoop-yarn-client-2.3.0.jar
77+
hadoop-yarn-common-2.3.0.jar
78+
hadoop-yarn-server-common-2.3.0.jar
79+
hadoop-yarn-server-web-proxy-2.3.0.jar
80+
httpclient-4.3.2.jar
81+
httpcore-4.3.2.jar
82+
ivy-2.4.0.jar
83+
jackson-annotations-2.4.4.jar
84+
jackson-core-2.4.4.jar
85+
jackson-core-asl-1.9.13.jar
86+
jackson-databind-2.4.4.jar
87+
jackson-jaxrs-1.9.13.jar
88+
jackson-mapper-asl-1.9.13.jar
89+
jackson-module-scala_2.10-2.4.4.jar
90+
jackson-xc-1.9.13.jar
91+
janino-2.7.8.jar
92+
jansi-1.4.jar
93+
java-xmlbuilder-1.0.jar
94+
javax.inject-1.jar
95+
javax.servlet-3.0.0.v201112011016.jar
96+
javolution-5.5.1.jar
97+
jaxb-api-2.2.2.jar
98+
jaxb-impl-2.2.3-1.jar
99+
jcl-over-slf4j-1.7.10.jar
100+
jdo-api-3.0.1.jar
101+
jersey-core-1.9.jar
102+
jersey-guice-1.9.jar
103+
jersey-json-1.9.jar
104+
jersey-server-1.9.jar
105+
jets3t-0.9.3.jar
106+
jettison-1.1.jar
107+
jetty-6.1.26.jar
108+
jetty-all-7.6.0.v20120127.jar
109+
jetty-util-6.1.26.jar
110+
jline-2.10.5.jar
111+
jline-2.12.jar
112+
joda-time-2.9.jar
113+
jodd-core-3.5.2.jar
114+
jpam-1.1.jar
115+
json-20090211.jar
116+
json4s-ast_2.10-3.2.10.jar
117+
json4s-core_2.10-3.2.10.jar
118+
json4s-jackson_2.10-3.2.10.jar
119+
jsr305-1.3.9.jar
120+
jta-1.1.jar
121+
jtransforms-2.4.0.jar
122+
jul-to-slf4j-1.7.10.jar
123+
kryo-2.21.jar
124+
leveldbjni-all-1.8.jar
125+
libfb303-0.9.2.jar
126+
libthrift-0.9.2.jar
127+
log4j-1.2.17.jar
128+
lz4-1.3.0.jar
129+
mail-1.4.7.jar
130+
mesos-0.21.1-shaded-protobuf.jar
131+
metrics-core-3.1.2.jar
132+
metrics-graphite-3.1.2.jar
133+
metrics-json-3.1.2.jar
134+
metrics-jvm-3.1.2.jar
135+
minlog-1.2.jar
136+
mx4j-3.0.2.jar
137+
netty-3.8.0.Final.jar
138+
netty-all-4.0.29.Final.jar
139+
objenesis-1.2.jar
140+
opencsv-2.3.jar
141+
oro-2.0.8.jar
142+
paranamer-2.6.jar
143+
parquet-column-1.7.0.jar
144+
parquet-common-1.7.0.jar
145+
parquet-encoding-1.7.0.jar
146+
parquet-format-2.3.0-incubating.jar
147+
parquet-generator-1.7.0.jar
148+
parquet-hadoop-1.7.0.jar
149+
parquet-hadoop-bundle-1.6.0.jar
150+
parquet-jackson-1.7.0.jar
151+
pmml-agent-1.2.7.jar
152+
pmml-model-1.2.7.jar
153+
pmml-schema-1.2.7.jar
154+
protobuf-java-2.5.0.jar
155+
py4j-0.9.jar
156+
pyrolite-4.9.jar
157+
quasiquotes_2.10-2.0.0-M8.jar
158+
reflectasm-1.07-shaded.jar
159+
scala-compiler-2.10.5.jar
160+
scala-library-2.10.5.jar
161+
scala-reflect-2.10.5.jar
162+
scalap-2.10.5.jar
163+
servlet-api-2.5.jar
164+
slf4j-api-1.7.10.jar
165+
slf4j-log4j12-1.7.10.jar
166+
snappy-0.2.jar
167+
snappy-java-1.1.2.jar
168+
spire-macros_2.10-0.7.4.jar
169+
spire_2.10-0.7.4.jar
170+
stax-api-1.0-2.jar
171+
stax-api-1.0.1.jar
172+
stream-2.7.0.jar
173+
stringtemplate-3.2.1.jar
174+
super-csv-2.2.0.jar
175+
tachyon-client-0.8.2.jar
176+
tachyon-underfs-hdfs-0.8.2.jar
177+
tachyon-underfs-local-0.8.2.jar
178+
tachyon-underfs-s3-0.8.2.jar
179+
uncommons-maths-1.2.2a.jar
180+
unused-1.0.0.jar
181+
xbean-asm5-shaded-4.4.jar
182+
xmlenc-0.52.jar
183+
xz-1.0.jar
184+
zookeeper-3.4.5.jar

‎dev/deps/spark-deps-hadoop-2.4

+185
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
JavaEWAH-0.3.2.jar
2+
RoaringBitmap-0.5.11.jar
3+
ST4-4.0.4.jar
4+
activation-1.1.1.jar
5+
akka-actor_2.10-2.3.11.jar
6+
akka-remote_2.10-2.3.11.jar
7+
akka-slf4j_2.10-2.3.11.jar
8+
antlr-2.7.7.jar
9+
antlr-runtime-3.4.jar
10+
aopalliance-1.0.jar
11+
apache-log4j-extras-1.2.17.jar
12+
arpack_combined_all-0.1.jar
13+
asm-3.1.jar
14+
asm-commons-3.1.jar
15+
asm-tree-3.1.jar
16+
avro-1.7.7.jar
17+
avro-ipc-1.7.7-tests.jar
18+
avro-ipc-1.7.7.jar
19+
avro-mapred-1.7.7-hadoop2.jar
20+
base64-2.3.8.jar
21+
bcprov-jdk15on-1.51.jar
22+
bonecp-0.8.0.RELEASE.jar
23+
breeze-macros_2.10-0.11.2.jar
24+
breeze_2.10-0.11.2.jar
25+
calcite-avatica-1.2.0-incubating.jar
26+
calcite-core-1.2.0-incubating.jar
27+
calcite-linq4j-1.2.0-incubating.jar
28+
chill-java-0.5.0.jar
29+
chill_2.10-0.5.0.jar
30+
commons-beanutils-1.7.0.jar
31+
commons-beanutils-core-1.8.0.jar
32+
commons-cli-1.2.jar
33+
commons-codec-1.10.jar
34+
commons-collections-3.2.2.jar
35+
commons-compiler-2.7.6.jar
36+
commons-compress-1.4.1.jar
37+
commons-configuration-1.6.jar
38+
commons-dbcp-1.4.jar
39+
commons-digester-1.8.jar
40+
commons-httpclient-3.1.jar
41+
commons-io-2.4.jar
42+
commons-lang-2.6.jar
43+
commons-lang3-3.3.2.jar
44+
commons-logging-1.1.3.jar
45+
commons-math3-3.4.1.jar
46+
commons-net-2.2.jar
47+
commons-pool-1.5.4.jar
48+
compress-lzf-1.0.3.jar
49+
config-1.2.1.jar
50+
core-1.1.2.jar
51+
curator-client-2.4.0.jar
52+
curator-framework-2.4.0.jar
53+
curator-recipes-2.4.0.jar
54+
datanucleus-api-jdo-3.2.6.jar
55+
datanucleus-core-3.2.10.jar
56+
datanucleus-rdbms-3.2.9.jar
57+
derby-10.10.1.1.jar
58+
eigenbase-properties-1.1.5.jar
59+
geronimo-annotation_1.0_spec-1.1.1.jar
60+
geronimo-jaspic_1.0_spec-1.0.jar
61+
geronimo-jta_1.1_spec-1.1.1.jar
62+
groovy-all-2.1.6.jar
63+
guice-3.0.jar
64+
guice-servlet-3.0.jar
65+
hadoop-annotations-2.4.0.jar
66+
hadoop-auth-2.4.0.jar
67+
hadoop-client-2.4.0.jar
68+
hadoop-common-2.4.0.jar
69+
hadoop-hdfs-2.4.0.jar
70+
hadoop-mapreduce-client-app-2.4.0.jar
71+
hadoop-mapreduce-client-common-2.4.0.jar
72+
hadoop-mapreduce-client-core-2.4.0.jar
73+
hadoop-mapreduce-client-jobclient-2.4.0.jar
74+
hadoop-mapreduce-client-shuffle-2.4.0.jar
75+
hadoop-yarn-api-2.4.0.jar
76+
hadoop-yarn-client-2.4.0.jar
77+
hadoop-yarn-common-2.4.0.jar
78+
hadoop-yarn-server-common-2.4.0.jar
79+
hadoop-yarn-server-web-proxy-2.4.0.jar
80+
httpclient-4.3.2.jar
81+
httpcore-4.3.2.jar
82+
ivy-2.4.0.jar
83+
jackson-annotations-2.4.4.jar
84+
jackson-core-2.4.4.jar
85+
jackson-core-asl-1.9.13.jar
86+
jackson-databind-2.4.4.jar
87+
jackson-jaxrs-1.9.13.jar
88+
jackson-mapper-asl-1.9.13.jar
89+
jackson-module-scala_2.10-2.4.4.jar
90+
jackson-xc-1.9.13.jar
91+
janino-2.7.8.jar
92+
jansi-1.4.jar
93+
java-xmlbuilder-1.0.jar
94+
javax.inject-1.jar
95+
javax.servlet-3.0.0.v201112011016.jar
96+
javolution-5.5.1.jar
97+
jaxb-api-2.2.2.jar
98+
jaxb-impl-2.2.3-1.jar
99+
jcl-over-slf4j-1.7.10.jar
100+
jdo-api-3.0.1.jar
101+
jersey-client-1.9.jar
102+
jersey-core-1.9.jar
103+
jersey-guice-1.9.jar
104+
jersey-json-1.9.jar
105+
jersey-server-1.9.jar
106+
jets3t-0.9.3.jar
107+
jettison-1.1.jar
108+
jetty-6.1.26.jar
109+
jetty-all-7.6.0.v20120127.jar
110+
jetty-util-6.1.26.jar
111+
jline-2.10.5.jar
112+
jline-2.12.jar
113+
joda-time-2.9.jar
114+
jodd-core-3.5.2.jar
115+
jpam-1.1.jar
116+
json-20090211.jar
117+
json4s-ast_2.10-3.2.10.jar
118+
json4s-core_2.10-3.2.10.jar
119+
json4s-jackson_2.10-3.2.10.jar
120+
jsr305-1.3.9.jar
121+
jta-1.1.jar
122+
jtransforms-2.4.0.jar
123+
jul-to-slf4j-1.7.10.jar
124+
kryo-2.21.jar
125+
leveldbjni-all-1.8.jar
126+
libfb303-0.9.2.jar
127+
libthrift-0.9.2.jar
128+
log4j-1.2.17.jar
129+
lz4-1.3.0.jar
130+
mail-1.4.7.jar
131+
mesos-0.21.1-shaded-protobuf.jar
132+
metrics-core-3.1.2.jar
133+
metrics-graphite-3.1.2.jar
134+
metrics-json-3.1.2.jar
135+
metrics-jvm-3.1.2.jar
136+
minlog-1.2.jar
137+
mx4j-3.0.2.jar
138+
netty-3.8.0.Final.jar
139+
netty-all-4.0.29.Final.jar
140+
objenesis-1.2.jar
141+
opencsv-2.3.jar
142+
oro-2.0.8.jar
143+
paranamer-2.6.jar
144+
parquet-column-1.7.0.jar
145+
parquet-common-1.7.0.jar
146+
parquet-encoding-1.7.0.jar
147+
parquet-format-2.3.0-incubating.jar
148+
parquet-generator-1.7.0.jar
149+
parquet-hadoop-1.7.0.jar
150+
parquet-hadoop-bundle-1.6.0.jar
151+
parquet-jackson-1.7.0.jar
152+
pmml-agent-1.2.7.jar
153+
pmml-model-1.2.7.jar
154+
pmml-schema-1.2.7.jar
155+
protobuf-java-2.5.0.jar
156+
py4j-0.9.jar
157+
pyrolite-4.9.jar
158+
quasiquotes_2.10-2.0.0-M8.jar
159+
reflectasm-1.07-shaded.jar
160+
scala-compiler-2.10.5.jar
161+
scala-library-2.10.5.jar
162+
scala-reflect-2.10.5.jar
163+
scalap-2.10.5.jar
164+
servlet-api-2.5.jar
165+
slf4j-api-1.7.10.jar
166+
slf4j-log4j12-1.7.10.jar
167+
snappy-0.2.jar
168+
snappy-java-1.1.2.jar
169+
spire-macros_2.10-0.7.4.jar
170+
spire_2.10-0.7.4.jar
171+
stax-api-1.0-2.jar
172+
stax-api-1.0.1.jar
173+
stream-2.7.0.jar
174+
stringtemplate-3.2.1.jar
175+
super-csv-2.2.0.jar
176+
tachyon-client-0.8.2.jar
177+
tachyon-underfs-hdfs-0.8.2.jar
178+
tachyon-underfs-local-0.8.2.jar
179+
tachyon-underfs-s3-0.8.2.jar
180+
uncommons-maths-1.2.2a.jar
181+
unused-1.0.0.jar
182+
xbean-asm5-shaded-4.4.jar
183+
xmlenc-0.52.jar
184+
xz-1.0.jar
185+
zookeeper-3.4.5.jar

‎dev/run-tests-jenkins.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ def run_tests(tests_timeout):
124124
ERROR_CODES["BLOCK_R_STYLE"]: 'R style tests',
125125
ERROR_CODES["BLOCK_DOCUMENTATION"]: 'to generate documentation',
126126
ERROR_CODES["BLOCK_BUILD"]: 'to build',
127+
ERROR_CODES["BLOCK_BUILD_TESTS"]: 'build dependency tests',
127128
ERROR_CODES["BLOCK_MIMA"]: 'MiMa tests',
128129
ERROR_CODES["BLOCK_SPARK_UNIT_TESTS"]: 'Spark unit tests',
129130
ERROR_CODES["BLOCK_PYSPARK_UNIT_TESTS"]: 'PySpark unit tests',
@@ -193,7 +194,6 @@ def main():
193194
pr_tests = [
194195
"pr_merge_ability",
195196
"pr_public_classes"
196-
# DISABLED (pwendell) "pr_new_dependencies"
197197
]
198198

199199
# `bind_message_base` returns a function to generate messages for Github posting

0 commit comments

Comments
 (0)