Skip to content

Commit 0d165ec

Browse files
JoshRosenJoshRosen
committed
[SPARK-12612][PROJECT-INFRA] Add missing Hadoop profiles to dev/run-tests-*.py scripts and dev/deps
There are a couple of places in the `dev/run-tests-*.py` scripts which deal with Hadoop profiles, but the set of profiles that they handle does not include all Hadoop profiles defined in our POM. Similarly, the `hadoop-2.2` and `hadoop-2.6` profiles were missing from `dev/deps`. This patch updates these scripts to include all four Hadoop profiles defined in our POM. Author: Josh Rosen <joshrosen@databricks.com> Closes apache#10565 from JoshRosen/add-missing-hadoop-profiles-in-test-scripts.
1 parent 84f8492 commit 0d165ec

5 files changed

+394
-2
lines changed

‎dev/deps/spark-deps-hadoop-2.2

+193
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
JavaEWAH-0.3.2.jar
2+
RoaringBitmap-0.5.11.jar
3+
ST4-4.0.4.jar
4+
activation-1.1.jar
5+
akka-actor_2.10-2.3.11.jar
6+
akka-remote_2.10-2.3.11.jar
7+
akka-slf4j_2.10-2.3.11.jar
8+
antlr-2.7.7.jar
9+
antlr-runtime-3.4.jar
10+
aopalliance-1.0.jar
11+
apache-log4j-extras-1.2.17.jar
12+
arpack_combined_all-0.1.jar
13+
asm-3.1.jar
14+
asm-commons-3.1.jar
15+
asm-tree-3.1.jar
16+
avro-1.7.7.jar
17+
avro-ipc-1.7.7-tests.jar
18+
avro-ipc-1.7.7.jar
19+
avro-mapred-1.7.7-hadoop2.jar
20+
bonecp-0.8.0.RELEASE.jar
21+
breeze-macros_2.10-0.11.2.jar
22+
breeze_2.10-0.11.2.jar
23+
calcite-avatica-1.2.0-incubating.jar
24+
calcite-core-1.2.0-incubating.jar
25+
calcite-linq4j-1.2.0-incubating.jar
26+
chill-java-0.5.0.jar
27+
chill_2.10-0.5.0.jar
28+
commons-beanutils-1.7.0.jar
29+
commons-beanutils-core-1.8.0.jar
30+
commons-cli-1.2.jar
31+
commons-codec-1.10.jar
32+
commons-collections-3.2.2.jar
33+
commons-compiler-2.7.6.jar
34+
commons-compress-1.4.1.jar
35+
commons-configuration-1.6.jar
36+
commons-dbcp-1.4.jar
37+
commons-digester-1.8.jar
38+
commons-httpclient-3.1.jar
39+
commons-io-2.4.jar
40+
commons-lang-2.6.jar
41+
commons-lang3-3.3.2.jar
42+
commons-logging-1.1.3.jar
43+
commons-math-2.1.jar
44+
commons-math3-3.4.1.jar
45+
commons-net-2.2.jar
46+
commons-pool-1.5.4.jar
47+
compress-lzf-1.0.3.jar
48+
config-1.2.1.jar
49+
core-1.1.2.jar
50+
curator-client-2.4.0.jar
51+
curator-framework-2.4.0.jar
52+
curator-recipes-2.4.0.jar
53+
datanucleus-api-jdo-3.2.6.jar
54+
datanucleus-core-3.2.10.jar
55+
datanucleus-rdbms-3.2.9.jar
56+
derby-10.10.1.1.jar
57+
eigenbase-properties-1.1.5.jar
58+
geronimo-annotation_1.0_spec-1.1.1.jar
59+
geronimo-jaspic_1.0_spec-1.0.jar
60+
geronimo-jta_1.1_spec-1.1.1.jar
61+
gmbal-api-only-3.0.0-b023.jar
62+
grizzly-framework-2.1.2.jar
63+
grizzly-http-2.1.2.jar
64+
grizzly-http-server-2.1.2.jar
65+
grizzly-http-servlet-2.1.2.jar
66+
grizzly-rcm-2.1.2.jar
67+
groovy-all-2.1.6.jar
68+
guice-3.0.jar
69+
guice-servlet-3.0.jar
70+
hadoop-annotations-2.2.0.jar
71+
hadoop-auth-2.2.0.jar
72+
hadoop-client-2.2.0.jar
73+
hadoop-common-2.2.0.jar
74+
hadoop-hdfs-2.2.0.jar
75+
hadoop-mapreduce-client-app-2.2.0.jar
76+
hadoop-mapreduce-client-common-2.2.0.jar
77+
hadoop-mapreduce-client-core-2.2.0.jar
78+
hadoop-mapreduce-client-jobclient-2.2.0.jar
79+
hadoop-mapreduce-client-shuffle-2.2.0.jar
80+
hadoop-yarn-api-2.2.0.jar
81+
hadoop-yarn-client-2.2.0.jar
82+
hadoop-yarn-common-2.2.0.jar
83+
hadoop-yarn-server-common-2.2.0.jar
84+
hadoop-yarn-server-web-proxy-2.2.0.jar
85+
httpclient-4.3.2.jar
86+
httpcore-4.3.2.jar
87+
ivy-2.4.0.jar
88+
jackson-annotations-2.4.4.jar
89+
jackson-core-2.4.4.jar
90+
jackson-core-asl-1.9.13.jar
91+
jackson-databind-2.4.4.jar
92+
jackson-jaxrs-1.9.13.jar
93+
jackson-mapper-asl-1.9.13.jar
94+
jackson-module-scala_2.10-2.4.4.jar
95+
jackson-xc-1.9.13.jar
96+
janino-2.7.8.jar
97+
jansi-1.4.jar
98+
javax.inject-1.jar
99+
javax.servlet-3.0.0.v201112011016.jar
100+
javax.servlet-3.1.jar
101+
javax.servlet-api-3.0.1.jar
102+
javolution-5.5.1.jar
103+
jaxb-api-2.2.2.jar
104+
jaxb-impl-2.2.3-1.jar
105+
jcl-over-slf4j-1.7.10.jar
106+
jdo-api-3.0.1.jar
107+
jersey-client-1.9.jar
108+
jersey-core-1.9.jar
109+
jersey-grizzly2-1.9.jar
110+
jersey-guice-1.9.jar
111+
jersey-json-1.9.jar
112+
jersey-server-1.9.jar
113+
jersey-test-framework-core-1.9.jar
114+
jersey-test-framework-grizzly2-1.9.jar
115+
jets3t-0.7.1.jar
116+
jettison-1.1.jar
117+
jetty-all-7.6.0.v20120127.jar
118+
jetty-util-6.1.26.jar
119+
jline-2.10.5.jar
120+
jline-2.12.jar
121+
joda-time-2.9.jar
122+
jodd-core-3.5.2.jar
123+
jpam-1.1.jar
124+
json-20090211.jar
125+
json4s-ast_2.10-3.2.10.jar
126+
json4s-core_2.10-3.2.10.jar
127+
json4s-jackson_2.10-3.2.10.jar
128+
jsr305-1.3.9.jar
129+
jta-1.1.jar
130+
jtransforms-2.4.0.jar
131+
jul-to-slf4j-1.7.10.jar
132+
kryo-2.21.jar
133+
leveldbjni-all-1.8.jar
134+
libfb303-0.9.2.jar
135+
libthrift-0.9.2.jar
136+
log4j-1.2.17.jar
137+
lz4-1.3.0.jar
138+
mail-1.4.1.jar
139+
management-api-3.0.0-b012.jar
140+
mesos-0.21.1-shaded-protobuf.jar
141+
metrics-core-3.1.2.jar
142+
metrics-graphite-3.1.2.jar
143+
metrics-json-3.1.2.jar
144+
metrics-jvm-3.1.2.jar
145+
minlog-1.2.jar
146+
netty-3.8.0.Final.jar
147+
netty-all-4.0.29.Final.jar
148+
objenesis-1.2.jar
149+
opencsv-2.3.jar
150+
oro-2.0.8.jar
151+
paranamer-2.6.jar
152+
parquet-column-1.7.0.jar
153+
parquet-common-1.7.0.jar
154+
parquet-encoding-1.7.0.jar
155+
parquet-format-2.3.0-incubating.jar
156+
parquet-generator-1.7.0.jar
157+
parquet-hadoop-1.7.0.jar
158+
parquet-hadoop-bundle-1.6.0.jar
159+
parquet-jackson-1.7.0.jar
160+
pmml-agent-1.2.7.jar
161+
pmml-model-1.2.7.jar
162+
pmml-schema-1.2.7.jar
163+
protobuf-java-2.5.0.jar
164+
py4j-0.9.jar
165+
pyrolite-4.9.jar
166+
quasiquotes_2.10-2.0.0-M8.jar
167+
reflectasm-1.07-shaded.jar
168+
scala-compiler-2.10.5.jar
169+
scala-library-2.10.5.jar
170+
scala-reflect-2.10.5.jar
171+
scalap-2.10.5.jar
172+
servlet-api-2.5.jar
173+
slf4j-api-1.7.10.jar
174+
slf4j-log4j12-1.7.10.jar
175+
snappy-0.2.jar
176+
snappy-java-1.1.2.jar
177+
spire-macros_2.10-0.7.4.jar
178+
spire_2.10-0.7.4.jar
179+
stax-api-1.0-2.jar
180+
stax-api-1.0.1.jar
181+
stream-2.7.0.jar
182+
stringtemplate-3.2.1.jar
183+
super-csv-2.2.0.jar
184+
tachyon-client-0.8.2.jar
185+
tachyon-underfs-hdfs-0.8.2.jar
186+
tachyon-underfs-local-0.8.2.jar
187+
tachyon-underfs-s3-0.8.2.jar
188+
uncommons-maths-1.2.2a.jar
189+
unused-1.0.0.jar
190+
xbean-asm5-shaded-4.4.jar
191+
xmlenc-0.52.jar
192+
xz-1.0.jar
193+
zookeeper-3.4.5.jar

‎dev/deps/spark-deps-hadoop-2.6

+192
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
JavaEWAH-0.3.2.jar
2+
RoaringBitmap-0.5.11.jar
3+
ST4-4.0.4.jar
4+
activation-1.1.1.jar
5+
akka-actor_2.10-2.3.11.jar
6+
akka-remote_2.10-2.3.11.jar
7+
akka-slf4j_2.10-2.3.11.jar
8+
antlr-2.7.7.jar
9+
antlr-runtime-3.4.jar
10+
aopalliance-1.0.jar
11+
apache-log4j-extras-1.2.17.jar
12+
apacheds-i18n-2.0.0-M15.jar
13+
apacheds-kerberos-codec-2.0.0-M15.jar
14+
api-asn1-api-1.0.0-M20.jar
15+
api-util-1.0.0-M20.jar
16+
arpack_combined_all-0.1.jar
17+
asm-3.1.jar
18+
asm-commons-3.1.jar
19+
asm-tree-3.1.jar
20+
avro-1.7.7.jar
21+
avro-ipc-1.7.7-tests.jar
22+
avro-ipc-1.7.7.jar
23+
avro-mapred-1.7.7-hadoop2.jar
24+
base64-2.3.8.jar
25+
bcprov-jdk15on-1.51.jar
26+
bonecp-0.8.0.RELEASE.jar
27+
breeze-macros_2.10-0.11.2.jar
28+
breeze_2.10-0.11.2.jar
29+
calcite-avatica-1.2.0-incubating.jar
30+
calcite-core-1.2.0-incubating.jar
31+
calcite-linq4j-1.2.0-incubating.jar
32+
chill-java-0.5.0.jar
33+
chill_2.10-0.5.0.jar
34+
commons-beanutils-1.7.0.jar
35+
commons-beanutils-core-1.8.0.jar
36+
commons-cli-1.2.jar
37+
commons-codec-1.10.jar
38+
commons-collections-3.2.2.jar
39+
commons-compiler-2.7.6.jar
40+
commons-compress-1.4.1.jar
41+
commons-configuration-1.6.jar
42+
commons-dbcp-1.4.jar
43+
commons-digester-1.8.jar
44+
commons-httpclient-3.1.jar
45+
commons-io-2.4.jar
46+
commons-lang-2.6.jar
47+
commons-lang3-3.3.2.jar
48+
commons-logging-1.1.3.jar
49+
commons-math3-3.4.1.jar
50+
commons-net-2.2.jar
51+
commons-pool-1.5.4.jar
52+
compress-lzf-1.0.3.jar
53+
config-1.2.1.jar
54+
core-1.1.2.jar
55+
curator-client-2.6.0.jar
56+
curator-framework-2.6.0.jar
57+
curator-recipes-2.6.0.jar
58+
datanucleus-api-jdo-3.2.6.jar
59+
datanucleus-core-3.2.10.jar
60+
datanucleus-rdbms-3.2.9.jar
61+
derby-10.10.1.1.jar
62+
eigenbase-properties-1.1.5.jar
63+
geronimo-annotation_1.0_spec-1.1.1.jar
64+
geronimo-jaspic_1.0_spec-1.0.jar
65+
geronimo-jta_1.1_spec-1.1.1.jar
66+
groovy-all-2.1.6.jar
67+
gson-2.2.4.jar
68+
guice-3.0.jar
69+
guice-servlet-3.0.jar
70+
hadoop-annotations-2.6.0.jar
71+
hadoop-auth-2.6.0.jar
72+
hadoop-client-2.6.0.jar
73+
hadoop-common-2.6.0.jar
74+
hadoop-hdfs-2.6.0.jar
75+
hadoop-mapreduce-client-app-2.6.0.jar
76+
hadoop-mapreduce-client-common-2.6.0.jar
77+
hadoop-mapreduce-client-core-2.6.0.jar
78+
hadoop-mapreduce-client-jobclient-2.6.0.jar
79+
hadoop-mapreduce-client-shuffle-2.6.0.jar
80+
hadoop-yarn-api-2.6.0.jar
81+
hadoop-yarn-client-2.6.0.jar
82+
hadoop-yarn-common-2.6.0.jar
83+
hadoop-yarn-server-common-2.6.0.jar
84+
hadoop-yarn-server-web-proxy-2.6.0.jar
85+
htrace-core-3.0.4.jar
86+
httpclient-4.3.2.jar
87+
httpcore-4.3.2.jar
88+
ivy-2.4.0.jar
89+
jackson-annotations-2.4.4.jar
90+
jackson-core-2.4.4.jar
91+
jackson-core-asl-1.9.13.jar
92+
jackson-databind-2.4.4.jar
93+
jackson-jaxrs-1.9.13.jar
94+
jackson-mapper-asl-1.9.13.jar
95+
jackson-module-scala_2.10-2.4.4.jar
96+
jackson-xc-1.9.13.jar
97+
janino-2.7.8.jar
98+
jansi-1.4.jar
99+
java-xmlbuilder-1.0.jar
100+
javax.inject-1.jar
101+
javax.servlet-3.0.0.v201112011016.jar
102+
javolution-5.5.1.jar
103+
jaxb-api-2.2.2.jar
104+
jaxb-impl-2.2.3-1.jar
105+
jcl-over-slf4j-1.7.10.jar
106+
jdo-api-3.0.1.jar
107+
jersey-client-1.9.jar
108+
jersey-core-1.9.jar
109+
jersey-guice-1.9.jar
110+
jersey-json-1.9.jar
111+
jersey-server-1.9.jar
112+
jets3t-0.9.3.jar
113+
jettison-1.1.jar
114+
jetty-6.1.26.jar
115+
jetty-all-7.6.0.v20120127.jar
116+
jetty-util-6.1.26.jar
117+
jline-2.10.5.jar
118+
jline-2.12.jar
119+
joda-time-2.9.jar
120+
jodd-core-3.5.2.jar
121+
jpam-1.1.jar
122+
json-20090211.jar
123+
json4s-ast_2.10-3.2.10.jar
124+
json4s-core_2.10-3.2.10.jar
125+
json4s-jackson_2.10-3.2.10.jar
126+
jsr305-1.3.9.jar
127+
jta-1.1.jar
128+
jtransforms-2.4.0.jar
129+
jul-to-slf4j-1.7.10.jar
130+
kryo-2.21.jar
131+
leveldbjni-all-1.8.jar
132+
libfb303-0.9.2.jar
133+
libthrift-0.9.2.jar
134+
log4j-1.2.17.jar
135+
lz4-1.3.0.jar
136+
mail-1.4.7.jar
137+
mesos-0.21.1-shaded-protobuf.jar
138+
metrics-core-3.1.2.jar
139+
metrics-graphite-3.1.2.jar
140+
metrics-json-3.1.2.jar
141+
metrics-jvm-3.1.2.jar
142+
minlog-1.2.jar
143+
mx4j-3.0.2.jar
144+
netty-3.8.0.Final.jar
145+
netty-all-4.0.29.Final.jar
146+
objenesis-1.2.jar
147+
opencsv-2.3.jar
148+
oro-2.0.8.jar
149+
paranamer-2.6.jar
150+
parquet-column-1.7.0.jar
151+
parquet-common-1.7.0.jar
152+
parquet-encoding-1.7.0.jar
153+
parquet-format-2.3.0-incubating.jar
154+
parquet-generator-1.7.0.jar
155+
parquet-hadoop-1.7.0.jar
156+
parquet-hadoop-bundle-1.6.0.jar
157+
parquet-jackson-1.7.0.jar
158+
pmml-agent-1.2.7.jar
159+
pmml-model-1.2.7.jar
160+
pmml-schema-1.2.7.jar
161+
protobuf-java-2.5.0.jar
162+
py4j-0.9.jar
163+
pyrolite-4.9.jar
164+
quasiquotes_2.10-2.0.0-M8.jar
165+
reflectasm-1.07-shaded.jar
166+
scala-compiler-2.10.5.jar
167+
scala-library-2.10.5.jar
168+
scala-reflect-2.10.5.jar
169+
scalap-2.10.5.jar
170+
servlet-api-2.5.jar
171+
slf4j-api-1.7.10.jar
172+
slf4j-log4j12-1.7.10.jar
173+
snappy-0.2.jar
174+
snappy-java-1.1.2.jar
175+
spire-macros_2.10-0.7.4.jar
176+
spire_2.10-0.7.4.jar
177+
stax-api-1.0-2.jar
178+
stax-api-1.0.1.jar
179+
stream-2.7.0.jar
180+
stringtemplate-3.2.1.jar
181+
super-csv-2.2.0.jar
182+
tachyon-client-0.8.2.jar
183+
tachyon-underfs-hdfs-0.8.2.jar
184+
tachyon-underfs-local-0.8.2.jar
185+
tachyon-underfs-s3-0.8.2.jar
186+
uncommons-maths-1.2.2a.jar
187+
unused-1.0.0.jar
188+
xbean-asm5-shaded-4.4.jar
189+
xercesImpl-2.9.1.jar
190+
xmlenc-0.52.jar
191+
xz-1.0.jar
192+
zookeeper-3.4.6.jar

‎dev/run-tests-jenkins.py

+4
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,10 @@ def main():
168168
os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop2.2"
169169
if "test-hadoop2.3" in ghprb_pull_title:
170170
os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop2.3"
171+
if "test-hadoop2.4" in ghprb_pull_title:
172+
os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop2.4"
173+
if "test-hadoop2.6" in ghprb_pull_title:
174+
os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop2.6"
171175

172176
build_display_name = os.environ["BUILD_DISPLAY_NAME"]
173177
build_url = os.environ["BUILD_URL"]

0 commit comments

Comments
 (0)