Skip to content

Commit

Permalink
shade necessary packages
Browse files Browse the repository at this point in the history
  • Loading branch information
@FMX
FMX committed Dec 29, 2024
1 parent fe0ad49 commit cee1ea7
Show file tree
Hide file tree
Showing 10 changed files with 558 additions and 111 deletions.
4 changes: 2 additions & 2 deletions build/make-distribution.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ function build_tez_client {
| grep -v "INFO" \
| grep -v "WARNING" \
| tail -n 1)
BUILD_COMMAND=("$MVN" clean package $MVN_DIST_OPT -pl :celeborn-client-tez_${SCALA_VERSION} -am $@)
BUILD_COMMAND=("$MVN" clean package $MVN_DIST_OPT -pl :celeborn-client-tez-shaded_${SCALA_VERSION} -am $@)

# Actually build the jar
echo -e "\nBuilding with..."
Expand All @@ -263,7 +263,7 @@ function build_tez_client {

## flink spark client jars
mkdir -p "$DIST_DIR/tez"
cp "$PROJECT_DIR"/client-tez/tez/target/celeborn-client-tez_${SCALA_VERSION}-$VERSION.jar "$DIST_DIR/tez/"
cp "$PROJECT_DIR"/client-tez/tez-shaded/target/celeborn-client-tez-shaded_${SCALA_VERSION}-$VERSION.jar "$DIST_DIR/tez/"
}


Expand Down
163 changes: 163 additions & 0 deletions client-tez/tez-shaded/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.celeborn</groupId>
<artifactId>celeborn-parent_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<relativePath>../../pom.xml</relativePath>
</parent>

<artifactId>celeborn-client-tez-shaded_${scala.binary.version}</artifactId>
<packaging>jar</packaging>
<name>Celeborn Client shaded for Tez</name>

<dependencies>
<dependency>
<groupId>org.apache.celeborn</groupId>
<artifactId>celeborn-client-tez_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<configuration>
<relocations>
<relocation>
<pattern>com.google.protobuf</pattern>
<shadedPattern>${shading.prefix}.com.google.protobuf</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.common</pattern>
<shadedPattern>${shading.prefix}.com.google.common</shadedPattern>
<excludes>
<exclude>com.google.common.util.concurrent.*</exclude>
</excludes>
</relocation>
<relocation>
<pattern>io.netty</pattern>
<shadedPattern>${shading.prefix}.io.netty</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.commons</pattern>
<shadedPattern>${shading.prefix}.org.apache.commons</shadedPattern>
</relocation>
<relocation>
<pattern>org.roaringbitmap</pattern>
<shadedPattern>${shading.prefix}.org.roaringbitmap</shadedPattern>
</relocation>
<relocation>
<pattern>org.roaringbitmap</pattern>
<shadedPattern>${shading.prefix}.org.roaringbitmap</shadedPattern>
</relocation>
<relocation>
<pattern>io.dropwizard.metrics</pattern>
<shadedPattern>${shading.prefix}.io.dropwizard.metrics</shadedPattern>
</relocation>
<relocation>
<pattern>com.codahale.metrics</pattern>
<shadedPattern>${shading.prefix}.com.codahale.metrics</shadedPattern>
</relocation>
<relocation>
<pattern>com.github.luben</pattern>
<shadedPattern>${shading.prefix}.com.github.luben</shadedPattern>
</relocation>
</relocations>
<artifactSet>
<includes>
<include>org.apache.celeborn:*</include>
<include>com.google.protobuf:protobuf-java</include>
<include>com.google.guava:guava</include>
<include>com.google.guava:failureaccess</include>
<include>io.netty:*</include>
<include>org.apache.commons:commons-lang3</include>
<include>org.roaringbitmap:RoaringBitmap</include>
<include>org.scala-lang:scala-library</include>
<include>org.scala-lang:scala-reflect</include>
<include>org.lz4:lz4-java</include>
<include>io.dropwizard.metrics:metrics-core</include>
<include>com.codahale.metrics:metrics-core</include>
<include>com.github.luben:zstd-jni</include>
</includes>
</artifactSet>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>**/*.proto</exclude>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>**/log4j.properties</exclude>
<exclude>META-INF/LICENSE.txt</exclude>
<exclude>META-INF/NOTICE.txt</exclude>
<exclude>LICENSE.txt</exclude>
<exclude>NOTICE.txt</exclude>
</excludes>
</filter>
</filters>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"></transformer>
</transformers>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>${maven.plugin.antrun.version}</version>
<executions>
<execution>
<id>rename-native-library</id>
<goals>
<goal>run</goal>
</goals>
<phase>package</phase>
<configuration>
<target>
<echo message="unpacking netty jar"></echo>
<unzip dest="${project.build.directory}/unpacked/" src="${project.build.directory}/${artifactId}-${version}.jar"></unzip>
<echo message="renaming native epoll library"></echo>
<move includeemptydirs="false" todir="${project.build.directory}/unpacked/META-INF/native">
<fileset dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
<mapper from="libnetty_transport_native_epoll_x86_64.so" to="liborg_apache_celeborn_shaded_netty_transport_native_epoll_x86_64.so" type="glob"></mapper>
</move>
<move includeemptydirs="false" todir="${project.build.directory}/unpacked/META-INF/native">
<fileset dir="${project.build.directory}/unpacked/META-INF/native"></fileset>
<mapper from="libnetty_transport_native_epoll_aarch_64.so" to="liborg_apache_celeborn_shaded_netty_transport_native_epoll_aarch_64.so" type="glob"></mapper>
</move>
<echo message="deleting native kqueue library"></echo>
<delete file="${project.build.directory}/unpacked/META-INF/native/libnetty_transport_native_kqueue_x86_64.jnilib"></delete>
<delete file="${project.build.directory}/unpacked/META-INF/native/libnetty_transport_native_kqueue_aarch_64.jnilib"></delete>
<delete file="${project.build.directory}/unpacked/META-INF/native/libnetty_resolver_dns_native_macos_aarch_64.jnilib"></delete>
<delete file="${project.build.directory}/unpacked/META-INF/native/libnetty_resolver_dns_native_macos_x86_64.jnilib"></delete>
<echo message="repackaging netty jar"></echo>
<jar basedir="${project.build.directory}/unpacked" destfile="${project.build.directory}/${artifactId}-${version}.jar"></jar>
</target>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

</project>
Loading

0 comments on commit cee1ea7

Please # to comment.