Skip to content

Commit

Permalink
correct sbt build scripts for tez
Browse files Browse the repository at this point in the history
  • Loading branch information
@FMX
FMX committed Dec 28, 2024
1 parent a470611 commit fe0ad49
Showing 1 changed file with 0 additions and 146 deletions.
146 changes: 0 additions & 146 deletions project/CelebornBuild.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,6 @@ object Dependencies {
// Tez
val tezVersion = "0.9.1"

val apLoader = "me.bechberger" % "ap-loader-all" % apLoaderVersion

val apLoader = "me.bechberger" % "ap-loader-all" % apLoaderVersion
val commonsCompress = "org.apache.commons" % "commons-compress" % commonsCompressVersion
val commonsCrypto = "org.apache.commons" % "commons-crypto" % commonsCryptoVersion excludeAll(
Expand Down Expand Up @@ -259,14 +257,6 @@ object Dependencies {
ExclusionRule("org.apache.httpcomponents", "httpclient"),
ExclusionRule("org.slf4j", "slf4j-log4j12")
)
val jerseyTestFrameworkCore = "org.glassfish.jersey.test-framework" % "jersey-test-framework-core" % jerseyVersion
val jerseyTestFrameworkProviderJetty = "org.glassfish.jersey.test-framework.providers" % "jersey-test-framework-provider-jetty" % jerseyVersion excludeAll(
ExclusionRule("org.eclipse.jetty", "jetty-util"),
ExclusionRule("org.eclipse.jetty", "jetty-continuation"))

// SSL support
val bouncycastleBcprovJdk18on = "org.bouncycastle" % "bcprov-jdk18on" % bouncycastleVersion % "test"
val bouncycastleBcpkixJdk18on = "org.bouncycastle" % "bcpkix-jdk18on" % bouncycastleVersion % "test"

val picocli = "info.picocli" % "picocli" % picocliVersion
}
Expand Down Expand Up @@ -1786,139 +1776,3 @@ object TezClientProjects {
}
)
}


object CelebornOpenApi {
val openApiSpecDir = "openapi/openapi-client/src/main/openapi3"
val openApiMasterInternalOutputDir = "openapi/openapi-client/target/master/generated-sources/java"
val openApiWorkerInternalOutputDir = "openapi/openapi-client/target/worker/generated-sources/java"
val openApiClientOutputDir = "openapi/openapi-client/src/main/java"

val generate = TaskKey[Unit]("generate", "generate openapi client code")

val commonOpenApiClientGenerateSettings = Seq(
openApiGeneratorName := "java",
openApiGenerateApiTests := SettingDisabled,
openApiGenerateModelTests := SettingDisabled,
openApiModelPackage := "org.apache.celeborn.rest.v1.model",
openApiAdditionalProperties := Map(
"dateLibrary" -> "java8",
"useGzipFeature" -> "true",
"library" -> "apache-httpclient",
"hideGenerationTimestamp" -> "true",
"supportUrlQuery" -> "false",
"annotationLibrary" -> "none",
"templateDir" -> s"$openApiSpecDir/templates",
)
)

lazy val openApiClientMasterGenerate = Project("celeborn-openapi-client-master-generate", file("openapi/openapi-client/target/master"))
.enablePlugins(OpenApiGeneratorPlugin)
.settings(
commonSettings,
openApiInputSpec := (file(openApiSpecDir) / "master_rest_v1.yaml").toString,
openApiOutputDir := openApiMasterInternalOutputDir,
openApiApiPackage := "org.apache.celeborn.rest.v1.master",
openApiInvokerPackage := "org.apache.celeborn.rest.v1.master.invoker",
commonOpenApiClientGenerateSettings
)

lazy val openApiClientWorkerGenerate = Project("celeborn-openapi-client-worker-generate", file("openapi/openapi-client/target/worker"))
.enablePlugins(OpenApiGeneratorPlugin)
.settings(
commonSettings,
openApiInputSpec := (file(openApiSpecDir) / "worker_rest_v1.yaml").toString,
openApiOutputDir := openApiWorkerInternalOutputDir,
openApiApiPackage := "org.apache.celeborn.rest.v1.worker",
openApiInvokerPackage := "org.apache.celeborn.rest.v1.worker.invoker",
commonOpenApiClientGenerateSettings
)

lazy val openApiClient = Project("celeborn-openapi-client", file("openapi/openapi-client"))
.settings (
commonSettings,
releaseSettings,
libraryDependencies ++= Seq(
Dependencies.jacksonAnnotations,
Dependencies.jacksonCore,
Dependencies.jacksonDatabind,
Dependencies.jacksonDataTypeJsr310,
Dependencies.jacksonJaxrsJsonProvider,
Dependencies.findbugsJsr305,
Dependencies.javaxAnnotationApi,
Dependencies.httpClient5,
Dependencies.httpCore5,
Dependencies.httpCore5H2,
Dependencies.openApiToolsJacksonBindNullable,
Dependencies.slf4jApi
),

generate := {
(openApiClientMasterGenerate / Compile / openApiGenerate).value
(openApiClientWorkerGenerate / Compile / openApiGenerate).value

streams.value.log.info("Cleaning up openapi generate output directory: " + openApiClientOutputDir)
val dstDir = file(openApiClientOutputDir)
IO.delete(dstDir)

val masterSrcDir = file(openApiMasterInternalOutputDir) / "src" / "main" / "java"
streams.value.log.info(s"Copying openapi generated master sources from $masterSrcDir to $dstDir")
IO.copyDirectory(masterSrcDir, dstDir)

val workerSrcDir = file(openApiWorkerInternalOutputDir) / "src" / "main" / "java"
streams.value.log.info(s"Copying openapi generated worker sources from $workerSrcDir to $dstDir")
IO.copyDirectory(workerSrcDir, dstDir)
},

(assembly / test) := { },
(assembly / assemblyJarName) := {
s"${moduleName.value}_${scalaBinaryVersion.value}-${version.value}.${artifact.value.extension}"
},
(assembly / logLevel) := Level.Info,
// Exclude `scala-library` from assembly.
(assembly / assemblyPackageScala / assembleArtifact) := false,
(assembly / assemblyExcludedJars) := {
val cp = (assembly / fullClasspath).value
cp filter { v =>
val name = v.data.getName
!(name.startsWith("celeborn-") ||
name.startsWith("jackson-annotations-") ||
name.startsWith("jackson-core-") ||
name.startsWith("jackson-databind-") ||
name.startsWith("jackson-datatype-jsr310-") ||
name.startsWith("jackson-jaxrs-json-provider-") ||
name.startsWith("jsr305-") ||
name.startsWith("jakarta.annotation-api-") ||
name.startsWith("httpclient5-") ||
name.startsWith("httpcore5-") ||
name.startsWith("httpcore5-h2-") ||
name.startsWith("jackson-databind-nullable-") ||
name.startsWith("slf4j-api-"))
}
},

(assembly / assemblyShadeRules) := Seq(
ShadeRule.rename("org.openapitools.**" -> "org.apache.celeborn.shaded.org.openapitools.@1").inAll,
ShadeRule.rename("javax.annotation.**" -> "org.apache.celeborn.shaded.javax.annotation.@1").inAll,
ShadeRule.rename("com.fasterxml.jackson.**" -> "org.apache.celeborn.shaded.com.fasterxml.jackson.@1").inAll,
ShadeRule.rename("jakarta.validation.**" -> "org.apache.celeborn.shaded.jakarta.validation.@1").inAll,
ShadeRule.rename("javax.validation.**" -> "org.apache.celeborn.shaded.javax.validation.@1").inAll,
ShadeRule.rename("javax.ws.rs.ext.**" -> "org.apache.celeborn.shaded.javax.ws.rs.ext.@1").inAll,
ShadeRule.rename("org.apache.hc.**" -> "org.apache.celeborn.shaded.org.apache.hc.@1").inAll,
ShadeRule.rename("org.slf4j.**" -> "org.apache.celeborn.shaded.org.slf4j.@1").inAll
),

(assembly / assemblyMergeStrategy) := {
case m if m.toLowerCase(Locale.ROOT).endsWith("license") => MergeStrategy.discard
case m if m.toLowerCase(Locale.ROOT).endsWith("manifest.mf") => MergeStrategy.discard
case m if m.toLowerCase(Locale.ROOT).endsWith("meta-inf/dependencies") => MergeStrategy.discard
case m if m.toLowerCase(Locale.ROOT).endsWith("module-info.class") => MergeStrategy.discard
case m if m.toLowerCase(Locale.ROOT).endsWith("mozilla/public-suffix-list.txt") => MergeStrategy.discard
case m if m.toLowerCase(Locale.ROOT).endsWith("notice") => MergeStrategy.discard
case PathList(ps@_*) if Assembly.isLicenseFile(ps.last) => MergeStrategy.discard
case _ => MergeStrategy.first
},
Compile / packageBin := assembly.value,
pomPostProcess := removeDependenciesTransformer
)
}

0 comments on commit fe0ad49

Please # to comment.