diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 64e93490a..3c8467b98 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -43,6 +43,12 @@ jobs: with: fetch-depth: 0 + - name: Set up JDK 21 + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'temurin' + - name: Set up Helm uses: azure/setup-helm@v4.2.0 with: @@ -83,25 +89,34 @@ jobs: if: steps.list-changed.outputs.changed == 'true' uses: medyagh/setup-minikube@v0.0.18 - - name: Docker build + - name: Print Docker info + if: steps.list-changed.outputs.changed == 'true' + run: | + docker -v + minikube docker-env + + - name: Image build if: steps.list-changed.outputs.changed == 'true' run: | - eval $(minikube docker-env) - docker build -f ./Dockerfile \ - --build-arg ECLIPSELINK=true \ - --build-arg ECLIPSELINK_DEPS=com.h2database:h2:2.3.232 \ - -t polaris:latest . + export BUILD_TAG=helm-ci-$(date +%s) + echo BUILD_TAG=${BUILD_TAG} >> ${GITHUB_ENV} + eval $(minikube -p minikube docker-env) + ./gradlew assemble \ + -Dquarkus.container-image.build=true \ + -Dquarkus.container-image.tag=${BUILD_TAG} + minikube image ls - name: Install fixtures if: steps.list-changed.outputs.changed == 'true' run: | kubectl create namespace polaris-ns - kubectl apply --namespace polaris-ns $(find helm/polaris/ci/fixtures -name "*.yaml" -exec echo -n "-f {} " \;) + kubectl apply --namespace polaris-ns -f helm/polaris/ci/fixtures - name: Run chart-testing (install) if: steps.list-changed.outputs.changed == 'true' run: | - ct install --target-branch ${{ github.event.repository.default_branch }} \ + ct install --debug \ --namespace polaris-ns \ - --helm-extra-set-args "--set=image.repository=polaris --set=image.tag=latest" \ - --debug --charts ./helm/polaris + --target-branch ${{ github.event.repository.default_branch }} \ + --helm-extra-set-args "--set=image.tag=${BUILD_TAG} --set=bootstrap.image.tag=${BUILD_TAG}" \ + --charts ./helm/polaris diff --git a/.github/workflows/regtest.yml b/.github/workflows/regtest.yml index b7f095c08..8988026d4 100644 --- a/.github/workflows/regtest.yml +++ b/.github/workflows/regtest.yml @@ -33,10 +33,39 @@ jobs: steps: - uses: actions/checkout@v4 - - name: fix permissions + + - name: Set up JDK 21 + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'temurin' + + - name: Fix permissions run: mkdir -p regtests/output && chmod 777 regtests/output && chmod 777 regtests/t_*/ref/* + + - name: Image build + run: | + export BUILD_TAG=regtests-ci-$(date +%s) + echo BUILD_TAG=${BUILD_TAG} >> ${GITHUB_ENV} + ./gradlew :polaris-quarkus-server:assemble \ + -Dquarkus.container-image.build=true \ + -Dquarkus.container-image.name=polaris-testing \ + -Dquarkus.container-image.tag=${BUILD_TAG} + - name: Regression Test env: AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}} AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}} - run: docker compose up --build --exit-code-from regtest + run: | + # create compose override file and set image tag + cat < ./build/docker-compose.override.yml + services: + polaris: + image: apache/polaris-testing:${BUILD_TAG} + EOF + docker compose \ + -f regtests/docker-compose.yml \ + -f build/docker-compose.override.yml \ + up \ + --build \ + --exit-code-from regtest \ No newline at end of file diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 7b8cbfb80..000000000 --- a/Dockerfile +++ /dev/null @@ -1,67 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Base Image -# Use a non-docker-io registry, because pulling images from docker.io is -# subject to aggressive request rate limiting and bandwidth shaping. -FROM registry.access.redhat.com/ubi9/openjdk-21:1.21-3.1733995526 AS build -ARG ECLIPSELINK=false -ARG ECLIPSELINK_DEPS - -# Copy the REST catalog into the container -COPY --chown=default:root . /app - -# Set the working directory in the container, nuke any existing builds -WORKDIR /app -RUN rm -rf build - -# Build the rest catalog -RUN ./gradlew --no-daemon --info ${ECLIPSELINK_DEPS+"-PeclipseLinkDeps=$ECLIPSELINK_DEPS"} -PeclipseLink=$ECLIPSELINK clean :polaris-quarkus-service:build -x test - -FROM registry.access.redhat.com/ubi9/openjdk-21-runtime:1.21-1.1733995527 - -LABEL org.opencontainers.image.source=https://github.com/apache/polaris -LABEL org.opencontainers.image.description="Apache Polaris (incubating)" -LABEL org.opencontainers.image.licenses=Apache-2.0 - -ENV LANGUAGE='en_US:en' - -USER root -RUN groupadd --gid 10001 polaris \ - && useradd --uid 10000 --gid polaris polaris \ - && chown -R polaris:polaris /opt/jboss/container \ - && chown -R polaris:polaris /deployments - -USER polaris -WORKDIR /home/polaris -ENV USER=polaris -ENV UID=10000 -ENV HOME=/home/polaris - -# We make four distinct layers so if there are application changes the library layers can be re-used -COPY --from=build --chown=polaris:polaris /app/quarkus/service/build/quarkus-app/lib/ /deployments/lib/ -COPY --from=build --chown=polaris:polaris /app/quarkus/service/build/quarkus-app/*.jar /deployments/ -COPY --from=build --chown=polaris:polaris /app/quarkus/service/build/quarkus-app/app/ /deployments/app/ -COPY --from=build --chown=polaris:polaris /app/quarkus/service/build/quarkus-app/quarkus/ /deployments/quarkus/ - -EXPOSE 8181 -EXPOSE 8182 - -ENV AB_JOLOKIA_OFF="" -ENV JAVA_APP_JAR="/deployments/quarkus-run.jar" diff --git a/LICENSE-BINARY-DIST b/LICENSE-BINARY-DIST index 205a3eedc..0d151d441 100644 --- a/LICENSE-BINARY-DIST +++ b/LICENSE-BINARY-DIST @@ -688,6 +688,561 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +--- +com.h2database:h2 + +H2 is dual licensed and available under the MPL 2.0 (Mozilla Public License +Version 2.0) or under the EPL 1.0 (Eclipse Public License). + +------------------------------------------------------------------------------- + +Mozilla Public License, version 2.0 + +1. Definitions + + 1.1. “Contributor” + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + + 1.2. “Contributor Version” + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + + 1.3. “Contribution” + means Covered Software of a particular Contributor. + + 1.4. “Covered Software” + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, + and Modifications of such Source Code Form, in each case + including portions thereof. + + 1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms + of a Secondary License. + + 1.6. “Executable Form” + means any form of the work other than Source Code Form. + + 1.7. “Larger Work” + means a work that combines Covered Software with other material, + in a separate file or files, that is not Covered Software. + + 1.8. “License” + means this document. + + 1.9. “Licensable” + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, + any and all of the rights conveyed by this License. + + 1.10. “Modifications” + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + + 1.11. “Patent Claims” of a Contributor + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + + 1.12. “Secondary License” + means either the GNU General Public License, Version 2.0, the + GNU Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those licenses. + + 1.13. “Source Code Form” + means the form of the work preferred for making modifications. + + 1.14. “You” (or “Your”) + means an individual or a legal entity exercising rights under this License. + For legal entities, “You” includes any entity that controls, + is controlled by, or is under common control with You. For purposes of + this definition, “control” means (a) the power, direct or indirect, + to cause the direction or management of such entity, whether by contract + or otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + +2. License Grants and Conditions + + 2.1. Grants + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, + or as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, + offer for sale, have made, import, and otherwise transfer either + its Contributions or its Contributor Version. + + 2.2. Effective Date + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor + first distributes such Contribution. + + 2.3. Limitations on Grant Scope + The licenses granted in this Section 2 are the only rights granted + under this License. No additional rights or licenses will be implied + from the distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted + by a Contributor: + + a. for any code that a Contributor has removed from + Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its + Contributor Version); or + + c. under Patent Claims infringed by Covered Software in the + absence of its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + + 2.4. Subsequent Licenses + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License + (if permitted under the terms of Section 3.3). + + 2.5. Representation + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights + to grant the rights to its Contributions conveyed by this License. + + 2.6. Fair Use + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, + or other equivalents. + + 2.7. Conditions + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the + licenses granted in Section 2.1. + +3. Responsibilities + + 3.1. Distribution of Source Form + All distribution of Covered Software in Source Code Form, including + any Modifications that You create or to which You contribute, must be + under the terms of this License. You must inform recipients that the + Source Code Form of the Covered Software is governed by the terms + of this License, and how they can obtain a copy of this License. + You may not attempt to alter or restrict the recipients’ rights + in the Source Code Form. + + 3.2. Distribution of Executable Form + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more than + the cost of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients’ rights in the Source Code Form under this License. + + 3.3. Distribution of a Larger Work + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of + Covered Software with a work governed by one or more Secondary Licenses, + and the Covered Software is not Incompatible With Secondary Licenses, + this License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the + Covered Software under the terms of either this License or such + Secondary License(s). + + 3.4. Notices + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, + or limitations of liability) contained within the Source Code Form of + the Covered Software, except that You may alter any license notices to + the extent required to remedy known factual inaccuracies. + + 3.5. Application of Additional Terms + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of + Covered Software. However, You may do so only on Your own behalf, + and not on behalf of any Contributor. You must make it absolutely clear + that any such warranty, support, indemnity, or liability obligation is + offered by You alone, and You hereby agree to indemnify every Contributor + for any liability incurred by such Contributor as a result of warranty, + support, indemnity or liability terms You offer. You may include + additional disclaimers of warranty and limitations of liability + specific to any jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + +If it is impossible for You to comply with any of the terms of this License +with respect to some or all of the Covered Software due to statute, +judicial order, or regulation then You must: (a) comply with the terms of +this License to the maximum extent possible; and (b) describe the limitations +and the code they affect. Such description must be placed in a text file +included with all distributions of the Covered Software under this License. +Except to the extent prohibited by statute or regulation, such description +must be sufficiently detailed for a recipient of ordinary skill +to be able to understand it. + +5. Termination + + 5.1. The rights granted under this License will terminate automatically + if You fail to comply with any of its terms. However, if You become + compliant, then the rights granted under this License from a particular + Contributor are reinstated (a) provisionally, unless and until such + Contributor explicitly and finally terminates Your grants, and (b) on an + ongoing basis, if such Contributor fails to notify You of the + non-compliance by some reasonable means prior to 60 days after You have + come back into compliance. Moreover, Your grants from a particular + Contributor are reinstated on an ongoing basis if such Contributor + notifies You of the non-compliance by some reasonable means, + this is the first time You have received notice of non-compliance with + this License from such Contributor, and You become compliant prior to + 30 days after Your receipt of the notice. + + 5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted + to You by any and all Contributors for the Covered Software under + Section 2.1 of this License shall terminate. + + 5.3. In the event of termination under Sections 5.1 or 5.2 above, all + end user license agreements (excluding distributors and resellers) which + have been validly granted by You or Your distributors under this License + prior to termination shall survive termination. + +6. Disclaimer of Warranty + +Covered Software is provided under this License on an “as is” basis, without +warranty of any kind, either expressed, implied, or statutory, including, +without limitation, warranties that the Covered Software is free of defects, +merchantable, fit for a particular purpose or non-infringing. The entire risk +as to the quality and performance of the Covered Software is with You. +Should any Covered Software prove defective in any respect, You +(not any Contributor) assume the cost of any necessary servicing, repair, +or correction. This disclaimer of warranty constitutes an essential part of +this License. No use of any Covered Software is authorized under this +License except under this disclaimer. + +7. Limitation of Liability + +Under no circumstances and under no legal theory, whether tort +(including negligence), contract, or otherwise, shall any Contributor, or +anyone who distributes Covered Software as permitted above, be liable to +You for any direct, indirect, special, incidental, or consequential damages +of any character including, without limitation, damages for lost profits, +loss of goodwill, work stoppage, computer failure or malfunction, or any and +all other commercial damages or losses, even if such party shall have been +informed of the possibility of such damages. This limitation of liability +shall not apply to liability for death or personal injury resulting from +such party’s negligence to the extent applicable law prohibits such +limitation. Some jurisdictions do not allow the exclusion or limitation of +incidental or consequential damages, so this exclusion and limitation may +not apply to You. + +8. Litigation + +Any litigation relating to this License may be brought only in the courts of +a jurisdiction where the defendant maintains its principal place of business +and such litigation shall be governed by laws of that jurisdiction, without +reference to its conflict-of-law provisions. Nothing in this Section shall +prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + +This License represents the complete agreement concerning the subject matter +hereof. If any provision of this License is held to be unenforceable, +such provision shall be reformed only to the extent necessary to make it +enforceable. Any law or regulation which provides that the language of a +contract shall be construed against the drafter shall not be used to construe +this License against a Contributor. + +10. Versions of the License + + 10.1. New Versions + Mozilla Foundation is the license steward. Except as provided in + Section 10.3, no one other than the license steward has the right to + modify or publish new versions of this License. Each version will be + given a distinguishing version number. + + 10.2. Effect of New Versions + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published + by the license steward. + + 10.3. Modified Versions + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + + 10.4. Distributing Source Code Form that is + Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this + License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the terms of the + Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed + with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to +look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible With Secondary Licenses”, + as defined by the Mozilla Public License, v. 2.0. + +------------------------------------------------------------------------------- + +Eclipse Public License, Version 1.0 (EPL-1.0) + +THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC +LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM +CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + + a) in the case of the initial Contributor, the initial code and + documentation distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + i) changes to the Program, and + ii) additions to the Program; + +where such changes and/or additions to the Program originate from and are +distributed by that particular Contributor. A Contribution 'originates' +from a Contributor if it was added to the Program by such Contributor itself +or anyone acting on such Contributor's behalf. Contributions do not include +additions to the Program which: (i) are separate modules of software +distributed in conjunction with the Program under their own license agreement, +and (ii) are not derivative works of the Program. + +"Contributor" means any person or entity that distributes the Program. + +"Licensed Patents " mean patent claims licensable by a Contributor which are +necessarily infringed by the use or sale of its Contribution alone or +when combined with the Program. + +"Program" means the Contributions distributed in accordance with +this Agreement. + +"Recipient" means anyone who receives the Program under this Agreement, +including all Contributors. + +2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free copyright license to + reproduce, prepare derivative works of, publicly display, publicly + perform, distribute and sublicense the Contribution of such + Contributor, if any, and such derivative works, + in source code and object code form. + + b) Subject to the terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free patent license under + Licensed Patents to make, use, sell, offer to sell, import and + otherwise transfer the Contribution of such Contributor, if any, + in source code and object code form. This patent license shall apply + to the combination of the Contribution and the Program if, at the time + the Contribution is added by the Contributor, such addition of the + Contribution causes such combination to be covered by the + Licensed Patents. The patent license shall not apply to any other + combinations which include the Contribution. + No hardware per se is licensed hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the + rights and licenses granted hereunder, each Recipient hereby assumes + sole responsibility to secure any other intellectual property rights + needed, if any. For example, if a third party patent license is + required to allow Recipient to distribute the Program, it is + Recipient's responsibility to acquire that license + before distributing the Program. + + d) Each Contributor represents that to its knowledge it has sufficient + copyright rights in its Contribution, if any, to grant the copyright + license set forth in this Agreement. + +3. REQUIREMENTS + +A Contributor may choose to distribute the Program in object code form under +its own license agreement, provided that: + + a) it complies with the terms and conditions of this Agreement; and + + b) its license agreement: + + i) effectively disclaims on behalf of all Contributors all warranties + and conditions, express and implied, including warranties or + conditions of title and non-infringement, and implied warranties or + conditions of merchantability and fitness for a particular purpose; + + ii) effectively excludes on behalf of all Contributors all liability + for damages, including direct, indirect, special, incidental and + consequential damages, such as lost profits; + + iii) states that any provisions which differ from this Agreement are + offered by that Contributor alone and not by any other party; and + + iv) states that source code for the Program is available from such + Contributor, and informs licensees how to obtain it in a reasonable + manner on or through a medium customarily used for software exchange. + +When the Program is made available in source code form: + + a) it must be made available under this Agreement; and + b) a copy of this Agreement must be included with each copy of the Program. + +Contributors may not remove or alter any copyright notices contained +within the Program. + +Each Contributor must identify itself as the originator of its Contribution, +if any, in a manner that reasonably allows subsequent Recipients to +identify the originator of the Contribution. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities with +respect to end users, business partners and the like. While this license is +intended to facilitate the commercial use of the Program, the Contributor who +includes the Program in a commercial product offering should do so in a manner +which does not create potential liability for other Contributors. Therefore, +if a Contributor includes the Program in a commercial product offering, +such Contributor ("Commercial Contributor") hereby agrees to defend and +indemnify every other Contributor ("Indemnified Contributor") against any +losses, damages and costs (collectively "Losses") arising from claims, +lawsuits and other legal actions brought by a third party against the +Indemnified Contributor to the extent caused by the acts or omissions of +such Commercial Contributor in connection with its distribution of the Program +in a commercial product offering. The obligations in this section do not apply +to any claims or Losses relating to any actual or alleged intellectual +property infringement. In order to qualify, an Indemnified Contributor must: +a) promptly notify the Commercial Contributor in writing of such claim, +and b) allow the Commercial Contributor to control, and cooperate with the +Commercial Contributor in, the defense and any related settlement +negotiations. The Indemnified Contributor may participate in any such +claim at its own expense. + +For example, a Contributor might include the Program in a commercial product +offering, Product X. That Contributor is then a Commercial Contributor. +If that Commercial Contributor then makes performance claims, or offers +warranties related to Product X, those performance claims and warranties +are such Commercial Contributor's responsibility alone. Under this section, +the Commercial Contributor would have to defend claims against the other +Contributors related to those performance claims and warranties, and if a +court requires any other Contributor to pay any damages as a result, +the Commercial Contributor must pay those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR +IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, +NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. +Each Recipient is solely responsible for determining the appropriateness of +using and distributing the Program and assumes all risks associated with its +exercise of rights under this Agreement , including but not limited to the +risks and costs of program errors, compliance with applicable laws, damage to +or loss of data, programs or equipment, and unavailability +or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY +CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION +LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE +EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under +applicable law, it shall not affect the validity or enforceability of the +remainder of the terms of this Agreement, and without further action by +the parties hereto, such provision shall be reformed to the minimum extent +necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Program itself +(excluding combinations of the Program with other software or hardware) +infringes such Recipient's patent(s), then such Recipient's rights granted +under Section 2(b) shall terminate as of the date such litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it fails to +comply with any of the material terms or conditions of this Agreement and +does not cure such failure in a reasonable period of time after becoming +aware of such noncompliance. If all Recipient's rights under this +Agreement terminate, Recipient agrees to cease use and distribution of the +Program as soon as reasonably practicable. However, Recipient's obligations +under this Agreement and any licenses granted by Recipient relating to the +Program shall continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, +but in order to avoid inconsistency the Agreement is copyrighted and may +only be modified in the following manner. The Agreement Steward reserves +the right to publish new versions (including revisions) of this Agreement +from time to time. No one other than the Agreement Steward has the right to +modify this Agreement. The Eclipse Foundation is the initial +Agreement Steward. The Eclipse Foundation may assign the responsibility to +serve as the Agreement Steward to a suitable separate entity. Each new version +of the Agreement will be given a distinguishing version number. The Program +(including Contributions) may always be distributed subject to the version +of the Agreement under which it was received. In addition, after a new version +of the Agreement is published, Contributor may elect to distribute the Program +(including its Contributions) under the new version. Except as expressly +stated in Sections 2(a) and 2(b) above, Recipient receives no rights or +licenses to the intellectual property of any Contributor under this Agreement, +whether expressly, by implication, estoppel or otherwise. All rights in the +Program not expressly granted under this Agreement are reserved. + +This Agreement is governed by the laws of the State of New York and the +intellectual property laws of the United States of America. No party to +this Agreement will bring a legal action under this Agreement more than one +year after the cause of action arose. Each party waives its rights to a +jury trial in any resulting litigation. --- com.google.protobuf:protobuf-java @@ -6863,6 +7418,97 @@ org.latencyutils:LatencyUtils +--- +org.postgresql:postgresql + + + + Copyright (c) 1997, PostgreSQL Global Development Group + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + Additional License files can be found in the 'licenses' folder located in the same directory as the LICENSE file (i.e. this file) + + - Software produced outside the ASF which is available under other licenses (not Apache-2.0) + + BSD-2-Clause + * com.ongres.scram:scram-client:3.1 + * com.ongres.scram:scram-common:3.1 + * com.ongres.stringprep:saslprep:2.2 + * com.ongres.stringprep:stringprep:2.2 + + + + Copyright (c) 2017 OnGres, Inc. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + + Copyright (c) 2019 OnGres, Inc. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + --- org.reactivestreams:reactive-streams diff --git a/README.md b/README.md index da1a27eb5..e76b18b63 100644 --- a/README.md +++ b/README.md @@ -39,15 +39,36 @@ for contribution guidelines. ## Building and Running Apache Polaris is organized into the following modules: + - `polaris-core` - The main Polaris entity definitions and core business logic -- `polaris-server` - The Polaris REST API server -- `polaris-eclipselink` - The Eclipselink implementation of the MetaStoreManager interface +- API modules (implementing the Iceberg REST API and Polaris management API): + - `polaris-api-management-model` - The Polaris management model + - `polaris-api-management-service` - The Polaris management service + - `polaris-api-iceberg-service` - The Iceberg REST service +- Service modules: + - `polaris-service-common` - The main components of the Polaris server +- Quarkus runtime modules: + - `polaris-quarkus-service` - The Quarkus-specific components of the Polaris server + - `polaris-quarkus-server` - The Polaris server runtime + - `polaris-quarkus-admin-tool` - The Polaris admin & maintenance tool +- Persistence modules + - `polaris-jpa-model` - The JPA entity definitions + - `polaris-eclipselink` - The Eclipselink implementation of the MetaStoreManager interface Apache Polaris is built using Gradle with Java 21+ and Docker 27+. + - `./gradlew build` - To build and run tests. Make sure Docker is running, as the integration tests depend on it. - `./gradlew assemble` - To skip tests. - `./gradlew test` - To run unit tests and integration tests. -- `./gradlew runApp` - To run the Polaris server locally on localhost:8181. + +For local development, you can run the following commands: + +- `./gradlew polarisServerRun` - To run the Polaris server + locally, with profile `prod`; the server is reachable at localhost:8181. +- `./gradlew polarisServerDev` - To run the Polaris server + locally, in [Dev mode](https://quarkus.io/guides/dev-mode-differences). In dev mode, + Polaris uses the `test` Authenticator and `test` TokenBroker; this configuration is suitable for + running regressions tests, or for connecting with Spark. - `./regtests/run_spark_sql.sh` - To connect from Spark SQL. Here are some example commands to run in the Spark SQL shell: ```sql create database db1; @@ -57,36 +78,76 @@ insert into db1.table1 values (1, 'a'); select * from db1.table1; ``` -Apache Polaris supports the following optional build options: -- `-PeclipseLink=true` – Enables the EclipseLink extension. -- `-PeclipseLinkDeps=[groupId]:[artifactId]:[version],...` – Specifies one or more additional dependencies for EclipseLink (e.g., JDBC drivers) separated by commas. - ### More build and run options -Running in Docker -- `docker build -t localhost:5001/polaris:latest .` - To build the image. - - Optional build options: - - `docker build -t localhost:5001/polaris:latest --build-arg ECLIPSELINK=true .` - Enables the EclipseLink extension. - - `docker build -t localhost:5001/polaris:latest --build-arg ECLIPSELINK=true --build-arg ECLIPSELINK_DEPS=[groupId]:[artifactId]:[version],... .` – Enables the EclipseLink extension with one or more additional dependencies for EclipseLink (e.g. JDBC drivers) separated by commas. -- `docker run -p 8181:8181 localhost:5001/polaris:latest` - To run the image in standalone mode. - -Running in Kubernetes -- `./run.sh` - To run Polaris as a mini-deployment locally. This will create one pod that bind itself to ports `8181` and `8182`. - - Optional run options: - - `./run.sh -b "ECLIPSELINK=true"` - Enables the EclipseLink extension. - - `./run.sh -b "ECLIPSELINK=true;ECLIPSELINK_DEPS=[groupId]:[artifactId]:[version],..."` – Enables the EclipseLink extension with one or more additional dependencies for EclipseLink (e.g. JDBC drivers) separated by commas. -- `kubectl port-forward svc/polaris-service -n polaris 8181:8181 8182:8182` - To create secure connections between a local machine and a pod within the cluster for both service and metrics endpoints. - - Currently supported metrics endpoints: - - localhost:8182/metrics - - localhost:8182/healthcheck + +#### Running in Docker + +Please note: there are no official Docker images for Apache Polaris yet. For now, you can build the +Docker images locally. + +To build the Polaris server Docker image locally: + +```shell +./gradlew clean :polaris-quarkus-server:assemble -Dquarkus.container-image.build=true +``` + +To run the Polaris server Docker image: + +```shell +docker run -p 8181:8181 -p 8182:8182 apache/polaris:latest +``` + +#### Running in Kubernetes + +- `./run.sh` - To run Polaris as a mini-deployment locally. This will create a Kind cluster, + then deploy one pod and one service. The service is available on ports `8181` and `8182`. +- `kubectl port-forward svc/polaris-service -n polaris 8181:8181 8182:8182` - To create secure + connections between a local machine and a pod within the cluster for both service and metrics + endpoints. + - Currently supported metrics and health endpoints: + - http://localhost:8182/q/metrics + - http://localhost:8182/q/health - `kubectl get pods -n polaris` - To check the status of the pods. - `kubectl get deployment -n polaris` - To check the status of the deployment. - `kubectl describe deployment polaris-deployment -n polaris` - To troubleshoot if things aren't working as expected. -Running regression tests -- `./regtests/run.sh` - To run regression tests in another terminal. -- `docker compose up --build --exit-code-from regtest` - To run regression tests in a Docker environment. +#### Running regression tests + +Regression tests can be run in a local environment or in a Docker environment. + +To run regression tests locally, you need to have a Polaris server running locally, with the +`test` Authenticator enabled. You can do this by running Polaris in Quarkus Dev mode, as explained +above: + +```shell +./gradlew polarisServerDev +``` + +Then, you can run the regression tests using the following command: + +```shell +./regtests/run.sh +``` + +To run regression tests in a Docker environment, you can use the following command: + +```shell +docker compose -f regtests/docker-compose.yml up --build --exit-code-from regtest +``` + +The above command will by default run Polaris with the Docker image `apache/polaris:latest`; if you +want to use a different image, you can modify the `docker-compose.yaml` file prior to running it; +alternatively, you can use the following commands to override the image: + +```shell +cat < regtests/docker-compose.override.yml +services: { polaris: { image: localhost:5001/apache/polaris:latest } } +EOF +docker compose -f regtests/docker-compose.yml up --build --exit-code-from regtest +``` + +#### Building docs -Building docs - Docs are generated using [Hugo](https://gohugo.io/) using the [Docsy](https://www.docsy.dev/docs/) theme. - To view the site locally, run ```bash diff --git a/extension/persistence/eclipselink/build.gradle.kts b/extension/persistence/eclipselink/build.gradle.kts index 80bd7c61e..30e266612 100644 --- a/extension/persistence/eclipselink/build.gradle.kts +++ b/extension/persistence/eclipselink/build.gradle.kts @@ -65,6 +65,10 @@ dependencies { compileOnly("com.fasterxml.jackson.core:jackson-annotations") compileOnly("com.fasterxml.jackson.core:jackson-core") + // JDBC Drivers + runtimeOnly(libs.h2) + runtimeOnly(libs.postgresql) + testImplementation(libs.h2) testImplementation(testFixtures(project(":polaris-core"))) } diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java index 851320eb6..2c90a6011 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java @@ -44,8 +44,8 @@ public class EclipseLinkPolarisMetaStoreManagerFactory extends LocalPolarisMetaStoreManagerFactory { - private final EclipseLinkConfiguration eclipseLinkConfiguration; - private final PolarisStorageIntegrationProvider storageIntegrationProvider; + @Inject EclipseLinkConfiguration eclipseLinkConfiguration; + @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; public EclipseLinkPolarisMetaStoreManagerFactory() { this(null, null, null, null, null); diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkPersistenceUnit.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkPersistenceUnit.java index 20ac6795f..0e2369350 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkPersistenceUnit.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkPersistenceUnit.java @@ -47,6 +47,7 @@ import org.apache.polaris.extension.persistence.impl.eclipselink.PolarisEclipseLinkPersistenceUnit.ClasspathResourcePolarisEclipseLinkPersistenceUnit; import org.apache.polaris.extension.persistence.impl.eclipselink.PolarisEclipseLinkPersistenceUnit.FileSystemPolarisEclipseLinkPersistenceUnit; import org.apache.polaris.extension.persistence.impl.eclipselink.PolarisEclipseLinkPersistenceUnit.JarFilePolarisEclipseLinkPersistenceUnit; +import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.w3c.dom.NamedNodeMap; import org.w3c.dom.NodeList; @@ -200,7 +201,14 @@ private static Map loadProperties( } // Replace database name in JDBC URL with realm if (properties.containsKey(JDBC_URL)) { - properties.put(JDBC_URL, properties.get(JDBC_URL).replace("{realm}", realmId.id())); + String jdbcUrl = properties.get(JDBC_URL).replace("{realm}", realmId.id()); + properties.put(JDBC_URL, jdbcUrl); + if (jdbcUrl.startsWith("jdbc:h2")) { + LoggerFactory.getLogger(PolarisEclipseLinkPersistenceUnit.class) + .warn( + "Polaris is configured with EclipseLink and an in-memory H2 database; " + + "this is not recommended for production use!"); + } } return properties; } catch (XPathExpressionException diff --git a/extension/persistence/eclipselink/src/main/resources/META-INF/persistence.xml b/extension/persistence/eclipselink/src/main/resources/META-INF/persistence.xml index bee247bb6..69769f05a 100644 --- a/extension/persistence/eclipselink/src/main/resources/META-INF/persistence.xml +++ b/extension/persistence/eclipselink/src/main/resources/META-INF/persistence.xml @@ -34,8 +34,7 @@ org.apache.polaris.jpa.models.ModelSequenceId NONE - + diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6ce53d62b..d7943a046 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -85,6 +85,7 @@ opentelemetry-bom = { module = "io.opentelemetry:opentelemetry-bom", version = " opentelemetry-semconv = { module = "io.opentelemetry.semconv:opentelemetry-semconv", version = "1.25.0-alpha" } picocli = { module = "info.picocli:picocli-codegen", version.ref = "picocli" } picocli-codegen = { module = "info.picocli:picocli-codegen", version.ref = "picocli" } +postgresql = { module = "org.postgresql:postgresql", version = "42.7.4" } prometheus-metrics-exporter-servlet-jakarta = { module = "io.prometheus:prometheus-metrics-exporter-servlet-jakarta", version = "1.3.5" } quarkus-bom = { module = "io.quarkus.platform:quarkus-bom", version.ref = "quarkus" } scala212-lang-library = { module = "org.scala-lang:scala-library", version.ref = "scala212" } diff --git a/helm/polaris/README.md b/helm/polaris/README.md index 3230cc172..1f7aa35a0 100644 --- a/helm/polaris/README.md +++ b/helm/polaris/README.md @@ -45,41 +45,113 @@ A Helm chart for Polaris. ### Optional -When using a custom `persistence.xml`, a Kubernetes Secret must be created for `.persistenceConfigSecret`. Below is a sample command: +When using a custom `persistence.xml`, a Kubernetes Secret must be created for it. Below is a sample command: ```bash kubectl create secret generic polaris-secret -n polaris --from-file=persistence.xml ``` ### From local directory (for development purposes) -From Polaris repo root: +The below instructions assume Minikube is running and Helm is installed. + +If necessary, load the Docker images into Minikube: + +```bash +eval $(minikube -p minikube docker-env) + +./gradlew clean assemble \ + -Dquarkus.container-image.build=true \ + -Dquarkus.container-image.tag=unstable +``` + +Then create and populate the namespace: + +```bash +kubectl create namespace polaris +kubectl apply --namespace polaris -f helm/polaris/fixtures/ +``` + +Finally, install the chart. From Polaris repo root: + +```bash +helm upgrade --install --namespace polaris polaris helm/polaris \ + --set image.tag=unstable \ + --set bootstrap.image.tag=unstable \ + --debug \ + --values helm/polaris/ci/simple-values.yaml +``` + +You can also run `ct` (chart-testing): ```bash -$ helm install polaris helm/polaris --namespace polaris --create-namespace +ct lint --charts helm/polaris +ct install --helm-extra-set-args "--set=image.tag=unstable --set bootstrap.image.tag=unstable" \ + --debug \ + --namespace polaris \ + --charts ./helm/polaris ``` ### Uninstalling the chart ```bash -$ helm uninstall --namespace polaris polaris +helm uninstall --namespace polaris polaris ``` ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| +| advancedConfig | object | `{}` | Advanced configuration. You can pass here any valid Polaris or Quarkus configuration property. Any property that is defined here takes precedence over all the other configuration values generated by this chart. Properties can be passed "flattened" or as nested YAML objects (see examples below). Note: values should be strings; avoid using numbers, booleans, or other types. | | affinity | object | `{}` | Affinity and anti-affinity for polaris pods. See https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity. | +| authentication.authenticator.type | string | `"default"` | | +| authentication.tokenBroker.maxTokenGeneration | string | `"PT1H"` | | +| authentication.tokenBroker.secret.name | string | `nil` | | +| authentication.tokenBroker.secret.privateKey | string | `"private.pem"` | | +| authentication.tokenBroker.secret.publicKey | string | `"public.pem"` | | +| authentication.tokenBroker.secret.secretKey | string | `"secret"` | | +| authentication.tokenBroker.type | string | `"rsa-key-pair"` | | +| authentication.tokenService.type | string | `"default"` | | | autoscaling.enabled | bool | `false` | Specifies whether automatic horizontal scaling should be enabled. Do not enable this when using in-memory version store type. | | autoscaling.maxReplicas | int | `3` | The maximum number of replicas to maintain. | | autoscaling.minReplicas | int | `1` | The minimum number of replicas to maintain. | | autoscaling.targetCPUUtilizationPercentage | int | `80` | Optional; set to zero or empty to disable. | | autoscaling.targetMemoryUtilizationPercentage | string | `nil` | Optional; set to zero or empty to disable. | -| bootstrapExtraEnv | list | `[]` | Extra environment variables to add to the bootstrap metastore manager job (see `extraEnv` for an example) | -| bootstrapMetastoreManager | bool | `false` | Configures whether to enable the bootstrap metastore manager job | +| bootstrap | object | `{"credentials":[],"enabled":false,"extraEnv":[],"image":{"configDir":"/deployments/config","pullPolicy":"IfNotPresent","repository":"apache/polaris-admin-tool","tag":"latest"},"realms":[]}` | Configures whether to enable the bootstrap metastore manager job | +| bootstrap.credentials | list | `[]` | The credentials to create during the bootstrap. If you don't provide credentials for the root principal of each realm to bootstrap, random credentials will be generated. Each entry in the array must be of the form: realm,userName,clientId,clientSecret | +| bootstrap.extraEnv | list | `[]` | Extra environment variables to add to the bootstrap metastore manager job (see `extraEnv` for an example) | +| bootstrap.image | object | `{"configDir":"/deployments/config","pullPolicy":"IfNotPresent","repository":"apache/polaris-admin-tool","tag":"latest"}` | The image configuration for the bootstrap metastore manager job. | +| bootstrap.image.configDir | string | `"/deployments/config"` | The path to the directory where the application.properties file, and other configuration files, if any, should be mounted. | +| bootstrap.image.pullPolicy | string | `"IfNotPresent"` | The image pull policy. | +| bootstrap.image.repository | string | `"apache/polaris-admin-tool"` | The image repository to pull from. | +| bootstrap.image.tag | string | `"latest"` | The image tag. | +| bootstrap.realms | list | `[]` | The names of the realms to bootstrap. | | configMapLabels | object | `{}` | Additional Labels to apply to polaris configmap. | +| containerSecurityContext | object | `{}` | Security context for the polaris container. See https://kubernetes.io/docs/tasks/configure-pod-container/security-context/. | +| cors.accessControlAllowCredentials | bool | `true` | | +| cors.accessControlMaxAge | int | `600` | | +| cors.allowedHeaders[0] | string | `"*"` | | +| cors.allowedMethods[0] | string | `"PATCH"` | | +| cors.allowedMethods[1] | string | `"POST"` | | +| cors.allowedMethods[2] | string | `"DELETE"` | | +| cors.allowedMethods[3] | string | `"GET"` | | +| cors.allowedMethods[4] | string | `"PUT"` | | +| cors.allowedOrigins[0] | string | `"http://localhost:8080"` | | +| cors.exposedHeaders[0] | string | `"*"` | | | extraEnv | list | `[]` | Advanced configuration via Environment Variables. Extra environment variables to add to the Polaris server container. You can pass here any valid EnvVar object: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#envvar-v1-core This can be useful to get configuration values from Kubernetes secrets or config maps. | +| extraInitContainers | list | `[]` | Add additional init containers to the polaris pod(s) See https://kubernetes.io/docs/concepts/workloads/pods/init-containers/. | +| extraServices | list | `[]` | Additional service definitions. All service definitions always select all Polaris pods. Use this if you need to expose specific ports with different configurations, e.g. expose polaris-http with an alternate LoadBalancer service instead of ClusterIP. | +| extraVolumeMounts | list | `[]` | Extra volume mounts to add to the polaris container. See https://kubernetes.io/docs/concepts/storage/volumes/. | +| extraVolumes | list | `[]` | Extra volumes to add to the polaris pod. See https://kubernetes.io/docs/concepts/storage/volumes/. | +| features.defaults.ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING | bool | `false` | | +| features.defaults.SUPPORTED_CATALOG_STORAGE_TYPES[0] | string | `"S3"` | | +| features.defaults.SUPPORTED_CATALOG_STORAGE_TYPES[1] | string | `"GCS"` | | +| features.defaults.SUPPORTED_CATALOG_STORAGE_TYPES[2] | string | `"AZURE"` | | +| features.defaults.SUPPORTED_CATALOG_STORAGE_TYPES[3] | string | `"FILE"` | | +| features.realmOverrides | object | `{}` | | +| fileIo.type | string | `"default"` | | +| image.configDir | string | `"/deployments/config"` | The path to the directory where the application.properties file, and other configuration files, if any, should be mounted. | | image.pullPolicy | string | `"IfNotPresent"` | The image pull policy. | -| image.repository | string | `"localhost:5001/polaris"` | The image repository to pull from. | +| image.repository | string | `"apache/polaris"` | The image repository to pull from. | | image.tag | string | `"latest"` | The image tag. | | imagePullSecrets | list | `[]` | References to secrets in the same namespace to use for pulling any of the images used by this chart. Each entry is a LocalObjectReference to an existing secret in the namespace. The secret must contain a .dockerconfigjson key with a base64-encoded Docker configuration file. See https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ for more information. | | ingress.annotations | object | `{}` | Annotations to add to the ingress. | @@ -94,30 +166,81 @@ $ helm uninstall --namespace polaris polaris | livenessProbe.successThreshold | int | `1` | Minimum consecutive successes for the probe to be considered successful after having failed. Minimum value is 1. | | livenessProbe.terminationGracePeriodSeconds | int | `30` | Optional duration in seconds the pod needs to terminate gracefully upon probe failure. Minimum value is 1. | | livenessProbe.timeoutSeconds | int | `10` | Number of seconds after which the probe times out. Minimum value is 1. | +| logging | object | `{"categories":{"org.apache.iceberg.rest":"INFO","org.apache.polaris":"INFO"},"console":{"enabled":true,"format":"%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] [%X{requestId},%X{realmId}] [%X{traceId},%X{parentId},%X{spanId},%X{sampled}] (%t) %s%e%n","json":false,"threshold":"ALL"},"file":{"enabled":false,"fileName":"polaris.log","format":"%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] [%X{requestId},%X{realmId}] [%X{traceId},%X{parentId},%X{spanId},%X{sampled}] (%t) %s%e%n","json":false,"logsDir":"/deployments/logs","rotation":{"fileSuffix":null,"maxBackupIndex":5,"maxFileSize":"100Mi"},"storage":{"className":"standard","selectorLabels":{},"size":"512Gi"},"threshold":"ALL"},"level":"INFO","mdc":{},"requestIdHeaderName":"request_id"}` | Logging configuration. | +| logging.categories | object | `{"org.apache.iceberg.rest":"INFO","org.apache.polaris":"INFO"}` | Configuration for specific log categories. | +| logging.console | object | `{"enabled":true,"format":"%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] [%X{requestId},%X{realmId}] [%X{traceId},%X{parentId},%X{spanId},%X{sampled}] (%t) %s%e%n","json":false,"threshold":"ALL"}` | Configuration for the console appender. | +| logging.console.enabled | bool | `true` | Whether to enable the console appender. | +| logging.console.format | string | `"%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] [%X{requestId},%X{realmId}] [%X{traceId},%X{parentId},%X{spanId},%X{sampled}] (%t) %s%e%n"` | The log format to use. Ignored if JSON format is enabled. See https://quarkus.io/guides/logging#logging-format for details. | +| logging.console.json | bool | `false` | Whether to log in JSON format. | +| logging.console.threshold | string | `"ALL"` | The log level of the console appender. | +| logging.file | object | `{"enabled":false,"fileName":"polaris.log","format":"%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] [%X{requestId},%X{realmId}] [%X{traceId},%X{parentId},%X{spanId},%X{sampled}] (%t) %s%e%n","json":false,"logsDir":"/deployments/logs","rotation":{"fileSuffix":null,"maxBackupIndex":5,"maxFileSize":"100Mi"},"storage":{"className":"standard","selectorLabels":{},"size":"512Gi"},"threshold":"ALL"}` | Configuration for the file appender. | +| logging.file.enabled | bool | `false` | Whether to enable the file appender. | +| logging.file.fileName | string | `"polaris.log"` | The log file name. | +| logging.file.format | string | `"%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] [%X{requestId},%X{realmId}] [%X{traceId},%X{parentId},%X{spanId},%X{sampled}] (%t) %s%e%n"` | The log format to use. Ignored if JSON format is enabled. See https://quarkus.io/guides/logging#logging-format for details. | +| logging.file.json | bool | `false` | Whether to log in JSON format. | +| logging.file.logsDir | string | `"/deployments/logs"` | The local directory where log files are stored. The persistent volume claim will be mounted here. | +| logging.file.rotation | object | `{"fileSuffix":null,"maxBackupIndex":5,"maxFileSize":"100Mi"}` | Log rotation configuration. | +| logging.file.rotation.fileSuffix | string | `nil` | An optional suffix to append to the rotated log files. If present, the rotated log files will be grouped in time buckets, and each bucket will contain at most maxBackupIndex files. The suffix must be in a date-time format that is understood by DateTimeFormatter. If the suffix ends with .gz or .zip, the rotated files will also be compressed using the corresponding algorithm. | +| logging.file.rotation.maxBackupIndex | int | `5` | The maximum number of backup files to keep. | +| logging.file.rotation.maxFileSize | string | `"100Mi"` | The maximum size of the log file before it is rotated. Should be expressed as a Kubernetes quantity. | +| logging.file.storage | object | `{"className":"standard","selectorLabels":{},"size":"512Gi"}` | The log storage configuration. A persistent volume claim will be created using these settings. | +| logging.file.storage.className | string | `"standard"` | The storage class name of the persistent volume claim to create. | +| logging.file.storage.selectorLabels | object | `{}` | Labels to add to the persistent volume claim spec selector; a persistent volume with matching labels must exist. Leave empty if using dynamic provisioning. | +| logging.file.storage.size | string | `"512Gi"` | The size of the persistent volume claim to create. | +| logging.file.threshold | string | `"ALL"` | The log level of the file appender. | +| logging.level | string | `"INFO"` | The log level of the root category, which is used as the default log level for all categories. | +| logging.mdc | object | `{}` | Configuration for MDC (Mapped Diagnostic Context). Values specified here will be added to the log context of all incoming requests and can be used in log patterns. | +| logging.requestIdHeaderName | string | `"request_id"` | The header name to use for the request ID. | +| managementService | object | `{"annotations":{},"clusterIP":"None","externalTrafficPolicy":null,"internalTrafficPolicy":null,"ports":[{"name":"polaris-mgmt","port":8182}],"sessionAffinity":null,"trafficDistribution":null,"type":"ClusterIP"}` | Management service settings. These settings are used to configure liveness and readiness probes, and to configure the dedicated headless service that will expose health checks and metrics, e.g. for metrics scraping and service monitoring. | +| managementService.annotations | object | `{}` | Annotations to add to the service. | +| managementService.ports | list | `[{"name":"polaris-mgmt","port":8182}]` | The ports the management service will listen on. At least one port is required; the first port implicitly becomes the HTTP port that the application will use for serving management requests. By default, it's 8182. Note: port names must be unique and no more than 15 characters long. | +| managementService.ports[0] | object | `{"name":"polaris-mgmt","port":8182}` | The name of the management port. Required. | +| managementService.ports[0].port | int | `8182` | The port the management service listens on. By default, the management interface is exposed on HTTP port 8182. | +| managementService.type | string | `"ClusterIP"` | The type of service to create. | +| metrics.enabled | bool | `true` | Specifies whether metrics for the polaris server should be enabled. | +| metrics.tags | object | `{}` | Additional tags (dimensional labels) to add to the metrics. | | nodeSelector | object | `{}` | Node labels which must match for the polaris pod to be scheduled on that node. See https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector. | -| persistenceConfigSecret | string | `nil` | The secret name to pull persistence.xml from (ensure the key name is 'persistence.xml') | +| persistence.eclipseLink.persistenceUnit | string | `"polaris"` | The persistence unit name to use. Only required if persistenceConfigSecret is set. | +| persistence.eclipseLink.secret | object | `{"key":"persistence.xml","name":null}` | The secret name to pull persistence.xml from. If not provided, the default persistence.xml will be used. | +| persistence.type | string | `"eclipse-link"` | | | podAnnotations | object | `{}` | Annotations to apply to polaris pods. | | podLabels | object | `{}` | Additional Labels to apply to polaris pods. | | podSecurityContext | object | `{}` | Security context for the polaris pod. See https://kubernetes.io/docs/tasks/configure-pod-container/security-context/. | -| polarisServerConfig | object | `{"authenticator":{"class":"org.apache.polaris.service.auth.TestInlineBearerTokenPolarisAuthenticator"},"callContextResolver":{"type":"default"},"cors":{"allowed-credentials":true,"allowed-headers":["*"],"allowed-methods":["PATCH","POST","DELETE","GET","PUT"],"allowed-origins":["http://localhost:8080"],"allowed-timing-origins":["http://localhost:8080"],"exposed-headers":["*"],"preflight-max-age":600},"defaultRealms":["default-realm"],"featureConfiguration":{"ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING":false,"SUPPORTED_CATALOG_STORAGE_TYPES":["S3","GCS","AZURE","FILE"]},"io":{"factoryType":"default"},"logging":{"appenders":[{"logFormat":"%-5p [%d{ISO8601} - %-6r] [%t] [%X{aid}%X{sid}%X{tid}%X{wid}%X{oid}%X{srv}%X{job}%X{rid}] %c{30}: %m %kvp%n%ex","threshold":"ALL","type":"console"}],"level":"INFO","loggers":{"org.apache.iceberg.rest":"DEBUG","org.apache.polaris":"DEBUG"}},"maxRequestBodyBytes":-1,"metaStoreManager":{"type":"in-memory"},"oauth2":{"type":"test"},"rateLimiter":{"type":"no-op"},"realmContextResolver":{"type":"default"},"server":{"adminConnectors":[{"port":8182,"type":"http"}],"applicationConnectors":[{"port":8181,"type":"http"}],"maxThreads":200,"minThreads":10,"requestLog":{"appenders":[{"type":"console"}]}}}` | Configures for polaris-server.yml | +| rateLimiter.tokenBucket.requestsPerSecond | int | `1000` | | +| rateLimiter.tokenBucket.type | string | `"default"` | | +| rateLimiter.tokenBucket.window | string | `"PT10S"` | | +| rateLimiter.type | string | `"no-op"` | | | readinessProbe | object | `{"failureThreshold":3,"initialDelaySeconds":5,"periodSeconds":10,"successThreshold":1,"timeoutSeconds":10}` | Configures the readiness probe for polaris pods. | | readinessProbe.failureThreshold | int | `3` | Minimum consecutive failures for the probe to be considered failed after having succeeded. Minimum value is 1. | | readinessProbe.initialDelaySeconds | int | `5` | Number of seconds after the container has started before readiness probes are initiated. Minimum value is 0. | | readinessProbe.periodSeconds | int | `10` | How often (in seconds) to perform the probe. Minimum value is 1. | | readinessProbe.successThreshold | int | `1` | Minimum consecutive successes for the probe to be considered successful after having failed. Minimum value is 1. | | readinessProbe.timeoutSeconds | int | `10` | Number of seconds after which the probe times out. Minimum value is 1. | +| realmContext.realms[0] | string | `"default-realm"` | | +| realmContext.type | string | `"default"` | | | replicaCount | int | `1` | The number of replicas to deploy (horizontal scaling). Beware that replicas are stateless; don't set this number > 1 when using in-memory meta store manager. | | resources | object | `{}` | Configures the resources requests and limits for polaris pods. We usually recommend not to specify default resources and to leave this as a conscious choice for the user. This also increases chances charts run on environments with little resources, such as Minikube. If you do want to specify resources, uncomment the following lines, adjust them as necessary, and remove the curly braces after 'resources:'. | | revisionHistoryLimit | string | `nil` | The number of old ReplicaSets to retain to allow rollback (if not set, the default Kubernetes value is set to 10). | -| securityContext | object | `{}` | Security context for the polaris container. See https://kubernetes.io/docs/tasks/configure-pod-container/security-context/. | +| service | object | `{"annotations":{},"clusterIP":"","externalTrafficPolicy":"Cluster","internalTrafficPolicy":"Cluster","ports":[{"name":"polaris-http","port":8181}],"sessionAffinity":"None","trafficDistribution":"PreferClose","type":"ClusterIP"}` | Polaris main service settings. | | service.annotations | object | `{}` | Annotations to add to the service. | -| service.ports | object | `{"polaris-metrics":8182,"polaris-service":8181}` | The ports the service will listen on. Two ports are required: one for the Polaris service and one for the metrics API. Note: port names must be unique and no more than 15 characters long. | +| service.clusterIP | string | `""` | You can specify your own cluster IP address If you define a Service that has the .spec.clusterIP set to "None" then Kubernetes does not assign an IP address. Instead, DNS records for the service will return the IP addresses of each pod targeted by the server. This is called a headless service. See https://kubernetes.io/docs/concepts/services-networking/service/#headless-services | +| service.internalTrafficPolicy | string | `"Cluster"` | The traffic policy fields control how traffic from internal and external sources are routed respectively. Valid values are Cluster and Local. Set the field to Cluster to route traffic to all ready endpoints. Set the field to Local to only route to ready node-local endpoints. If the traffic policy is Local and there are no node-local endpoints, traffic is dropped by kube-proxy | +| service.ports | list | `[{"name":"polaris-http","port":8181}]` | The ports the service will listen on. At least one port is required; the first port implicitly becomes the HTTP port that the application will use for serving API requests. By default, it's 8181. Note: port names must be unique and no more than 15 characters long. | | service.sessionAffinity | string | `"None"` | The session affinity for the service. Valid values are: None, ClientIP. ClientIP enables sticky sessions based on the client's IP address. This is generally beneficial to Polaris deployments, but some testing may be required in order to make sure that the load is distributed evenly among the pods. Also, this setting affects only internal clients, not external ones. If Ingress is enabled, it is recommended to set sessionAffinity to None. | +| service.trafficDistribution | string | `"PreferClose"` | The traffic distribution field provides another way to influence traffic routing within a Kubernetes Service. While traffic policies focus on strict semantic guarantees, traffic distribution allows you to express preferences such as routing to topologically closer endpoints. Valid values are: PreferClose | | service.type | string | `"ClusterIP"` | The type of service to create. | | serviceAccount.annotations | object | `{}` | Annotations to add to the service account. | | serviceAccount.create | bool | `true` | Specifies whether a service account should be created. | | serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template. | +| serviceMonitor.enabled | bool | `true` | Specifies whether a ServiceMonitor for Prometheus operator should be created. | +| serviceMonitor.interval | string | `""` | The scrape interval; leave empty to let Prometheus decide. Must be a valid duration, e.g. 1d, 1h30m, 5m, 10s. | +| serviceMonitor.labels | object | `{}` | Labels for the created ServiceMonitor so that Prometheus operator can properly pick it up. | +| serviceMonitor.metricRelabelings | list | `[]` | Relabeling rules to apply to metrics. Ref https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config. | +| storage | object | `{"aws":{"accessKey":null,"secretKey":null},"gcp":{"lifespan":"PT1H","token":null}}` | Storage credentials for the server. If the following properties are unset, default credentials will be used. | +| tasks.maxConcurrentTasks | int | `100` | | +| tasks.maxQueuedTasks | int | `1000` | | | tolerations | list | `[]` | A list of tolerations to apply to polaris pods. See https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/. | -| toolsImage.pullPolicy | string | `"IfNotPresent"` | The image pull policy. | -| toolsImage.repository | string | `"registry.access.redhat.com/ubi9/openjdk-21"` | The image repository to pull from (must have jar binary included). | -| toolsImage.tag | string | `"latest"` | The image tag. | \ No newline at end of file +| tracing.attributes | object | `{}` | Resource attributes to identify the polaris service among other tracing sources. See https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/#service. If left empty, traces will be attached to a service named "Polaris"; to change this, provide a service.name attribute here. | +| tracing.enabled | bool | `false` | Specifies whether tracing for the polaris server should be enabled. | +| tracing.endpoint | string | `"http://otlp-collector:4317"` | The collector endpoint URL to connect to (required). The endpoint URL must have either the http:// or the https:// scheme. The collector must talk the OpenTelemetry protocol (OTLP) and the port must be its gRPC port (by default 4317). See https://quarkus.io/guides/opentelemetry for more information. | +| tracing.sample | string | `"1.0d"` | Which requests should be sampled. Valid values are: "all", "none", or a ratio between 0.0 and "1.0d" (inclusive). E.g. "0.5d" means that 50% of the requests will be sampled. | \ No newline at end of file diff --git a/helm/polaris/README.md.gotmpl b/helm/polaris/README.md.gotmpl index 43a5f2be0..79a6a6d92 100644 --- a/helm/polaris/README.md.gotmpl +++ b/helm/polaris/README.md.gotmpl @@ -49,23 +49,56 @@ ### Optional -When using a custom `persistence.xml`, a Kubernetes Secret must be created for `.persistenceConfigSecret`. Below is a sample command: +When using a custom `persistence.xml`, a Kubernetes Secret must be created for it. Below is a sample command: ```bash kubectl create secret generic polaris-secret -n polaris --from-file=persistence.xml ``` ### From local directory (for development purposes) -From Polaris repo root: +The below instructions assume Minikube is running and Helm is installed. + +If necessary, load the Docker images into Minikube: + +```bash +eval $(minikube -p minikube docker-env) + +./gradlew clean assemble \ + -Dquarkus.container-image.build=true \ + -Dquarkus.container-image.tag=unstable +``` + +Then create and populate the namespace: + +```bash +kubectl create namespace polaris +kubectl apply --namespace polaris -f helm/polaris/fixtures/ +``` + +Finally, install the chart. From Polaris repo root: + +```bash +helm upgrade --install --namespace polaris polaris helm/polaris \ + --set image.tag=unstable \ + --set bootstrap.image.tag=unstable \ + --debug \ + --values helm/polaris/ci/simple-values.yaml +``` + +You can also run `ct` (chart-testing): ```bash -$ helm install polaris helm/polaris --namespace polaris --create-namespace +ct lint --charts helm/polaris +ct install --helm-extra-set-args "--set=image.tag=unstable --set bootstrap.image.tag=unstable" \ + --debug \ + --namespace polaris \ + --charts ./helm/polaris ``` ### Uninstalling the chart ```bash -$ helm uninstall --namespace polaris polaris +helm uninstall --namespace polaris polaris ``` {{ template "chart.valuesSection" . }} \ No newline at end of file diff --git a/helm/polaris/ci/fixtures/token-broker.yaml b/helm/polaris/ci/fixtures/token-broker.yaml new file mode 100644 index 000000000..5a10fbc08 --- /dev/null +++ b/helm/polaris/ci/fixtures/token-broker.yaml @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +apiVersion: v1 +kind: Secret +metadata: + name: polaris-token-broker +type: Opaque +stringData: + private.pem: |- + -----BEGIN PRIVATE KEY----- + MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDiae5jQnQ+Dt8Optpa4fLNuUfI + 37Sy29JPfn22CkGNo54SJi8NtR38Q958Bx9B+30IZPm3KtZGPC38Tm78/C508h7/Bd0jO7S3/UuQ + rYlg9H8gC59j14fE5cKEiKCvHrNK35Act9LrfhwEoGBs7G+90KD3SBIWzoc4vS8AB/ey09kRN3+M + dQ/DuRrafRgEx8ecq0x4KPggrmvworVVm1ya+H2JNKUQ+uQugmwxeRDE1GuJrgNnn4+ajueEnGdc + ohgz067AgZExzoyRlujJXBuL6JRtOSbEYVpNV1laVdA8UJ2b5I1PiNnVvx1m4gFi/Cela479EO6t + +AC+a1ez+MxTAgMBAAECggEAA1QiXJQ4CEeIsPrz/90NiEe+je34Bp8jtuZ5q6j38MCkjMjB7VYp + 1DEdPr6OD5bGbk6puy8mik7NMEXo7NA6UJ7UpNQtvZxdmkzfhQeAGwxNKCfzlnCvPs5P70ng7uIp + KaMix8FOGQc4GztOX5uUQvqjuy70kSBFLajeXYZAdoEOu+E3QDaCUqiJlK3cRAJhg39Br+XfOHDp + cBdhlbxqQKZHWqDR010XUNoQpMHXWTu/e779Xze9uAvmYw98YGtCW8JB5AqjnqAmCZFaktKYw5Z3 + IHZCA9hTVngk0WTanuIeTlB2bbl0df+87xHfim2yDuXEkTfAOCug/Qpf+pEhfQKBgQD9M14a1MQ2 + Omf9f7X401Ig32e2YfX6sW73THB8Lupz9MCBZxNFCZlzVl55bL2o2vOROEeaneGdboqASW+1ph6e + QbdipiYAfHsG/mvkysqH/plo8QT69GUlzs2yt5zdmFjvkwaZmRmbB6Gjg2sSHKBt3YjTPbdmpuKV + oLvpmaslvQKBgQDk6r+4CGTZwfKYcrTNqJYPdOwfaVjSCulm0wHRw3PvJEi5Eb5qNaJbNICLl5FD + wkOAg33CZ+9h9FJnPd2IbBUBUGcNL5sGNZZNMH16LuFlRYKGgYxjlOjktOC5OXXa7iqkwMqf1dFf + GYU/wHGXYamUKOlinA9rFmbQCUNJL6mzTwKBgQC5Ky/O/3rfDaevRA6YgjK1x6vZPZreU5SLVmOM + 7eKYGo4OFIuLGZIXPzqoIlYxfq5RrY7wDDZLI+Q2HX6MoYgSxIyQoGF6SY7PdpEBKS0kd5VJobm6 + rbCrDapw4MbfZ+LiketpuQV40wPqyNkszbKlpjXCBohxkepy8rF6DN+VIQKBgFYHFnedx/DB316F + NQdYxNQYN6hyWavN5/r5b2SaVFZZx26tiWa2s0YdS/WpxzC0r9N7FubZUm/4doNQD5H2NEqjIacl + 7dd+ifaGM5GYTqJVZgEQbGb8Di3s/8r8Ghtlh+cUgNFidGWN3LKhqs6eKDGC0np5dZ3j9E1YPc4i + OF8nAoGAYeFjfXBXKQhQXItTlL5NMd+yNvMGJDOvNYOn90acZrDld8GvqHI9A1kQf2IuYfA8Yfxg + O2lpCWjiPAsAC3W5BMhfkzjxpikW6YLpXXx7wpDzNAUBAuhEHlmFSMOoa4CIBnN2+6Zw8PSsKnrI + 49N28pje/ZyABaY7SYP3fcJY17o= + -----END PRIVATE KEY----- + public.pem: |- + -----BEGIN PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4mnuY0J0Pg7fDqbaWuHyzblHyN+0stvS + T359tgpBjaOeEiYvDbUd/EPefAcfQft9CGT5tyrWRjwt/E5u/PwudPIe/wXdIzu0t/1LkK2JYPR/ + IAufY9eHxOXChIigrx6zSt+QHLfS634cBKBgbOxvvdCg90gSFs6HOL0vAAf3stPZETd/jHUPw7ka + 2n0YBMfHnKtMeCj4IK5r8KK1VZtcmvh9iTSlEPrkLoJsMXkQxNRria4DZ5+Pmo7nhJxnXKIYM9Ou + wIGRMc6MkZboyVwbi+iUbTkmxGFaTVdZWlXQPFCdm+SNT4jZ1b8dZuIBYvwnpWuO/RDurfgAvmtX + s/jMUwIDAQAB + -----END PUBLIC KEY----- diff --git a/helm/polaris/ci/persistence-values.yaml b/helm/polaris/ci/persistence-values.yaml index 5c196bb73..e975e6318 100644 --- a/helm/polaris/ci/persistence-values.yaml +++ b/helm/polaris/ci/persistence-values.yaml @@ -18,16 +18,28 @@ # image: - repository: polaris - tag: latest pullPolicy: Never -bootstrapMetastoreManager: true +authentication: + tokenBroker: + secret: + name: polaris-token-broker -persistenceConfigSecret: polaris-persistence +logging: + file: + enabled: true + json: true + storage: + size: 50Mi -polarisServerConfig: - metaStoreManager: - type: eclipse-link - persistence-unit: polaris - conf-file: /eclipselink-config/conf.jar!/persistence.xml +persistence: + type: eclipse-link + eclipseLink: + secret: + name: polaris-persistence + +bootstrap: + enabled: true + realms: ["realm1", "realm2"] + image: + pullPolicy: Never diff --git a/helm/polaris/ci/simple-values.yaml b/helm/polaris/ci/simple-values.yaml index 9aa65dd5b..0e6d291da 100644 --- a/helm/polaris/ci/simple-values.yaml +++ b/helm/polaris/ci/simple-values.yaml @@ -18,6 +18,4 @@ # image: - repository: polaris - tag: latest pullPolicy: Never diff --git a/helm/polaris/templates/_helpers.tpl b/helm/polaris/templates/_helpers.tpl index 3514deb61..4069ff722 100644 --- a/helm/polaris/templates/_helpers.tpl +++ b/helm/polaris/templates/_helpers.tpl @@ -42,6 +42,29 @@ {{- end }} {{- end }} +{{/* + Create a default fully qualified app name, with a custom suffix. Useful when the name will + have a suffix appended to it, such as for the management service name. +*/}} +{{- define "polaris.fullnameWithSuffix" -}} +{{- $global := index . 0 }} +{{- $suffix := index . 1 }} +{{- if not (hasPrefix "-" $suffix) }} +{{- $suffix = printf "-%s" $suffix }} +{{- end }} +{{- $length := int (sub 63 (len $suffix)) }} +{{- if $global.Values.fullnameOverride }} +{{- $global.Values.fullnameOverride | trunc $length }}{{ $suffix }} +{{- else }} +{{- $name := default $global.Chart.Name $global.Values.nameOverride }} +{{- if contains $name $global.Release.Name }} +{{- $global.Release.Name | trunc $length }}{{ $suffix }} +{{- else }} +{{- printf "%s-%s" $global.Release.Name $name | trunc $length }}{{ $suffix }} +{{- end }} +{{- end }} +{{- end }} + {{/* Create chart name and version as used by the chart label. */}} @@ -79,3 +102,198 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} + +{{/* +Merges the advanced configuration into the destination map. +*/}} +{{- define "polaris.mergeAdvancedConfig" -}} +{{- $advConfig := index . 0 -}} +{{- $prefix := index . 1 -}} +{{- $dest := index . 2 -}} +{{- range $key, $val := $advConfig -}} +{{- $name := ternary $key (print $prefix "." $key) (eq $prefix "") -}} +{{- if kindOf $val | eq "map" -}} +{{- list $val $name $dest | include "polaris.mergeAdvancedConfig" -}} +{{- else -}} +{{- $_ := set $dest $name $val -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Prints the configuration option to the destination configmap entry. See confimap.yaml. +Any nil values will be printed as empty config options; otherwise, the value will be evaluated +as a template against the global context, then printed. Furthermore, if the value contains +line breaks, they will be escaped and a multi-line option will be printed. +*/}} +{{- define "polaris.appendConfigOption" -}} +{{- $key := index . 0 -}} +{{- $value := index . 1 -}} +{{- $global := index . 2 -}} +{{- $valAsString := "" -}} +{{- if ne $value nil -}} +{{- $valAsString = tpl (toString $value) $global -}} +{{- if contains "\r\n" $valAsString -}} +{{- $valAsString = $valAsString | nindent 4 | replace "\r\n" "\\\r\n" -}} +{{- else if contains "\n" $valAsString -}} +{{- $valAsString = $valAsString | nindent 4 | replace "\n" "\\\n" -}} +{{- end -}} +{{- end -}} +{{ print $key "=" $valAsString }} +{{- end -}} + +{{/* +Convert a dict into a string formed by a comma-separated list of key-value pairs: key1=value1,key2=value2, ... +*/}} +{{- define "polaris.dictToString" -}} +{{- $list := list -}} +{{- range $k, $v := . -}} +{{- $list = append $list (printf "%s=%s" $k $v) -}} +{{- end -}} +{{ join "," $list }} +{{- end -}} + +{{/* +Prints the config volume definition for deployments and jobs. +*/}} +{{- define "polaris.configVolume" -}} +- name: config-volume + projected: + sources: + - configMap: + name: {{ include "polaris.fullname" . }} + items: + - key: application.properties + path: application.properties + {{- if .Values.authentication.tokenBroker.secret.name }} + - secret: + name: {{ tpl .Values.authentication.tokenBroker.secret.name . }} + items: + {{- if eq .Values.authentication.tokenBroker.type "rsa-key-pair" }} + - key: {{ tpl .Values.authentication.tokenBroker.secret.publicKey . }} + path: public.pem + - key: {{ tpl .Values.authentication.tokenBroker.secret.privateKey . }} + path: private.pem + {{- end }} + {{- if eq .Values.authentication.tokenBroker.type "symmetric-key" }} + - key: {{ tpl .Values.authentication.tokenBroker.secret.secretKey . }} + path: symmetric.key + {{- end }} + {{- end }} + {{- if and ( eq .Values.persistence.type "eclipse-link" ) .Values.persistence.eclipseLink.secret.name }} + - secret: + name: {{ tpl .Values.persistence.eclipseLink.secret.name . }} + items: + - key: {{ tpl .Values.persistence.eclipseLink.secret.key . }} + path: persistence.xml + {{- end }} +{{- end -}} + +{{/* +Converts a Kubernetes quantity to a number (int64 if possible or float64 otherwise). +It handles raw numbers as well as quantities with suffixes +like m, k, M, G, T, P, E, ki, Mi, Gi, Ti, Pi, Ei. +It also handles scientific notation. +Quantities should be positive, so negative values, zero, or any unparseable number +will result in a failure. +https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/ +*/}} +{{- define "polaris.quantity" -}} +{{- $quantity := . -}} +{{- $n := $quantity | float64 -}} +{{- if kindIs "string" $quantity -}} +{{- if hasSuffix "m" $quantity -}} +{{- $n = divf (trimSuffix "m" $quantity | float64) 1000.0 -}} +{{- else if hasSuffix "k" $quantity -}} +{{- $n = trimSuffix "k" $quantity | int64 | mul 1000 -}} +{{- else if hasSuffix "M" $quantity -}} +{{- $n = trimSuffix "M" $quantity | int64 | mul 1000000 -}} +{{- else if hasSuffix "G" $quantity -}} +{{- $n = trimSuffix "G" $quantity | int64 | mul 1000000000 -}} +{{- else if hasSuffix "T" $quantity -}} +{{- $n = trimSuffix "T" $quantity | int64 | mul 1000000000000 -}} +{{- else if hasSuffix "P" $quantity -}} +{{- $n = trimSuffix "P" $quantity | int64 | mul 1000000000000000 -}} +{{- else if hasSuffix "E" $quantity -}} +{{- $n = trimSuffix "E" $quantity | int64 | mul 1000000000000000000 -}} +{{- else if hasSuffix "ki" $quantity -}} +{{- $n = trimSuffix "ki" $quantity | int64 | mul 1024 -}} +{{- else if hasSuffix "Mi" $quantity -}} +{{- $n = trimSuffix "Mi" $quantity | int64 | mul 1048576 -}} +{{- else if hasSuffix "Gi" $quantity -}} +{{- $n = trimSuffix "Gi" $quantity | int64 | mul 1073741824 -}} +{{- else if hasSuffix "Ti" $quantity -}} +{{- $n = trimSuffix "Ti" $quantity | int64 | mul 1099511627776 -}} +{{- else if hasSuffix "Pi" $quantity -}} +{{- $n = trimSuffix "Pi" $quantity | int64 | mul 1125899906842624 -}} +{{- else if hasSuffix "Ei" $quantity -}} +{{- $n = trimSuffix "Ei" $quantity | int64 | mul 1152921504606846976 -}} +{{- end -}} +{{- end -}} +{{- if le ($n | float64) 0.0 -}} +{{- fail (print "invalid quantity: " $quantity) -}} +{{- end -}} +{{- if kindIs "float64" $n -}} +{{- printf "%f" $n -}} +{{- else -}} +{{- printf "%v" $n -}} +{{- end -}} +{{- end -}} + +{{/* +Prints the ports section of the container spec. Also validates all port names and numbers to ensure +that they are consistent and that there are no overlaps. +*/}} +{{- define "polaris.containerPorts" -}} +ports: +{{- $ports := dict -}} + +{{- /* Main service ports */ -}} +{{- range $i, $port := .Values.service.ports -}} +{{- if hasKey $ports $port.name -}} +{{- fail (printf "service.ports[%d]: port name already taken: %v" $i $port.name) -}} +{{- end -}} +{{- $portNumber := coalesce $port.targetPort $port.port -}} +{{- if has $portNumber (values $ports) -}} +{{- fail (printf "service.ports[%d]: port number already taken: %v" $i $portNumber) -}} +{{- end -}} +{{- $_ := set $ports $port.name $portNumber }} +- name: {{ $port.name }} + containerPort: {{ coalesce $port.targetPort $port.port }} + protocol: {{ $port.protocol | default "TCP" }} +{{- end -}} + +{{- /* Management service ports */ -}} +{{- range $i, $port := .Values.managementService.ports -}} +{{- if hasKey $ports $port.name -}} +{{- fail (printf "managementService.ports[%d]: port name already taken: %v" $i $port.name) -}} +{{- end -}} +{{- $portNumber := coalesce $port.targetPort $port.port -}} +{{- if has $portNumber (values $ports) -}} +{{- fail (printf "managementService.ports[%d]: port number already taken: %v" $i $portNumber) -}} +{{- end -}} +{{- $_ := set $ports $port.name $portNumber }} +- name: {{ $port.name }} + containerPort: {{ coalesce $port.targetPort $port.port }} + protocol: {{ $port.protocol | default "TCP" }} +{{- end -}} + +{{- /* Extra service ports */ -}} +{{- range $i, $svc := .Values.extraServices -}} +{{- range $j, $port := $svc.ports -}} +{{- $portNumber := coalesce $port.targetPort $port.port -}} +{{- if hasKey $ports $port.name -}} +{{- if ne $portNumber (get $ports $port.name) -}} +{{- fail (printf "extraServices[%d].ports[%d]: wrong port number for port %s, expected %v, got %v" $i $j $port.name (get $ports $port.name) $portNumber) -}} +{{- end -}} +{{- else if has $portNumber (values $ports) -}} +{{- fail (printf "extraServices[%d].ports[%d]: port number already taken: %v" $i $j $portNumber) -}} +{{- end -}} +{{- $_ := set $ports $port.name $portNumber }} +- name: {{ $port.name }} + containerPort: {{ coalesce $port.targetPort $port.port }} + protocol: {{ $port.protocol | default "TCP" }} +{{- end -}} +{{- end -}} + +{{- end -}} diff --git a/helm/polaris/templates/configmap.yaml b/helm/polaris/templates/configmap.yaml index 97bc983e5..3cb1c91c1 100644 --- a/helm/polaris/templates/configmap.yaml +++ b/helm/polaris/templates/configmap.yaml @@ -28,5 +28,140 @@ metadata: {{- tpl (toYaml .Values.configMapLabels) . | nindent 4 }} {{- end }} data: - polaris-server.yml: |- -{{ toYaml .Values.polarisServerConfig | indent 4 }} \ No newline at end of file + application.properties: |- + {{- $map := dict -}} + + {{- /* Realm Context */ -}} + {{- $_ := set $map "polaris.realm-context.type" .Values.realmContext.type -}} + {{- $_ = set $map "polaris.realm-context.realms" (join "," .Values.realmContext.realms) -}} + + {{- /* Features */ -}} + {{- range $k, $v := .Values.features.defaults -}} + {{- $_ = set $map (printf "polaris.features.defaults.\"%s\"" $k) (toJson $v) -}} + {{- end -}} + {{- range $realm, $overrides := .Values.features.realmOverrides -}} + {{- range $k, $v := $overrides -}} + {{- $_ = set $map (printf "polaris.config.realm-overrides.\"%s\".\"%s\"" $realm $k) (toJson $v) -}} + {{- end -}} + {{- end -}} + + {{- /* Persistence */ -}} + {{- $_ = set $map "polaris.persistence.type" .Values.persistence.type -}} + {{- if and ( eq .Values.persistence.type "eclipse-link" ) .Values.persistence.eclipseLink.secret.name -}} + {{- $_ = set $map "polaris.persistence.eclipselink.persistence-unit" .Values.persistence.eclipseLink.persistenceUnit -}} + {{- $_ = set $map "polaris.persistence.eclipselink.configuration-file" (printf "%s/persistence.xml" .Values.image.configDir ) -}} + {{- end -}} + + {{- /* File IO */ -}} + {{- $_ = set $map "polaris.file-io.type" .Values.fileIo.type -}} + + {{- /* Storage */ -}} + {{- if .Values.storage.aws.accessKey -}} + {{- $_ = set $map "polaris.storage.aws.access-key" .Values.storage.aws.accessKey -}} + {{- end -}} + {{- if .Values.storage.aws.secretKey -}} + {{- $_ = set $map "polaris.storage.aws.secret-key" .Values.storage.aws.secretKey -}} + {{- end -}} + {{- if .Values.storage.gcp.token -}} + {{- $_ = set $map "polaris.storage.gcp.token" .Values.storage.gcp.token -}} + {{- end -}} + {{- if .Values.storage.gcp.lifespan -}} + {{- $_ = set $map "polaris.storage.gcp.lifespan" .Values.storage.gcp.lifespan -}} + {{- end -}} + + {{- /* Authentication */ -}} + {{- $_ = set $map "polaris.authentication.authenticator.type" .Values.authentication.authenticator.type -}} + {{- $_ = set $map "polaris.authentication.token-service.type" .Values.authentication.tokenService.type -}} + {{- $_ = set $map "polaris.authentication.token-broker.type" .Values.authentication.tokenBroker.type -}} + {{- $_ = set $map "polaris.authentication.token-broker.max-token-generation" .Values.authentication.tokenBroker.maxTokenGeneration -}} + {{- if .Values.authentication.tokenBroker.secret.name }} + {{- if eq .Values.authentication.tokenBroker.type "rsa-key-pair" -}} + {{- $_ = set $map "polaris.authentication.token-broker.rsa-key-pair.public-key-file" (printf "%s/public.pem" .Values.image.configDir ) -}} + {{- $_ = set $map "polaris.authentication.token-broker.rsa-key-pair.private-key-file" (printf "%s/private.pem" .Values.image.configDir ) -}} + {{- end -}} + {{- if eq .Values.authentication.tokenBroker.type "symmetric-key" -}} + {{- $_ = set $map "polaris.authentication.token-broker.symmetric-key.file" (printf "%s/symmetric.key" .Values.image.configDir ) -}} + {{- end -}} + {{- end -}} + + {{- /* HTTP ports */ -}} + {{- $_ = set $map "quarkus.http.port" (get (first .Values.service.ports) "port") -}} + {{- $_ = set $map "quarkus.management.port" (get (first .Values.managementService.ports) "port") -}} + + {{- /* Logging */ -}} + {{- $_ = set $map "quarkus.log.level" .Values.logging.level -}} + {{- if .Values.logging.console.enabled -}} + {{- $_ = set $map "quarkus.log.console.enable" "true" -}} + {{- $_ = set $map "quarkus.log.console.level" .Values.logging.console.threshold -}} + {{- if .Values.logging.console.json -}} + {{- $_ = set $map "quarkus.log.console.json" "true" -}} + {{- else -}} + {{- $_ = set $map "quarkus.log.console.format" .Values.logging.console.format -}} + {{- end -}} + {{- else -}} + {{- $_ = set $map "quarkus.log.console.enable" "false" -}} + {{- end -}} + {{- if .Values.logging.file.enabled -}} + {{- $_ = set $map "quarkus.log.file.enable" "true" -}} + {{- $_ = set $map "quarkus.log.file.level" .Values.logging.file.threshold -}} + {{- $_ = set $map "quarkus.log.file.path" (printf "%s/%s" .Values.logging.file.logsDir .Values.logging.file.fileName) -}} + {{- $_ = set $map "quarkus.log.file.rotation.max-file-size" (include "polaris.quantity" .Values.logging.file.rotation.maxFileSize) -}} + {{- $_ = set $map "quarkus.log.file.rotation.max-backup-index" .Values.logging.file.rotation.maxBackupIndex -}} + {{- if .Values.logging.file.rotation.fileSuffix -}} + {{- $_ = set $map "quarkus.log.file.rotation.file-suffix" .Values.logging.file.rotation.fileSuffix -}} + {{- end -}} + {{- if .Values.logging.file.json -}} + {{- $_ = set $map "quarkus.log.file.json" "true" -}} + {{- else -}} + {{- $_ = set $map "quarkus.log.file.format" .Values.logging.file.format -}} + {{- end -}} + {{- else -}} + {{- $_ = set $map "quarkus.log.file.enable" "false" -}} + {{- end -}} + {{- $categories := dict -}} + {{- list .Values.logging.categories "" $categories | include "polaris.mergeAdvancedConfig" -}} + {{- range $k, $v := $categories -}} + {{- $_ = set $map (printf "quarkus.log.category.\"%s\".level" $k) $v -}} + {{- end -}} + {{- $_ = set $map "polaris.log.request-id-header-name" .Values.logging.requestIdHeaderName -}} + {{- range $k, $v := .Values.logging.mdc -}} + {{- $_ = set $map (printf "polaris.log.mdc.\"%s\"" $k) $v -}} + {{- end -}} + + {{- /* Telemetry */ -}} + {{- if .Values.tracing.enabled -}} + {{- $_ = set $map "quarkus.otel.exporter.otlp.endpoint" .Values.tracing.endpoint -}} + {{- if .Values.tracing.attributes -}} + {{- $_ = set $map "quarkus.otel.resource.attributes" (include "polaris.dictToString" .Values.tracing.attributes) -}} + {{- end -}} + {{- if .Values.tracing.sample -}} + {{ if eq .Values.tracing.sample "all" -}} + {{- $_ = set $map "quarkus.otel.traces.sampler" "parentbased_always_on" -}} + {{- else if eq .Values.tracing.sample "none" -}} + {{- $_ = set $map "quarkus.otel.traces.sampler" "always_off" -}} + {{- else -}} + {{- $_ = set $map "quarkus.otel.traces.sampler" "parentbased_traceidratio" -}} + {{- $_ = set $map "quarkus.otel.traces.sampler.arg" .Values.tracing.sample -}} + {{- end -}} + {{- end -}} + {{- else -}} + {{- $_ = set $map "quarkus.otel.sdk.disabled" true -}} + {{- end -}} + + {{- /* Metrics */ -}} + {{- if .Values.metrics.enabled -}} + {{- range $name, $value := .Values.metrics.tags -}} + {{- $_ = set $map (print "polaris.metrics.tags." $name) $value -}} + {{- end -}} + {{- else -}} + {{- $_ = set $map "quarkus.micrometer.enabled" "false" -}} + {{- end -}} + + {{- /* Advanced Configuration (must be done last since it can override any of the settings above) */ -}} + {{- list .Values.advancedConfig "" $map | include "polaris.mergeAdvancedConfig" }} + + {{- /* Print the resulting configmap; each configuration option is templatized */ -}} + {{- $global := . -}} + {{- range $k, $v := $map }} + {{ include "polaris.appendConfigOption" (list $k $v $global) }} + {{- end }} diff --git a/helm/polaris/templates/deployment.yaml b/helm/polaris/templates/deployment.yaml index 286baafd1..a603429f7 100644 --- a/helm/polaris/templates/deployment.yaml +++ b/helm/polaris/templates/deployment.yaml @@ -49,23 +49,6 @@ spec: {{- tpl (toYaml .Values.podLabels) . | nindent 8 }} {{- end }} spec: - {{- if and .Values.persistenceConfigSecret (index .Values.polarisServerConfig.metaStoreManager "conf-file") }} - initContainers: - - name: init-config - image: "{{ tpl .Values.toolsImage.repository . }}:{{ tpl .Values.toolsImage.tag . }}" - imagePullPolicy: {{ tpl .Values.toolsImage.pullPolicy . }} - command: ["jar"] - args: ["-cf", "/eclipselink-config/conf.jar", "-C", "/secret", "persistence.xml"] - {{- if .Values.securityContext}} - securityContext: - {{- tpl (toYaml .Values.securityContext) . | nindent 12 }} - {{- end }} - volumeMounts: - - name: eclipselink-config-volume - mountPath: /eclipselink-config - - name: secret-volume - mountPath: /secret - {{- end }} {{- if .Values.imagePullSecrets }} imagePullSecrets: {{- range .Values.imagePullSecrets }} @@ -77,38 +60,42 @@ spec: securityContext: {{- tpl (toYaml .Values.podSecurityContext) . | nindent 8 }} {{- end }} + {{- if .Values.extraInitContainers }} + initContainers: + {{- tpl (toYaml .Values.extraInitContainers) . | nindent 8 }} + {{- end }} containers: - name: {{ .Chart.Name }} - {{- if .Values.securityContext}} + {{- if .Values.containerSecurityContext }} securityContext: - {{- tpl (toYaml .Values.securityContext) . | nindent 12 }} + {{- tpl (toYaml .Values.containerSecurityContext) . | nindent 12 }} {{- end }} image: "{{ tpl .Values.image.repository . }}:{{ tpl .Values.image.tag . | default .Chart.Version }}" imagePullPolicy: {{ tpl .Values.image.pullPolicy . }} - command: ["/app/bin/polaris-service"] - args: ["server", "/app/config/polaris-server.yml"] {{- if .Values.extraEnv }} env: {{- tpl (toYaml .Values.extraEnv) . | nindent 12 }} {{- end }} volumeMounts: - name: config-volume - mountPath: /app/config/polaris-server.yml - subPath: polaris-server.yml - {{- if and .Values.persistenceConfigSecret (index .Values.polarisServerConfig.metaStoreManager "conf-file") }} - - name: eclipselink-config-volume - mountPath: /eclipselink-config + mountPath: {{ trimSuffix "/" .Values.image.configDir }} + readOnly: true + {{- if .Values.logging.file.enabled }} + - name: logs-storage + mountPath: {{ .Values.logging.file.logsDir }} + readOnly: false {{- end }} - ports: - {{- range $portName, $portNumber := .Values.service.ports }} - - name: {{ $portName }} - containerPort: {{ $portNumber }} - protocol: TCP + - name: temp-dir + mountPath: /tmp + readOnly: false + {{- if .Values.extraVolumeMounts }} + {{- tpl (toYaml .Values.extraVolumeMounts) . | nindent 12 }} {{- end }} + {{- include "polaris.containerPorts" . | trim | nindent 10 }} livenessProbe: httpGet: - path: /healthcheck - port: "polaris-metrics" + path: /q/health/live + port: {{ get (first .Values.managementService.ports) "name" }} scheme: HTTP initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.livenessProbe.periodSeconds }} @@ -118,8 +105,8 @@ spec: terminationGracePeriodSeconds: {{ .Values.livenessProbe.terminationGracePeriodSeconds }} readinessProbe: httpGet: - path: /healthcheck - port: "polaris-metrics" + path: /q/health/ready + port: {{ get (first .Values.managementService.ports) "name" }} scheme: HTTP initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.readinessProbe.periodSeconds }} @@ -131,15 +118,16 @@ spec: {{- tpl (toYaml .Values.resources) . | nindent 12 }} {{- end }} volumes: - - name: config-volume - configMap: - name: {{ include "polaris.fullname" . }} - {{- if and .Values.persistenceConfigSecret (index .Values.polarisServerConfig.metaStoreManager "conf-file") }} - - name: eclipselink-config-volume + {{- include "polaris.configVolume" . | nindent 8 }} + {{- if .Values.logging.file.enabled }} + - name: logs-storage + persistentVolumeClaim: + claimName: {{ include "polaris.fullnameWithSuffix" (list . "logs") }} + {{- end }} + - name: temp-dir emptyDir: {} - - name: secret-volume - secret: - secretName: {{ tpl .Values.persistenceConfigSecret . }} + {{- if .Values.extraVolumes }} + {{- tpl (toYaml .Values.extraVolumes) . | nindent 8 }} {{- end }} {{- if .Values.nodeSelector }} nodeSelector: diff --git a/helm/polaris/templates/ingress.yaml b/helm/polaris/templates/ingress.yaml index 7495be5b5..f1909cc62 100644 --- a/helm/polaris/templates/ingress.yaml +++ b/helm/polaris/templates/ingress.yaml @@ -19,7 +19,7 @@ {{- if .Values.ingress.enabled }} {{- $fullName := include "polaris.fullname" . }} -{{- $svcPort := index .Values.service.ports "polaris-service" }} +{{- $svcPort := get (first .Values.service.ports) "port" }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: diff --git a/helm/polaris/templates/job.yaml b/helm/polaris/templates/job.yaml index 5dc54b2bb..18a3b8db1 100644 --- a/helm/polaris/templates/job.yaml +++ b/helm/polaris/templates/job.yaml @@ -17,7 +17,7 @@ under the License. */}} -{{- if .Values.bootstrapMetastoreManager }} +{{- if .Values.bootstrap.enabled }} apiVersion: batch/v1 kind: Job metadata: @@ -43,23 +43,6 @@ spec: {{- tpl (toYaml .Values.podLabels) . | nindent 8 }} {{- end }} spec: - {{- if and .Values.persistenceConfigSecret (index .Values.polarisServerConfig.metaStoreManager "conf-file") }} - initContainers: - - name: init-config - image: "{{ tpl .Values.toolsImage.repository . }}:{{ tpl .Values.toolsImage.tag . }}" - imagePullPolicy: {{ tpl .Values.toolsImage.pullPolicy . }} - command: ["jar"] - args: ["-cf", "/eclipselink-config/conf.jar", "-C", "/secret", "persistence.xml"] - {{- if .Values.securityContext}} - securityContext: - {{- tpl (toYaml .Values.securityContext) . | nindent 12 }} - {{- end }} - volumeMounts: - - name: eclipselink-config-volume - mountPath: /eclipselink-config - - name: secret-volume - mountPath: /secret - {{- end }} {{- if .Values.imagePullSecrets }} imagePullSecrets: {{- range .Values.imagePullSecrets }} @@ -67,47 +50,50 @@ spec: {{- end }} {{- end }} serviceAccountName: {{ include "polaris.serviceAccountName" . }} - {{- if .Values.podSecurityContext}} + {{- if .Values.podSecurityContext }} securityContext: {{- tpl (toYaml .Values.podSecurityContext) . | nindent 8 }} {{- end }} containers: - name: {{ .Chart.Name }} - {{- if .Values.securityContext}} + {{- if .Values.containerSecurityContext }} securityContext: - {{- tpl (toYaml .Values.securityContext) . | nindent 12 }} + {{- tpl (toYaml .Values.containerSecurityContext) . | nindent 12 }} {{- end }} - image: "{{ tpl .Values.image.repository . }}:{{ tpl .Values.image.tag . | default .Chart.Version }}" - imagePullPolicy: {{ tpl .Values.image.pullPolicy . }} - command: ["/app/bin/polaris-service"] - args: ["bootstrap", "/app/config/polaris-server.yml"] - {{- if .Values.bootstrapExtraEnv }} + image: "{{ tpl .Values.bootstrap.image.repository . }}:{{ tpl .Values.bootstrap.image.tag . | default .Chart.Version }}" + imagePullPolicy: {{ tpl .Values.bootstrap.image.pullPolicy . }} + args: + - "bootstrap" + {{- range $realm := .Values.bootstrap.realms }} + - --realm={{ $realm }} + {{- end }} + {{- range $credential := .Values.bootstrap.credentials }} + - --credential={{ $credential }} + {{- end }} env: - {{- tpl (toYaml .Values.bootstrapExtraEnv) . | nindent 12 }} - {{- end }} + {{- if .Values.bootstrap.extraEnv }} + {{- tpl (toYaml .Values.bootstrap.extraEnv) . | nindent 12 }} + {{- end }} + - name: "quarkus.log.file.enable" + value: "false" volumeMounts: - name: config-volume - mountPath: /app/config/polaris-server.yml - subPath: polaris-server.yml - {{- if and .Values.persistenceConfigSecret (index .Values.polarisServerConfig.metaStoreManager "conf-file") }} - - name: eclipselink-config-volume - mountPath: /eclipselink-config - {{- end }} + mountPath: {{ trimSuffix "/" .Values.bootstrap.image.configDir }} + readOnly: true + - name: temp-dir + mountPath: /tmp + readOnly: false {{- if .Values.resources }} resources: {{- tpl (toYaml .Values.resources) . | nindent 12 }} {{- end }} restartPolicy: Never volumes: - - name: config-volume - configMap: - name: {{ include "polaris.fullname" . }} - {{- if and .Values.persistenceConfigSecret (index .Values.polarisServerConfig.metaStoreManager "conf-file") }} - - name: eclipselink-config-volume + {{- include "polaris.configVolume" . | nindent 8 }} + - name: temp-dir emptyDir: {} - - name: secret-volume - secret: - secretName: {{ tpl .Values.persistenceConfigSecret . }} + {{- if .Values.extraVolumes }} + {{- tpl (toYaml .Values.extraVolumes) . | nindent 8 }} {{- end }} {{- if .Values.nodeSelector }} nodeSelector: diff --git a/helm/polaris/templates/service-extra.yaml b/helm/polaris/templates/service-extra.yaml new file mode 100644 index 000000000..b08f1b06c --- /dev/null +++ b/helm/polaris/templates/service-extra.yaml @@ -0,0 +1,69 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +{{- range $i, $svc := .Values.extraServices }} +{{- if not $svc.nameSuffix }} +{{- fail (printf "extraServices[%d]: missing required nameSuffix" $i) }} +{{- else if eq $svc.nameSuffix "-mgmt" }} +{{- fail (printf "extraServices[%d]: invalid nameSuffix" $i) }} +{{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "polaris.fullnameWithSuffix" (list . $svc.nameSuffix) }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "polaris.labels" $ | nindent 4 }} + {{- with $svc.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ $svc.type }} + selector: + {{- include "polaris.selectorLabels" $ | nindent 4 }} + ports: + {{- range $svc.ports }} + - name: {{ .name }} + port: {{ .port }} + {{- if .targetPort }} + targetPort: {{ .targetPort }} + {{- end }} + {{- if .nodePort }} + nodePort: {{ .nodePort }} + {{- end }} + protocol: {{ default "TCP" .protocol }} + {{- end }} + {{- if $svc.sessionAffinity }} + sessionAffinity: {{ $svc.sessionAffinity }} + {{- end }} + {{- if $svc.clusterIP }} + clusterIP: {{ $svc.clusterIP }} + {{- end }} + {{- if and $svc.externalTrafficPolicy (or (eq $svc.type "LoadBalancer") (eq $svc.type "NodePort")) }} + externalTrafficPolicy: {{ $svc.externalTrafficPolicy }} + {{- end }} + {{- if $svc.internalTrafficPolicy }} + internalTrafficPolicy: {{ $svc.internalTrafficPolicy }} + {{- end }} + {{- if and $svc.trafficDistribution (ge (int $.Capabilities.KubeVersion.Minor) 31) }} + trafficDistribution: {{ $svc.trafficDistribution }} + {{- end }} +{{- end }} diff --git a/helm/polaris/templates/service-mgmt.yaml b/helm/polaris/templates/service-mgmt.yaml new file mode 100644 index 000000000..4843458b4 --- /dev/null +++ b/helm/polaris/templates/service-mgmt.yaml @@ -0,0 +1,61 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "polaris.fullnameWithSuffix" (list . "mgmt") }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "polaris.labels" . | nindent 4 }} + {{- with .Values.managementService.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.managementService.type }} + selector: + {{- include "polaris.selectorLabels" . | nindent 4 }} + ports: + {{- range .Values.managementService.ports }} + - name: {{ .name }} + port: {{ .port }} + {{- if .targetPort }} + targetPort: {{ .targetPort }} + {{- end }} + {{- if .nodePort }} + nodePort: {{ .nodePort }} + {{- end }} + protocol: {{ default "TCP" .protocol }} + {{- end }} + {{- if .Values.managementService.sessionAffinity }} + sessionAffinity: {{ .Values.managementService.sessionAffinity }} + {{- end }} + {{- if .Values.managementService.clusterIP }} + clusterIP: {{ .Values.managementService.clusterIP }} + {{- end }} + {{- if and .Values.managementService.externalTrafficPolicy (or (eq .Values.managementService.type "LoadBalancer") (eq .Values.managementService.type "NodePort")) }} + externalTrafficPolicy: {{ .Values.managementService.externalTrafficPolicy }} + {{- end }} + {{- if and .Values.managementService.internalTrafficPolicy }} + internalTrafficPolicy: {{ .Values.managementService.internalTrafficPolicy }} + {{- end }} + {{- if and .Values.managementService.trafficDistribution (ge (int $.Capabilities.KubeVersion.Minor) 31) }} + trafficDistribution: {{ .Values.managementService.trafficDistribution }} + {{- end }} \ No newline at end of file diff --git a/helm/polaris/templates/service.yaml b/helm/polaris/templates/service.yaml index 628cd4716..6bde527ef 100644 --- a/helm/polaris/templates/service.yaml +++ b/helm/polaris/templates/service.yaml @@ -24,19 +24,38 @@ metadata: namespace: {{ .Release.Namespace }} labels: {{- include "polaris.labels" . | nindent 4 }} - {{- if .Values.service.annotations }} + {{- with .Values.service.annotations }} annotations: - {{- tpl (toYaml .Values.service.annotations) . | nindent 4 }} + {{- toYaml . | nindent 4 }} {{- end }} spec: type: {{ .Values.service.type }} selector: {{- include "polaris.selectorLabels" . | nindent 4 }} ports: - {{- range $portName, $portNumber := .Values.service.ports }} - - port: {{ $portNumber }} - targetPort: {{ $portNumber }} - protocol: TCP - name: {{ $portName }} + {{- range .Values.service.ports }} + - name: {{ .name }} + port: {{ .port }} + {{- if .targetPort }} + targetPort: {{ .targetPort }} + {{- end }} + {{- if .nodePort }} + nodePort: {{ .nodePort }} + {{- end }} + protocol: {{ default "TCP" .protocol }} {{- end }} + {{- if .Values.service.sessionAffinity }} sessionAffinity: {{ .Values.service.sessionAffinity }} + {{- end }} + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} + {{- if and .Values.service.externalTrafficPolicy (or (eq .Values.service.type "LoadBalancer") (eq .Values.service.type "NodePort")) }} + externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} + {{- end }} + {{- if and .Values.service.internalTrafficPolicy }} + internalTrafficPolicy: {{ .Values.service.internalTrafficPolicy }} + {{- end }} + {{- if and .Values.service.trafficDistribution (ge (int .Capabilities.KubeVersion.Minor) 31) }} + trafficDistribution: {{ .Values.service.trafficDistribution }} + {{- end }} diff --git a/helm/polaris/templates/servicemonitor.yaml b/helm/polaris/templates/servicemonitor.yaml new file mode 100644 index 000000000..e8eca1fde --- /dev/null +++ b/helm/polaris/templates/servicemonitor.yaml @@ -0,0 +1,48 @@ +{{/** + + Copyright (C) 2024 Dremio + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +**/}} + +{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "polaris.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "polaris.labels" . | nindent 4 }} + {{- if .Values.serviceMonitor.labels }} + {{ toYaml .Values.serviceMonitor.labels | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: {{ get (first .Values.managementService.ports) "port" }} + scheme: http + {{- if .Values.serviceMonitor.interval }} + interval: {{ .Values.serviceMonitor.interval }} + {{- end }} + path: /q/metrics + {{- with .Values.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + {{- include "polaris.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/helm/polaris/templates/storage.yaml b/helm/polaris/templates/storage.yaml new file mode 100644 index 000000000..ded3f9cf3 --- /dev/null +++ b/helm/polaris/templates/storage.yaml @@ -0,0 +1,40 @@ +{{/** + + Copyright (C) 2024 Dremio + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +**/}} + +{{- if .Values.logging.file.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "polaris.fullnameWithSuffix" (list . "logs") }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "polaris.labels" . | nindent 4 }} +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + storageClassName: {{ .Values.logging.file.storage.className }} + resources: + requests: + storage: "{{ .Values.logging.file.storage.size }}" +{{- if .Values.logging.file.storage.selectorLabels }} + selector: + matchLabels: + {{- toYaml .Values.logging.file.storage.selectorLabels | nindent 6 }} +{{- end }} +{{- end }} diff --git a/helm/polaris/templates/tests/test-connection.yaml b/helm/polaris/templates/tests/test-connection.yaml index 50f49939f..fef843dea 100644 --- a/helm/polaris/templates/tests/test-connection.yaml +++ b/helm/polaris/templates/tests/test-connection.yaml @@ -35,5 +35,7 @@ spec: - name: wget image: busybox command: ['wget'] - args: ['{{ include "polaris.fullname" . }}:{{ index .Values.service.ports "polaris-metrics" }}/healthcheck'] + args: + - --spider + - '{{ include "polaris.fullnameWithSuffix" (list . "mgmt") }}:{{ get (first .Values.managementService.ports) "port" }}/q/health/ready' restartPolicy: Never \ No newline at end of file diff --git a/helm/polaris/tests/configmap_test.yaml b/helm/polaris/tests/configmap_test.yaml index 859ae1992..9fa20bc32 100644 --- a/helm/polaris/tests/configmap_test.yaml +++ b/helm/polaris/tests/configmap_test.yaml @@ -78,213 +78,3 @@ tests: path: metadata.labels content: app.kubernetes.io/component: polaris - - # data - ## there is an auto newline for logFormat in the config map - ## auto newline of long string is reported in https://github.com/helm/helm/issues/5568 and due to https://github.com/go-yaml/yaml/issues/166 - - it: should set config map default data - asserts: - - equal: - path: data - value: - polaris-server.yml: |- - authenticator: - class: org.apache.polaris.service.auth.TestInlineBearerTokenPolarisAuthenticator - callContextResolver: - type: default - cors: - allowed-credentials: true - allowed-headers: - - '*' - allowed-methods: - - PATCH - - POST - - DELETE - - GET - - PUT - allowed-origins: - - http://localhost:8080 - allowed-timing-origins: - - http://localhost:8080 - exposed-headers: - - '*' - preflight-max-age: 600 - defaultRealms: - - default-realm - featureConfiguration: - ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING: false - SUPPORTED_CATALOG_STORAGE_TYPES: - - S3 - - GCS - - AZURE - - FILE - io: - factoryType: default - logging: - appenders: - - logFormat: '%-5p [%d{ISO8601} - %-6r] [%t] [%X{aid}%X{sid}%X{tid}%X{wid}%X{oid}%X{srv}%X{job}%X{rid}] - %c{30}: %m %kvp%n%ex' - threshold: ALL - type: console - level: INFO - loggers: - org.apache.iceberg.rest: DEBUG - org.apache.polaris: DEBUG - maxRequestBodyBytes: -1 - metaStoreManager: - type: in-memory - oauth2: - type: test - rateLimiter: - type: no-op - realmContextResolver: - type: default - server: - adminConnectors: - - port: 8182 - type: http - applicationConnectors: - - port: 8181 - type: http - maxThreads: 200 - minThreads: 10 - requestLog: - appenders: - - type: console - - it: should set config map data (auto sorted) - set: - polarisServerConfig: - server: - maxThreads: 200 - minThreads: 10 - applicationConnectors: - - type: http - port: 8181 - adminConnectors: - - type: http - port: 8182 - requestLog: - appenders: - - type: console - featureConfiguration: - ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING: false - SUPPORTED_CATALOG_STORAGE_TYPES: - - S3 - callContextResolver: - type: default - realmContextResolver: - type: default - defaultRealms: - - default-realm - metaStoreManager: - type: eclipse-link - persistence-unit: polaris - conf-file: /eclipselink-config/conf.jar!/persistence.xml - io: - factoryType: default - oauth2: - type: default - tokenBroker: - type: symmetric-key - secret: polaris - authenticator: - class: org.apache.polaris.service.auth.DefaultPolarisAuthenticator - cors: - allowed-origins: - - http://localhost:8080 - allowed-timing-origins: - - http://localhost:8080 - allowed-methods: - - PATCH - - POST - - DELETE - - GET - - PUT - allowed-headers: - - "*" - exposed-headers: - - "*" - preflight-max-age: 600 - allowed-credentials: true - logging: - level: INFO - loggers: - org.apache.iceberg.rest: INFO - org.apache.polaris: INFO - appenders: - - type: console - threshold: ALL - logFormat: "%-5p [%d{ISO8601} - %-6r] [%t] [%X{aid}%X{sid}%X{tid}%X{wid}%X{oid}%X{srv}%X{job}%X{rid}] %c{30}: %m %kvp%n%ex" - maxRequestBodyBytes: -1 - rateLimiter: - type: no-op - asserts: - - equal: - path: data - value: - polaris-server.yml: |- - authenticator: - class: org.apache.polaris.service.auth.DefaultPolarisAuthenticator - callContextResolver: - type: default - cors: - allowed-credentials: true - allowed-headers: - - '*' - allowed-methods: - - PATCH - - POST - - DELETE - - GET - - PUT - allowed-origins: - - http://localhost:8080 - allowed-timing-origins: - - http://localhost:8080 - exposed-headers: - - '*' - preflight-max-age: 600 - defaultRealms: - - default-realm - featureConfiguration: - ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING: false - SUPPORTED_CATALOG_STORAGE_TYPES: - - S3 - io: - factoryType: default - logging: - appenders: - - logFormat: '%-5p [%d{ISO8601} - %-6r] [%t] [%X{aid}%X{sid}%X{tid}%X{wid}%X{oid}%X{srv}%X{job}%X{rid}] - %c{30}: %m %kvp%n%ex' - threshold: ALL - type: console - level: INFO - loggers: - org.apache.iceberg.rest: INFO - org.apache.polaris: INFO - maxRequestBodyBytes: -1 - metaStoreManager: - conf-file: /eclipselink-config/conf.jar!/persistence.xml - persistence-unit: polaris - type: eclipse-link - oauth2: - type: default - rateLimiter: - type: no-op - realmContextResolver: - type: default - server: - adminConnectors: - - port: 8182 - type: http - applicationConnectors: - - port: 8181 - type: http - maxThreads: 200 - minThreads: 10 - requestLog: - appenders: - - type: console - tokenBroker: - secret: polaris - type: symmetric-key diff --git a/helm/polaris/tests/deployment_test.yaml b/helm/polaris/tests/deployment_test.yaml index 161f27383..360daefe2 100644 --- a/helm/polaris/tests/deployment_test.yaml +++ b/helm/polaris/tests/deployment_test.yaml @@ -175,37 +175,6 @@ tests: content: foo: bar - # spec.template.spec.initContainers - - it: should not set initContainers by default - asserts: - - notExists: - path: spec.template.spec.initContainers - - it: should set initContainers if persistence is enabled - set: - persistenceConfigSecret: polaris-persistence-secret - polarisServerConfig: - metaStoreManager: - conf-file: /eclipselink-config/conf.jar!/persistence.xml - toolsImage: - repository: test-repo - pullPolicy: Always - asserts: - - exists: - path: spec.template.spec.initContainers - - contains: - path: spec.template.spec.initContainers - content: - name: init-config - image: test-repo:latest - imagePullPolicy: Always - command: [ "jar" ] - args: [ "-cf", "/eclipselink-config/conf.jar", "-C", "/secret", "persistence.xml" ] - volumeMounts: - - name: eclipselink-config-volume - mountPath: /eclipselink-config - - name: secret-volume - mountPath: /secret - # spec.template.spec.imagePullSecrets - it: should not set imagePullSecrets by default asserts: @@ -291,7 +260,7 @@ tests: path: spec.template.spec.containers[0].securityContext - it: should set container securityContext set: - securityContext: + containerSecurityContext: runAsUser: 1000 asserts: - isSubset: @@ -299,30 +268,6 @@ tests: content: runAsUser: 1000 - # spec.template.spec.containers[0].securityContext - - it: should not set initContainer securityContext by default - set: - persistenceConfigSecret: polaris-persistence-secret - polarisServerConfig: - metaStoreManager: - conf-file: /eclipselink-config/conf.jar!/persistence.xml - asserts: - - notExists: - path: spec.template.spec.initContainers[0].securityContext - - it: should set initContainer securityContext - set: - persistenceConfigSecret: polaris-persistence-secret - polarisServerConfig: - metaStoreManager: - conf-file: /eclipselink-config/conf.jar!/persistence.xml - securityContext: - runAsUser: 1000 - asserts: - - isSubset: - path: spec.template.spec.initContainers[0].securityContext - content: - runAsUser: 1000 - # spec.template.spec.containers[0].image - it: should set container image set: @@ -384,58 +329,70 @@ tests: asserts: - lengthEqual: path: spec.template.spec.volumes - count: 1 + count: 2 # config-volume + temp-dir - lengthEqual: path: spec.template.spec.containers[0].volumeMounts - count: 1 + count: 2 # config-volume + temp-dir - contains: path: spec.template.spec.containers[0].volumeMounts content: name: config-volume - mountPath: /app/config/polaris-server.yml - subPath: polaris-server.yml + mountPath: /deployments/config + readOnly: true - it: should set persistence volumes if persistence is enabled set: - persistenceConfigSecret: polaris-persistence-secret - polarisServerConfig: - metaStoreManager: - conf-file: /eclipselink-config/conf.jar!/persistence.xml + persistence: + type: eclipse-link + eclipseLink: + secret: + name: polaris-persistence-secret asserts: - lengthEqual: path: spec.template.spec.volumes - count: 3 + count: 2 - lengthEqual: path: spec.template.spec.containers[0].volumeMounts count: 2 - contains: path: spec.template.spec.volumes content: - name: eclipselink-config-volume - emptyDir: { } - - contains: - path: spec.template.spec.volumes - content: - name: secret-volume - secret: - secretName: polaris-persistence-secret - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: eclipselink-config-volume - mountPath: /eclipselink-config + name: config-volume + projected: + sources: + - configMap: + items: + - key: application.properties + path: application.properties + name: polaris-release + - secret: + items: + - key: persistence.xml + path: persistence.xml + name: polaris-persistence-secret - it: should evaluate template expressions in persistence secret name set: - persistenceConfigSecret: '{{ .Release.Name }}-persistence-secret' - polarisServerConfig: - metaStoreManager: - conf-file: /eclipselink-config/conf.jar!/persistence.xml + persistence: + type: eclipse-link + eclipseLink: + secret: + name: "{{ .Release.Name }}-persistence-secret" asserts: - contains: path: spec.template.spec.volumes content: - name: secret-volume - secret: - secretName: polaris-release-persistence-secret + name: config-volume + projected: + sources: + - configMap: + items: + - key: application.properties + path: application.properties + name: polaris-release + - secret: + items: + - key: persistence.xml + path: persistence.xml + name: polaris-release-persistence-secret # spec.template.spec.containers[0].ports - it: should set container ports by default @@ -446,13 +403,13 @@ tests: - contains: path: spec.template.spec.containers[0].ports content: - name: polaris-service + name: polaris-http containerPort: 8181 protocol: TCP - contains: path: spec.template.spec.containers[0].ports content: - name: polaris-metrics + name: polaris-mgmt containerPort: 8182 protocol: TCP diff --git a/helm/polaris/tests/ingress_test.yaml b/helm/polaris/tests/ingress_test.yaml index ba296fd66..d9613bf54 100644 --- a/helm/polaris/tests/ingress_test.yaml +++ b/helm/polaris/tests/ingress_test.yaml @@ -45,7 +45,7 @@ tests: kind: Ingress apiVersion: networking.k8s.io/v1 - # metadata.name (with with ingress enabled) + # metadata.name (with ingress enabled) - it: should set ingress name set: ingress.enabled: true diff --git a/helm/polaris/tests/job_test.yaml b/helm/polaris/tests/job_test.yaml index 4c83e7e27..78a48739a 100644 --- a/helm/polaris/tests/job_test.yaml +++ b/helm/polaris/tests/job_test.yaml @@ -39,23 +39,23 @@ tests: not: true - it: should create job when enabled set: - bootstrapMetastoreManager: true + bootstrap.enabled: true asserts: - containsDocument: kind: Job apiVersion: batch/v1 - # metadata.name (with bootstrapMetastoreManager enabled) + # metadata.name (with bootstrap.enabled true) - it: should set job name set: - bootstrapMetastoreManager: true + bootstrap.enabled: true asserts: - equal: path: metadata.name value: polaris-release - it: should set job name with override set: - bootstrapMetastoreManager: true + bootstrap.enabled: true nameOverride: polaris-override asserts: - equal: @@ -63,26 +63,26 @@ tests: value: polaris-release-polaris-override - it: should set job name with full override set: - bootstrapMetastoreManager: true + bootstrap.enabled: true fullnameOverride: polaris-override asserts: - equal: path: metadata.name value: polaris-override - # metadata.namespace (with bootstrapMetastoreManager enabled) + # metadata.namespace (with bootstrap.enabled true) - it: should set job namespace set: - bootstrapMetastoreManager: true + bootstrap.enabled: true asserts: - equal: path: metadata.namespace value: polaris-ns - # metadata.labels (with bootstrapMetastoreManager enabled) + # metadata.labels (with bootstrap.enabled true) - it: should set job default labels set: - bootstrapMetastoreManager: true + bootstrap.enabled: true asserts: - isSubset: path: metadata.labels @@ -94,7 +94,7 @@ tests: helm.sh/chart: polaris-1.2.3 - it: should set podLabels in job labels set: - bootstrapMetastoreManager: true + bootstrap.enabled: true podLabels: app.kubernetes.io/component: polaris asserts: @@ -103,26 +103,26 @@ tests: content: app.kubernetes.io/component: polaris - # metadata.annotations (with bootstrapMetastoreManager enabled) + # metadata.annotations (with bootstrap.enabled true) - it: should set job annotations set: - bootstrapMetastoreManager: true + bootstrap.enabled: true asserts: - equal: path: metadata.annotations value: "helm.sh/hook": post-install - # spec.template.metadata.annotations (with bootstrapMetastoreManager enabled) + # spec.template.metadata.annotations (with bootstrap.enabled true) - it: should not set pod annotations by default set: - bootstrapMetastoreManager: true + bootstrap.enabled: true asserts: - notExists: path: spec.template.metadata.annotations - it: should set pod annotations set: - bootstrapMetastoreManager: true + bootstrap.enabled: true podAnnotations: foo: bar asserts: @@ -131,10 +131,10 @@ tests: content: foo: bar - # spec.template.metadata.labels (with bootstrapMetastoreManager enabled) + # spec.template.metadata.labels (with bootstrap.enabled true) - it: should set pod labels set: - bootstrapMetastoreManager: true + bootstrap.enabled: true asserts: - isSubset: path: spec.template.metadata.labels @@ -143,7 +143,7 @@ tests: app.kubernetes.io/instance: polaris-release - it: should set podLabels in pod labels set: - bootstrapMetastoreManager: true + bootstrap.enabled: true podLabels: app.kubernetes.io/component: polaris asserts: @@ -152,50 +152,16 @@ tests: content: app.kubernetes.io/component: polaris - # spec.template.spec.initContainers (with bootstrapMetastoreManager enabled) - - it: should not set initContainers by default - set: - bootstrapMetastoreManager: true - asserts: - - notExists: - path: spec.template.spec.initContainers - - it: should set initContainers if persistence is enabled - set: - bootstrapMetastoreManager: true - persistenceConfigSecret: polaris-persistence-secret - polarisServerConfig: - metaStoreManager: - conf-file: /eclipselink-config/conf.jar!/persistence.xml - toolsImage: - repository: test-repo - pullPolicy: Always - asserts: - - exists: - path: spec.template.spec.initContainers - - contains: - path: spec.template.spec.initContainers - content: - name: init-config - image: test-repo:latest - imagePullPolicy: Always - command: [ "jar" ] - args: [ "-cf", "/eclipselink-config/conf.jar", "-C", "/secret", "persistence.xml" ] - volumeMounts: - - name: eclipselink-config-volume - mountPath: /eclipselink-config - - name: secret-volume - mountPath: /secret - - # spec.template.spec.imagePullSecrets (with bootstrapMetastoreManager enabled) + # spec.template.spec.imagePullSecrets (with bootstrap.enabled true) - it: should not set imagePullSecrets by default set: - bootstrapMetastoreManager: true + bootstrap.enabled: true asserts: - notExists: path: spec.template.spec.imagePullSecrets - it: should set imagePullSecrets set: - bootstrapMetastoreManager: true + bootstrap.enabled: true imagePullSecrets: - test-secret asserts: @@ -204,17 +170,17 @@ tests: content: name: test-secret - # spec.template.spec.serviceAccountName (with bootstrapMetastoreManager enabled) + # spec.template.spec.serviceAccountName (with bootstrap.enabled true) - it: should set default service account name set: - bootstrapMetastoreManager: true + bootstrap.enabled: true asserts: - equal: path: spec.template.spec.serviceAccountName value: polaris-release - it: should set service account name when serviceAccount.create is true set: - bootstrapMetastoreManager: true + bootstrap.enabled: true serviceAccount: create: true asserts: @@ -223,7 +189,7 @@ tests: value: polaris-release - it: should set custom service account name when serviceAccount.create is true set: - bootstrapMetastoreManager: true + bootstrap.enabled: true serviceAccount: create: true name: polaris-sa @@ -233,7 +199,7 @@ tests: value: polaris-sa - it: should set service account name to default when serviceAccount.create is false set: - bootstrapMetastoreManager: true + bootstrap.enabled: true serviceAccount: create: false asserts: @@ -242,7 +208,7 @@ tests: value: default - it: should set custom service account name when serviceAccount.create is false set: - bootstrapMetastoreManager: true + bootstrap.enabled: true serviceAccount: create: false name: polaris-sa @@ -251,16 +217,16 @@ tests: path: spec.template.spec.serviceAccountName value: polaris-sa - # spec.template.spec.securityContext (with bootstrapMetastoreManager enabled) + # spec.template.spec.securityContext (with bootstrap.enabled true) - it: should not set securityContext by default set: - bootstrapMetastoreManager: true + bootstrap.enabled: true asserts: - notExists: path: spec.template.spec.securityContext - it: should set securityContext set: - bootstrapMetastoreManager: true + bootstrap.enabled: true podSecurityContext: runAsUser: 1000 asserts: @@ -269,26 +235,26 @@ tests: content: runAsUser: 1000 - # spec.template.spec.containers (with bootstrapMetastoreManager enabled) + # spec.template.spec.containers (with bootstrap.enabled true) - it: should set container name set: - bootstrapMetastoreManager: true + bootstrap.enabled: true asserts: - equal: path: spec.template.spec.containers[0].name value: polaris - # spec.template.spec.containers[0].securityContext (with bootstrapMetastoreManager enabled) + # spec.template.spec.containers[0].securityContext (with bootstrap.enabled true) - it: should not set container securityContext by default set: - bootstrapMetastoreManager: true + bootstrap.enabled: true asserts: - notExists: path: spec.template.spec.containers[0].securityContext - it: should set container securityContext set: - bootstrapMetastoreManager: true - securityContext: + bootstrap.enabled: true + containerSecurityContext: runAsUser: 1000 asserts: - isSubset: @@ -296,147 +262,63 @@ tests: content: runAsUser: 1000 - # spec.template.spec.containers[0].securityContext (with bootstrapMetastoreManager enabled) - - it: should not set initContainer securityContext by default - set: - bootstrapMetastoreManager: true - persistenceConfigSecret: polaris-persistence-secret - polarisServerConfig: - metaStoreManager: - conf-file: /eclipselink-config/conf.jar!/persistence.xml - asserts: - - notExists: - path: spec.template.spec.initContainers[0].securityContext - - it: should set initContainer securityContext - set: - bootstrapMetastoreManager: true - persistenceConfigSecret: polaris-persistence-secret - polarisServerConfig: - metaStoreManager: - conf-file: /eclipselink-config/conf.jar!/persistence.xml - securityContext: - runAsUser: 1000 - asserts: - - isSubset: - path: spec.template.spec.initContainers[0].securityContext - content: - runAsUser: 1000 - - # spec.template.spec.containers[0].image (with bootstrapMetastoreManager enabled) + # spec.template.spec.containers[0].image (with bootstrap.enabled true) - it: should set container image set: - bootstrapMetastoreManager: true - image: - repository: test-repo - tag: test-tag + bootstrap: + enabled: true + image: + repository: test-repo + tag: test-tag asserts: - equal: path: spec.template.spec.containers[0].image value: test-repo:test-tag - it: should set container image with template set: - bootstrapMetastoreManager: true - image: - repository: test-repo-{{ .Chart.Version }} - tag: test-tag-{{ .Release.Name }} + bootstrap: + enabled: true + image: + repository: test-repo-{{ .Chart.Version }} + tag: test-tag-{{ .Release.Name }} asserts: - equal: path: spec.template.spec.containers[0].image value: test-repo-1.2.3:test-tag-polaris-release - it: should set container image with chart version if no tag provided set: - bootstrapMetastoreManager: true - image: - repository: test-repo - tag: "" + bootstrap: + enabled: true + image: + repository: test-repo + tag: "" asserts: - equal: path: spec.template.spec.containers[0].image value: test-repo:1.2.3 - # spec.template.spec.containers[0].imagePullPolicy (with bootstrapMetastoreManager enabled) + # spec.template.spec.containers[0].imagePullPolicy (with bootstrap.enabled true) - it: should set container pull policy set: - bootstrapMetastoreManager: true - image: - pullPolicy: Always + bootstrap: + enabled: true + image: + pullPolicy: Always asserts: - equal: path: spec.template.spec.containers[0].imagePullPolicy value: Always - # spec.template.spec.containers[0].volumeMounts + spec.template.spec.volumes (with bootstrapMetastoreManager enabled) - - it: should not set persistence volumes by default - set: - bootstrapMetastoreManager: true - asserts: - - lengthEqual: - path: spec.template.spec.volumes - count: 1 - - lengthEqual: - path: spec.template.spec.containers[0].volumeMounts - count: 1 - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: config-volume - mountPath: /app/config/polaris-server.yml - subPath: polaris-server.yml - - it: should set persistence volumes if persistence is enabled - set: - bootstrapMetastoreManager: true - persistenceConfigSecret: polaris-persistence-secret - polarisServerConfig: - metaStoreManager: - conf-file: /eclipselink-config/conf.jar!/persistence.xml - asserts: - - lengthEqual: - path: spec.template.spec.volumes - count: 3 - - lengthEqual: - path: spec.template.spec.containers[0].volumeMounts - count: 2 - - contains: - path: spec.template.spec.volumes - content: - name: eclipselink-config-volume - emptyDir: { } - - contains: - path: spec.template.spec.volumes - content: - name: secret-volume - secret: - secretName: polaris-persistence-secret - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: eclipselink-config-volume - mountPath: /eclipselink-config - - it: should evaluate template expressions in persistence secret name - set: - bootstrapMetastoreManager: true - persistenceConfigSecret: '{{ .Release.Name }}-persistence-secret' - polarisServerConfig: - metaStoreManager: - conf-file: /eclipselink-config/conf.jar!/persistence.xml - asserts: - - contains: - path: spec.template.spec.volumes - content: - name: secret-volume - secret: - secretName: polaris-release-persistence-secret - - # spec.template.spec.containers[0].resources (with bootstrapMetastoreManager enabled) + # spec.template.spec.containers[0].resources (with bootstrap.enabled true) - it: should not set container resources by default set: - bootstrapMetastoreManager: true + bootstrap.enabled: true asserts: - notExists: path: spec.template.spec.containers[0].resources - it: should set container resources set: - bootstrapMetastoreManager: true + bootstrap.enabled: true resources: requests: cpu: 100m @@ -455,19 +337,14 @@ tests: cpu: 200m memory: 256Mi - # spec.template.spec.containers[0].env (with bootstrapMetastoreManager enabled) - - it: should not set container env by default - set: - bootstrapMetastoreManager: true - asserts: - - notExists: - path: spec.template.spec.containers[0].env + # spec.template.spec.containers[0].env (with bootstrap.enabled true) - it: should set container env set: - bootstrapMetastoreManager: true - bootstrapExtraEnv: - - name: foo - value: bar + bootstrap: + enabled: true + extraEnv: + - name: foo + value: bar asserts: - contains: path: spec.template.spec.containers[0].env @@ -475,16 +352,16 @@ tests: name: foo value: bar - # spec.template.spec.nodeSelector (with bootstrapMetastoreManager enabled) + # spec.template.spec.nodeSelector (with bootstrap.enabled true) - it: should not set nodeSelector by default set: - bootstrapMetastoreManager: true + bootstrap.enabled: true asserts: - notExists: path: spec.template.spec.nodeSelector - it: should set nodeSelector set: - bootstrapMetastoreManager: true + bootstrap.enabled: true nodeSelector: disktype: ssd asserts: @@ -493,16 +370,16 @@ tests: value: disktype: ssd - # spec.template.spec.affinity (with bootstrapMetastoreManager enabled) + # spec.template.spec.affinity (with bootstrap.enabled true) - it: should not set affinity by default set: - bootstrapMetastoreManager: true + bootstrap.enabled: true asserts: - notExists: path: spec.template.spec.affinity - it: should set affinity set: - bootstrapMetastoreManager: true + bootstrap.enabled: true affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -527,16 +404,16 @@ tests: - zone1 - zone2 - # spec.template.spec.tolerations (with bootstrapMetastoreManager enabled) + # spec.template.spec.tolerations (with bootstrap.enabled true) - it: should not set tolerations by default set: - bootstrapMetastoreManager: true + bootstrap.enabled: true asserts: - notExists: path: spec.template.spec.tolerations - it: should set tolerations set: - bootstrapMetastoreManager: true + bootstrap.enabled: true tolerations: - key: "key" operator: "Equal" diff --git a/helm/polaris/tests/quantity_test.yaml b/helm/polaris/tests/quantity_test.yaml new file mode 100644 index 000000000..497eb8ae1 --- /dev/null +++ b/helm/polaris/tests/quantity_test.yaml @@ -0,0 +1,98 @@ +## +## Copyright (C) 2024 Dremio +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + +templates: + - configmap.yaml + +tests: + - it: should convert quantity 12345 + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"12345\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=12345" } } ] + - it: should convert quantity 12345.3456 + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"12345.3456\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=12345.3456" } } ] + - it: should convert quantity 123451234512345 + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"123451234512345\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=123451234512345" } } ] + - it: should convert quantity 1m + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"1m\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=0.001" } } ] + - it: should convert quantity 100m + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"100m\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=0.1" } } ] + - it: should convert quantity 1000m + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"1000m\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=1" } } ] + - it: should convert quantity 10000m + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"10000m\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=10" } } ] + - it: should convert quantity 1k + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"1k\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=1000" } } ] + - it: should convert quantity 1ki + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"1ki\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=1024" } } ] + - it: should convert quantity 100k + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"100k\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=100000" } } ] + - it: should convert quantity 100ki + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"100ki\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=102400" } } ] + - it: should convert quantity 1M + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"1M\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=1000000" } } ] + - it: should convert quantity 1Mi + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"1Mi\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=1048576" } } ] + - it: should convert quantity 1G + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"1G\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=1000000000" } } ] + - it: should convert quantity 1Gi + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"1Gi\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=1073741824" } } ] + - it: should convert quantity 1T + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"1T\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=1000000000000" } } ] + - it: should convert quantity 1Ti + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"1Ti\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=1099511627776" } } ] + - it: should convert quantity 1P + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"1P\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=1000000000000000" } } ] + - it: should convert quantity 1Pi + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"1Pi\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=1125899906842624" } } ] + - it: should convert quantity 1E + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"1E\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=1000000000000000000" } } ] + - it: should convert quantity 1Ei + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"1Ei\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=1152921504606846976" } } ] + - it: should fail on invalid quantity + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"invalid\" }}" }} + asserts: [ { failedTemplate: { errorMessage: "invalid quantity: invalid" } } ] + - it: should fail on zero quantity + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"0\" }}" }} + asserts: [ { failedTemplate: { errorMessage: "invalid quantity: 0" } } ] + - it: should fail on negative quantity + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"-1\" }}" }} + asserts: [ { failedTemplate: { errorMessage: "invalid quantity: -1" } } ] + - it: should convert quantity in scientific notation 123e2 + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"123e2\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=12300" } } ] + - it: should convert quantity in scientific notation 1.23e2 + set: { advancedConfig: { test: "{{ include \"polaris.quantity\" \"1.23e2\" }}" }} + asserts: [ { matchRegex: { path: "data[\"application.properties\"]", pattern: "test=123" } } ] diff --git a/helm/polaris/tests/service_monitor_test.yaml b/helm/polaris/tests/service_monitor_test.yaml new file mode 100644 index 000000000..b160540cf --- /dev/null +++ b/helm/polaris/tests/service_monitor_test.yaml @@ -0,0 +1,55 @@ +## +## Copyright (C) 2024 Dremio +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + +release: + name: polaris + namespace: polaris-ns + +capabilities: + apiVersions: + - monitoring.coreos.com/v1 + +templates: + - servicemonitor.yaml + +tests: + - it: should create ServiceMonitor if enabled + set: { serviceMonitor: { enabled: true } } + asserts: [ { containsDocument: { kind: ServiceMonitor, apiVersion: monitoring.coreos.com/v1, name: polaris, namespace: polaris-ns } } ] + - it: should not create ServiceMonitor if disabled + set: { serviceMonitor: { enabled: false } } + asserts: [ { containsDocument: { kind: ServiceMonitor, apiVersion: monitoring.coreos.com/v1, name: polaris, namespace: polaris-ns }, not: true } ] + - it: should add metrics relabelings + set: + serviceMonitor: + enabled: true + metricRelabelings: + - source_labels: [ __meta_kubernetes_namespace ] + separator: ; + regex: (.*) + target_label: namespace + replacement: $1 + action: replace + asserts: + - equal: + path: spec.endpoints[ 0].metricRelabelings + value: + - source_labels: [ __meta_kubernetes_namespace ] + separator: ; + regex: (.*) + target_label: namespace + replacement: $1 + action: replace \ No newline at end of file diff --git a/helm/polaris/tests/service_test.yaml b/helm/polaris/tests/service_test.yaml index b7e3057ab..e4fefc62b 100644 --- a/helm/polaris/tests/service_test.yaml +++ b/helm/polaris/tests/service_test.yaml @@ -114,32 +114,24 @@ tests: - equal: path: spec.ports value: - - port: 8182 - targetPort: 8182 - protocol: TCP - name: polaris-metrics - port: 8181 - targetPort: 8181 protocol: TCP - name: polaris-service + name: polaris-http - it: should set service ports set: service: ports: - polaris-service: 18181 - polaris-metrics: 18182 + - port: 18181 + targetPort: 18181 + name: polaris-http asserts: - equal: path: spec.ports value: - - port: 18182 - targetPort: 18182 - protocol: TCP - name: polaris-metrics - port: 18181 targetPort: 18181 protocol: TCP - name: polaris-service + name: polaris-http # spec.sessionAffinity - it: should set service default session affinity diff --git a/helm/polaris/values.yaml b/helm/polaris/values.yaml index 72e849607..5746623f5 100644 --- a/helm/polaris/values.yaml +++ b/helm/polaris/values.yaml @@ -23,19 +23,14 @@ replicaCount: 1 image: # -- The image repository to pull from. - repository: localhost:5001/polaris - # -- The image pull policy. - pullPolicy: IfNotPresent - # -- The image tag. - tag: "latest" - -toolsImage: - # -- The image repository to pull from (must have jar binary included). - repository: registry.access.redhat.com/ubi9/openjdk-21 + repository: apache/polaris # -- The image pull policy. pullPolicy: IfNotPresent # -- The image tag. tag: "latest" + # -- The path to the directory where the application.properties file, and other configuration + # files, if any, should be mounted. + configDir: /deployments/config # -- References to secrets in the same namespace to use for pulling any of the images used by this # chart. Each entry is a LocalObjectReference to an existing secret in the namespace. The secret @@ -74,7 +69,7 @@ podSecurityContext: # fsGroup: 2000 # -- Security context for the polaris container. See https://kubernetes.io/docs/tasks/configure-pod-container/security-context/. -securityContext: +containerSecurityContext: {} # capabilities: # drop: @@ -83,18 +78,22 @@ securityContext: # runAsNonRoot: true # runAsUser: 1000 -# Polaris service settings. +# -- Polaris main service settings. service: # -- The type of service to create. type: ClusterIP - # -- The ports the service will listen on. Two ports are required: one for the Polaris service and - # one for the metrics API. + # -- The ports the service will listen on. + # At least one port is required; the first port implicitly becomes the HTTP port that the + # application will use for serving API requests. By default, it's 8181. # Note: port names must be unique and no more than 15 characters long. ports: - # polaris-server: The port the Polaris server listens on for API requests. - polaris-service: 8181 - # polaris-metrics: The port the Polaris server listens on for metrics API requests (health checks, metrics, etc.). - polaris-metrics: 8182 + - name: polaris-http + port: 8181 + # targetPort: 8181 + # nodePort: 30000 + # protocol: TCP + # - name: polaris-https + # port: 18181 # -- The session affinity for the service. Valid values are: None, ClientIP. # ClientIP enables sticky sessions based on the client's IP address. # This is generally beneficial to Polaris deployments, but some testing may be @@ -102,9 +101,73 @@ service: # Also, this setting affects only internal clients, not external ones. # If Ingress is enabled, it is recommended to set sessionAffinity to None. sessionAffinity: None + # -- You can specify your own cluster IP address + # If you define a Service that has the .spec.clusterIP set to "None" then Kubernetes does not assign an IP address. + # Instead, DNS records for the service will return the IP addresses of each pod targeted by the server. This is + # called a headless service. + # See https://kubernetes.io/docs/concepts/services-networking/service/#headless-services + clusterIP: "" + # -- The traffic policy fields control how traffic from internal and external sources are routed respectively. + # Valid values are Cluster and Local. + # Set the field to Cluster to route traffic to all ready endpoints. + # Set the field to Local to only route to ready node-local endpoints. + # If the traffic policy is Local and there are no node-local endpoints, traffic is dropped by kube-proxy + internalTrafficPolicy: Cluster + externalTrafficPolicy: Cluster + # -- The traffic distribution field provides another way to influence traffic routing within a Kubernetes Service. + # While traffic policies focus on strict semantic guarantees, traffic distribution allows you to express preferences + # such as routing to topologically closer endpoints. + # Valid values are: PreferClose + trafficDistribution: PreferClose + # -- Annotations to add to the service. + annotations: {} + +# -- Management service settings. These settings are used to configure liveness and readiness probes, +# and to configure the dedicated headless service that will expose health checks and metrics, e.g. +# for metrics scraping and service monitoring. +managementService: + # -- The type of service to create. + type: ClusterIP + # -- The ports the management service will listen on. + # At least one port is required; the first port implicitly becomes the HTTP port that the + # application will use for serving management requests. By default, it's 8182. + # Note: port names must be unique and no more than 15 characters long. + ports: + # -- The name of the management port. Required. + - name: polaris-mgmt + # -- The port the management service listens on. By default, the management interface is exposed + # on HTTP port 8182. + port: 8182 + # targetPort: polaris-service + # nodePort: 30000 + clusterIP: None + sessionAffinity: ~ + internalTrafficPolicy: ~ + externalTrafficPolicy: ~ + trafficDistribution: ~ # -- Annotations to add to the service. annotations: {} +# -- Additional service definitions. All service definitions always select all Polaris pods. Use +# this if you need to expose specific ports with different configurations, e.g. expose polaris-http +# with an alternate LoadBalancer service instead of ClusterIP. +extraServices: [] + # - # -- The suffix to append to the service name. Required. It must be unique. If it does not + # # start with a hyphen, a hyphen will be inserted between the base service name and the suffix. + # nameSuffix: "ext" + # # -- The type of service to create. + # type: LoadBalancer + # # -- The ports the service will listen on. + # ports: + # - name: polaris-http + # number: 8181 + # sessionAffinity: None + # clusterIP: "" + # internalTrafficPolicy: Cluster + # externalTrafficPolicy: Cluster + # trafficDistribution: PreferClose + # annotations: {} + # Polaris Ingress settings. # These settings generate an Ingress resource that routes external traffic to the Polaris service # using the "polaris-service" port described above. @@ -213,6 +276,36 @@ readinessProbe: # -- Number of seconds after which the probe times out. Minimum value is 1. timeoutSeconds: 10 +# -- Advanced configuration. +# You can pass here any valid Polaris or Quarkus configuration property. +# Any property that is defined here takes precedence over all the other configuration values generated by this chart. +# Properties can be passed "flattened" or as nested YAML objects (see examples below). +# Note: values should be strings; avoid using numbers, booleans, or other types. +advancedConfig: +# # Disable access log: +# quarkus.http.access-log.enabled: "true" +# # Reverse Proxy Settings +# # ---------------------- +# # +# # These config options are mentioned only for documentation purposes. Consult the +# # Quarkus documentation for "Running behind a reverse proxy" and configure those +# # depending on your actual needs. +# +# # See https://quarkus.io/guides/http-reference#reverse-proxy +# +# # Do NOT enable these option unless your reverse proxy (for example istio or nginx) +# # is properly setup to set these headers but also filter those from incoming requests. +# +# quarkus: +# http: +# proxy: +# proxy-address-forwarding: "true" +# allow-x-forwarded: "true" +# enable-forwarded-host: "true" +# enable-forwarded-prefix: "true" +# trusted-proxies: "127.0.0.1" + {} + # -- Advanced configuration via Environment Variables. # Extra environment variables to add to the Polaris server container. # You can pass here any valid EnvVar object: @@ -233,170 +326,240 @@ extraEnv: # name: aws-secret # key: secret_access_key -# -- Configures whether to enable the bootstrap metastore manager job -bootstrapMetastoreManager: false - -# -- Extra environment variables to add to the bootstrap metastore manager job (see `extraEnv` for an example) -bootstrapExtraEnv: [] - -# -- The secret name to pull persistence.xml from (ensure the key name is 'persistence.xml') -persistenceConfigSecret: ~ - -# -- Configures for polaris-server.yml -polarisServerConfig: - server: - # Maximum number of threads. - maxThreads: 200 +# -- Extra volumes to add to the polaris pod. See https://kubernetes.io/docs/concepts/storage/volumes/. +extraVolumes: [] + # - name: extra-volume + # emptyDir: {} - # Minimum number of thread to keep alive. - minThreads: 10 - applicationConnectors: - # HTTP-specific options. - - type: http +# -- Extra volume mounts to add to the polaris container. See https://kubernetes.io/docs/concepts/storage/volumes/. +extraVolumeMounts: [] + # - name: extra-volume + # mountPath: /usr/share/extra-volume - # The port on which the HTTP server listens for service requests. - port: 8181 +# -- Add additional init containers to the polaris pod(s) See https://kubernetes.io/docs/concepts/workloads/pods/init-containers/. +extraInitContainers: [] + # - name: your-image-name + # image: your-image + # imagePullPolicy: Always + # command: ['sh', '-c', 'echo "hello world"'] - adminConnectors: - - type: http - port: 8182 - - # The hostname of the interface to which the HTTP server socket wil be found. If omitted, the - # socket will listen on all interfaces. - # bindHost: localhost - - # ssl: - # keyStore: ./example.keystore - # keyStorePassword: example - # - # keyStoreType: JKS # (optional, JKS is default) - - # HTTP request log settings - requestLog: - appenders: - # Settings for logging to stdout. - - type: console - - # # Settings for logging to a file. - # - type: file - - # # The file to which statements will be logged. - # currentLogFilename: ./logs/request.log - - # # When the log file rolls over, the file will be archived to requests-2012-03-15.log.gz, - # # requests.log will be truncated, and new statements written to it. - # archivedLogFilenamePattern: ./logs/requests-%d.log.gz - - # # The maximum number of log files to archive. - # archivedFileCount: 14 - - # # Enable archiving if the request log entries go to the their own file - # archive: true +# -- Configures whether to enable the bootstrap metastore manager job +bootstrap: + enabled: false + # -- The names of the realms to bootstrap. + realms: [] + # -- The credentials to create during the bootstrap. If you don't provide credentials for the + # root principal of each realm to bootstrap, random credentials will be generated. + # Each entry in the array must be of the form: realm,userName,clientId,clientSecret + credentials: [] + # - realm1,root,root,s3cr3t + # - realm2,root,root,s3cr3t + # -- The image configuration for the bootstrap metastore manager job. + image: + # -- The image repository to pull from. + repository: apache/polaris-admin-tool + # -- The image pull policy. + pullPolicy: IfNotPresent + # -- The image tag. + tag: "latest" + # -- The path to the directory where the application.properties file, and other configuration + # files, if any, should be mounted. + configDir: /deployments/config + # -- Extra environment variables to add to the bootstrap metastore manager job (see `extraEnv` for an example) + extraEnv: [] + +tracing: + # -- Specifies whether tracing for the polaris server should be enabled. + enabled: false + # -- The collector endpoint URL to connect to (required). + # The endpoint URL must have either the http:// or the https:// scheme. + # The collector must talk the OpenTelemetry protocol (OTLP) and the port must be its gRPC port (by default 4317). + # See https://quarkus.io/guides/opentelemetry for more information. + endpoint: "http://otlp-collector:4317" + # -- Which requests should be sampled. Valid values are: "all", "none", or a ratio between 0.0 and + # "1.0d" (inclusive). E.g. "0.5d" means that 50% of the requests will be sampled. + sample: "1.0d" + # -- Resource attributes to identify the polaris service among other tracing sources. + # See https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/#service. + # If left empty, traces will be attached to a service named "Polaris"; to change this, provide a service.name attribute here. + attributes: + {} + # service.name: my-polaris + +metrics: + # -- Specifies whether metrics for the polaris server should be enabled. + enabled: true + # -- Additional tags (dimensional labels) to add to the metrics. + tags: + {} + # service: polaris + # environment: production + +serviceMonitor: + # -- Specifies whether a ServiceMonitor for Prometheus operator should be created. + enabled: true + # -- The scrape interval; leave empty to let Prometheus decide. Must be a valid duration, e.g. 1d, 1h30m, 5m, 10s. + interval: "" + # -- Labels for the created ServiceMonitor so that Prometheus operator can properly pick it up. + labels: + {} + # release: prometheus + # -- Relabeling rules to apply to metrics. Ref https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config. + metricRelabelings: [] + # - source_labels: [ __meta_kubernetes_namespace ] + # separator: ; + # regex: (.*) + # target_label: namespace + # replacement: $1 + # action: replace + +# -- Logging configuration. +logging: + # -- The log level of the root category, which is used as the default log level for all categories. + level: INFO + # -- The header name to use for the request ID. + requestIdHeaderName: request_id + # -- Configuration for the console appender. + console: + # -- Whether to enable the console appender. + enabled: true + # -- The log level of the console appender. + threshold: ALL + # -- Whether to log in JSON format. + json: false + # -- The log format to use. Ignored if JSON format is enabled. See + # https://quarkus.io/guides/logging#logging-format for details. + format: "%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] [%X{requestId},%X{realmId}] [%X{traceId},%X{parentId},%X{spanId},%X{sampled}] (%t) %s%e%n" + # -- Configuration for the file appender. + file: + # -- Whether to enable the file appender. + enabled: false + # -- The log level of the file appender. + threshold: ALL + # -- Whether to log in JSON format. + json: false + # -- The log format to use. Ignored if JSON format is enabled. See + # https://quarkus.io/guides/logging#logging-format for details. + format: "%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] [%X{requestId},%X{realmId}] [%X{traceId},%X{parentId},%X{spanId},%X{sampled}] (%t) %s%e%n" + # -- The local directory where log files are stored. The persistent volume claim will be mounted + # here. + logsDir: /deployments/logs + # -- The log file name. + fileName: polaris.log + # -- Log rotation configuration. + rotation: + # -- The maximum size of the log file before it is rotated. Should be expressed as a Kubernetes quantity. + maxFileSize: 100Mi + # -- The maximum number of backup files to keep. + maxBackupIndex: 5 + # -- An optional suffix to append to the rotated log files. If present, the rotated log files + # will be grouped in time buckets, and each bucket will contain at most maxBackupIndex files. + # The suffix must be in a date-time format that is understood by DateTimeFormatter. If the + # suffix ends with .gz or .zip, the rotated files will also be compressed using the + # corresponding algorithm. + fileSuffix: ~ # .yyyy-MM-dd.gz + # -- The log storage configuration. A persistent volume claim will be created using these + # settings. + storage: + # -- The storage class name of the persistent volume claim to create. + className: standard + # -- The size of the persistent volume claim to create. + size: 512Gi + # -- Labels to add to the persistent volume claim spec selector; a persistent volume with + # matching labels must exist. Leave empty if using dynamic provisioning. + selectorLabels: {} + # app.kubernetes.io/name: polaris + # app.kubernetes.io/instance: RELEASE-NAME + # -- Configuration for specific log categories. + categories: + org.apache.polaris: INFO + org.apache.iceberg.rest: INFO + # Useful to debug configuration issues: + # io.smallrye.config: DEBUG + # -- Configuration for MDC (Mapped Diagnostic Context). Values specified here will be added to the + # log context of all incoming requests and can be used in log patterns. + mdc: + # aid=polaris + # sid=polaris-service + {} + +realmContext: + type: default + realms: + - default-realm - featureConfiguration: +features: + defaults: ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING: false SUPPORTED_CATALOG_STORAGE_TYPES: - S3 - GCS - AZURE - FILE - - callContextResolver: + realmOverrides: {} + # my-realm: + # ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING: true + +persistence: + type: eclipse-link # in-memory + eclipseLink: + # -- The secret name to pull persistence.xml from. + # If not provided, the default persistence.xml will be used. + secret: + name: ~ + key: persistence.xml + # -- The persistence unit name to use. Only required if persistenceConfigSecret is set. + persistenceUnit: polaris + +fileIo: + type: default + +# -- Storage credentials for the server. If the following properties are unset, default credentials will be used. +storage: + aws: + accessKey: ~ + secretKey: ~ + gcp: + token: ~ + lifespan: PT1H + +authentication: + authenticator: type: default - - realmContextResolver: + tokenService: type: default + tokenBroker: + type: rsa-key-pair # symmetric-key + maxTokenGeneration: PT1H + secret: + name: ~ + publicKey: public.pem + privateKey: private.pem + secretKey: secret + +cors: + allowedOrigins: + - http://localhost:8080 + allowedMethods: + - PATCH + - POST + - DELETE + - GET + - PUT + allowedHeaders: + - "*" + exposedHeaders: + - "*" + accessControlMaxAge: 600 + accessControlAllowCredentials: true + +rateLimiter: + type: no-op + tokenBucket: + type: default + requestsPerSecond: 1000 + window: PT10S - defaultRealms: - - default-realm - - metaStoreManager: - type: in-memory - # uncomment below to use eclipse-link as metastore - # type: eclipse-link - # persistence-unit: polaris - # conf-file: /eclipselink-config/conf.jar!/persistence.xml - - io: - factoryType: default - - # TODO - avoid duplicating token broker config - oauth2: - type: test - # type: default # - uncomment to support Auth0 JWT tokens -# tokenBroker: -# type: symmetric-key -# secret: polaris - - authenticator: - class: org.apache.polaris.service.auth.TestInlineBearerTokenPolarisAuthenticator - # class: org.apache.polaris.service.auth.DefaultPolarisAuthenticator # - uncomment to support Auth0 JWT tokens - # tokenBroker: - # type: symmetric-key - # secret: polaris - - cors: - allowed-origins: - - http://localhost:8080 - allowed-timing-origins: - - http://localhost:8080 - allowed-methods: - - PATCH - - POST - - DELETE - - GET - - PUT - allowed-headers: - - "*" - exposed-headers: - - "*" - preflight-max-age: 600 - allowed-credentials: true - - # Logging settings. - - logging: - - # The default level of all loggers. Can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL. - level: INFO - - # Logger-specific levels. - loggers: - org.apache.iceberg.rest: DEBUG - org.apache.polaris: DEBUG - - appenders: - - - type: console - # If true, write log statements to stdout. - # enabled: true - # Do not display log statements below this threshold to stdout. - threshold: ALL - # Custom Logback PatternLayout with threadname. - logFormat: "%-5p [%d{ISO8601} - %-6r] [%t] [%X{aid}%X{sid}%X{tid}%X{wid}%X{oid}%X{srv}%X{job}%X{rid}] %c{30}: %m %kvp%n%ex" - - # # Settings for logging to a file. - # - type: file - # # If true, write log statements to a file. - # # enabled: true - # # Do not write log statements below this threshold to the file. - # threshold: ALL - # layout: - # type: polaris - # flattenKeyValues: false - # includeKeyValues: true - - # # The file to which statements will be logged. - # currentLogFilename: ./logs/polaris.log - # # When the log file rolls over, the file will be archived to polaris-2012-03-15.log.gz, - # # polaris.log will be truncated, and new statements written to it. - # archivedLogFilenamePattern: ./logs/polaris-%d.log.gz - # # The maximum number of log files to archive. - # archivedFileCount: 14 - - # Limits the size of request bodies sent to Polaris. -1 means no limit. - maxRequestBodyBytes: -1 - - # Optional, not specifying a "rateLimiter" section also means no rate limiter - rateLimiter: - type: no-op +tasks: + maxConcurrentTasks: 100 + maxQueuedTasks: 1000 diff --git a/k8/deployment.yaml b/k8/deployment.yaml index 2dc098244..4d30fda57 100644 --- a/k8/deployment.yaml +++ b/k8/deployment.yaml @@ -39,7 +39,7 @@ spec: spec: containers: - name: polaris - image: localhost:5001/polaris:latest + image: localhost:5001/apache/polaris:latest ports: - containerPort: 8181 - containerPort: 8182 diff --git a/quarkus/admin/build.gradle.kts b/quarkus/admin/build.gradle.kts index 145340c7c..7e934d8fc 100644 --- a/quarkus/admin/build.gradle.kts +++ b/quarkus/admin/build.gradle.kts @@ -35,6 +35,7 @@ dependencies { implementation(project(":polaris-api-iceberg-service")) implementation(project(":polaris-service-common")) implementation(project(":polaris-quarkus-service")) + runtimeOnly(project(":polaris-eclipselink")) implementation(enforcedPlatform(libs.quarkus.bom)) implementation("io.quarkus:quarkus-picocli") diff --git a/quarkus/server/README.md b/quarkus/server/README.md index bac32962b..aade36e83 100644 --- a/quarkus/server/README.md +++ b/quarkus/server/README.md @@ -2,8 +2,27 @@ This module contains the Quarkus-based Polaris server main artifact. +## Archive distribution + Building this module will create a zip/tar distribution with the Polaris server. +To build the distribution, you can use the following command: + +```shell +./gradlew :polaris-quarkus-server:build +``` + +You can manually unpack and run the distribution archives: + +```shell +cd quarkus/server/build/distributions +unzip polaris-quarkus-server-.zip +cd polaris-quarkus-server- +java -jar quarkus-run.jar +``` + +## Docker image + To also build the Docker image, you can use the following command (a running Docker daemon is required): diff --git a/quarkus/server/build.gradle.kts b/quarkus/server/build.gradle.kts index 83e94435d..bd2d554bb 100644 --- a/quarkus/server/build.gradle.kts +++ b/quarkus/server/build.gradle.kts @@ -61,6 +61,14 @@ tasks.named("distZip") { dependsOn("quarkusBuild") } tasks.named("distTar") { dependsOn("quarkusBuild") } +tasks.named("javadoc") { enabled = false } + +tasks.named("sourcesJar") { enabled = false } + +tasks.register("polarisServerRun") { dependsOn("quarkusRun") } + +tasks.register("polarisServerDev") { dependsOn("quarkusDev") } + distributions { main { contents { diff --git a/quarkus/server/src/main/java/org/apache/polaris/server/quarkus/package-info.java b/quarkus/server/src/main/java/org/apache/polaris/server/quarkus/package-info.java new file mode 100644 index 000000000..fc12d8cac --- /dev/null +++ b/quarkus/server/src/main/java/org/apache/polaris/server/quarkus/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.server.quarkus; + +// This file is here as a placeholder to allow quarkusDev and quarkusRun +// tasks to work, since this module currently has no source files. diff --git a/quarkus/service/README-quarkus.md b/quarkus/service/README-quarkus.md deleted file mode 100644 index 1add64333..000000000 --- a/quarkus/service/README-quarkus.md +++ /dev/null @@ -1,97 +0,0 @@ - - -This module contains the Polaris Service powered by Quarkus (instead of Dropwizard). - -# Main differences - -* Bean injection (CDI) is made using `@ApplicationScoped` annotation on class and injected in other classes using `@Inject` annotation (https://quarkus.io/guides/cdi-reference). -* Codehale metrics registry and opentelemetry boilerplate (prometheus exporter included) are not needed anymore: Quarkus provides it "out of the box" (https://quarkus.io/guides/opentelemetry) -* `PolarisHealthCheck` is not needed anymore: Quarkus provides it "out of the box" (https://quarkus.io/guides/smallrye-health) -* `TimedApplicationEventListener` and the `@TimedApi` annotation are replaced by Quarkus (micrometer) `@Timed` annotation (https://quarkus.io/guides/telemetry-micrometer) -* `PolarisJsonLayoutFactory` is not needed anymore: Quarkus provides it by configuration (using `quarkus.log.*` configuration) -* `PolarisApplication` is not needed, Quarkus provide a "main" application out of the box (it's possible to provide `QuarkusApplication` for control the startup and also using `@Startup` annotation) -* CORS boilerplate is not needed anymore: Quarkus supports it via configuration (using `quarkus.http.cors.*` configuration) -* CLI is not part of `polaris-service` anymore, we have (will have) a dedicated module (`polaris-cli`) - -# Build and run - -To build `polaris-service` you simply do: - -``` -./gradlew :polaris-service:build -``` - -The build creates ready to run package: -* in the `build/quarkus-app` folder, you can run with `java -jar quarkus-run.jar` -* the `build/distributions` folder contains tar/zip distributions you can extract - -You can directly run Polaris service (in the build scope) using: - -``` -./gradlew :polaris-quarkus-service:quarkusRun -``` - -You can run in Dev mode as well: - -``` -./gradlew --console=plain :polaris-quarkus-service:quarkusDev -``` - -You can directly build a Docker image using: - -``` -./gradlew :polaris-quarkus-service:imageBuild -``` - -# Configuration - -The main configuration file is not the `application.properties`. The default configuration is -packaged as part of the `polaris-quarkus-service`. `polaris-quarkus-service` uses several -configuration sources (in this order): -* system properties -* environment variables -* `.env` file in the current working directory -* `$PWD/config/application.properties` file -* the `application.properties` packaged in the `polaris-quarkus-service` application - -It means you can override some configuration property using environment variables for example. - -By default, `polaris-quarkus-service` uses 8181 as the HTTP port (defined in the `quarkus.http.port` -configuration property) and 8182 as the management port (defined in the `quarkus.management.port` -configuration property). - -You can find more details here: https://quarkus.io/guides/config - -# TODO - -* Modify `CallContext` and remove all usages of `ThreadLocal`, replace with proper context propagation. -* Remove `PolarisCallContext` – it's just an aggregation of CDI beans -* Use `@QuarkustIntegrationTest` for integration tests -* Remove `OAuthCredentialAuthFilter` and replace with Quarkus OIDC security -* Create `polaris-cli` module, add Bootstrap and Purge commands -* Adapt Helm charts, Dockerfiles, K8s examples -* Fix intermittent Gradle build error : SRCFG00011: Could not expand value - platform.quarkus.native.builder-image in property quarkus.native.builder-image - (https://github.com/quarkusio/quarkus/issues/19139). - The Quarkus issue says that using the Quarkus-platform bom helps (it's really what we should depend on). But IIRC there were some dependency issues with Spark/Scala, which prevents us from using the Quarkus-platform bom. - -* Update documentation/README/... - -* Do we want to support existing json configuration file as configuration source ? diff --git a/quarkus/service/build.gradle.kts b/quarkus/service/build.gradle.kts index aea4bd375..4e742e803 100644 --- a/quarkus/service/build.gradle.kts +++ b/quarkus/service/build.gradle.kts @@ -28,6 +28,7 @@ dependencies { implementation(project(":polaris-api-management-service")) implementation(project(":polaris-api-iceberg-service")) implementation(project(":polaris-service-common")) + runtimeOnly(project(":polaris-eclipselink")) implementation(platform(libs.iceberg.bom)) implementation("org.apache.iceberg:iceberg-api") diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index 66ce09699..4e2e4982c 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -65,6 +65,7 @@ import org.apache.polaris.service.task.TaskHandlerConfiguration; import org.eclipse.microprofile.context.ManagedExecutor; import org.eclipse.microprofile.context.ThreadContext; +import org.slf4j.LoggerFactory; public class QuarkusProducers { @@ -180,6 +181,10 @@ public void maybeInitializeInMemoryRealm( MetaStoreManagerFactory factory, RealmContextConfiguration realmContextConfiguration) { if (factory instanceof InMemoryPolarisMetaStoreManagerFactory) { + LoggerFactory.getLogger(QuarkusProducers.class) + .warn( + "Polaris is configured with an in-memory metastore; " + + "this is not recommended for production use!"); ((InMemoryPolarisMetaStoreManagerFactory) factory).onStartup(realmContextConfiguration); } } diff --git a/quarkus/service/src/main/resources/application.properties b/quarkus/service/src/main/resources/application.properties index 0915eb3f7..1d9cd8ed6 100644 --- a/quarkus/service/src/main/resources/application.properties +++ b/quarkus/service/src/main/resources/application.properties @@ -77,12 +77,14 @@ polaris.realm-context.realms=realm1,realm2,realm3 polaris.realm-context.header-name=Polaris-Realm polaris.features.defaults."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=false -polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE","FILE"] +polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE"] # realm overrides # polaris.features.realm-overrides."my-realm"."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST"=true # polaris.features.realm-overrides."my-realm"."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true -polaris.persistence.type=in-memory +polaris.persistence.type=eclipse-link +polaris.persistence.eclipselink.configuration-file=META-INF/persistence.xml +polaris.persistence.eclipselink.persistence-unit=polaris polaris.file-io.type=default @@ -139,9 +141,27 @@ polaris.authentication.token-broker.max-token-generation=PT1H %test.polaris.features.defaults."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST"=true %test.polaris.features.defaults."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true %test.polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["FILE","S3","GCS","AZURE"] +%test.polaris.persistence.type=in-memory %test.polaris.realm-context.realms=POLARIS %test.polaris.realm-context.type=test %test.polaris.storage.aws.access-key=accessKey %test.polaris.storage.aws.secret-key=secretKey %test.polaris.storage.gcp.token=token %test.polaris.storage.gcp.lifespan=PT1H + +%dev.quarkus.log.file.enable=false +%dev.quarkus.log.category."org.apache.polaris".level=DEBUG +%dev.quarkus.log.category."org.apache.iceberg.rest".level=DEBUG +%dev.quarkus.otel.sdk.disabled=true +%dev.polaris.authentication.authenticator.type=test +%dev.polaris.authentication.token-service.type=test +%dev.polaris.authentication.token-broker.type=symmetric-key +%dev.polaris.authentication.token-broker.symmetric-key.secret=polaris +%dev.polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["FILE","S3","GCS","AZURE"] +%dev.polaris.persistence.type=in-memory +%dev.polaris.realm-context.realms=default-realm +%dev.polaris.realm-context.type=test +%dev.polaris.storage.aws.access-key=accessKey +%dev.polaris.storage.aws.secret-key=secretKey +%dev.polaris.storage.gcp.token=token +%dev.polaris.storage.gcp.lifespan=PT1H diff --git a/regtests/README.md b/regtests/README.md index d0844f434..7922c6163 100644 --- a/regtests/README.md +++ b/regtests/README.md @@ -21,31 +21,97 @@ # End-to-end regression tests +Regression tests are either run in Docker, using docker-compose to orchestrate the tests, or +locally. + +## Prerequisites + +It is recommended to clean the `regtests/output` directory before running tests. This can be done by +running: + +```shell +rm -rf ./regtests/output && \ + mkdir -p ./regtests/output && \ + chmod -R 777 ./regtests/output +``` + ## Run Tests With Docker Compose -Tests can be run with docker-compose by executing +Tests can be run with docker-compose using the provided `./regtests/docker-compose.yml` file. + +In this setup, a Polaris container will be started using the `apache/polaris:latest` image. If the +`apache/polaris:latest` image is not available locally, it will be pulled from the Polaris Docker +Hub repository. + +_This means that, by default, the tests will be run against the latest published version of +Polaris._ If you want to run the tests against a locally-built Polaris image, for example to +validate changes before pushing to GitHub, _you must build the image first_. This can be done by +running the following command from the root of the Polaris project: + +```shell +./gradlew clean assemble -Dquarkus.container-image.build=true +``` + +Important: if you are also using Minikube, for example to test the Helm chart, you may need to +_unset_ the Docker environment that was pointing to the Minikube Docker daemon, otherwise the image +will be built by the Minikube Docker daemon and will not be available to the local Docker daemon. +This can be done by running, _before_ building the image: + +```shell +eval $(minikube -p minikube docker-env --unset) +``` + +Finally, run the tests from the root of the Polaris project: -```bash -docker compose up --build --exit-code-from regtest +```shell +docker compose -f ./regtests/docker-compose.yml up --build --exit-code-from regtest ``` -This is the flow used in CI and should be done locally before pushing to github to ensure that no environmental -factors contribute to the outcome of the tests. +This will start the Polaris container, then build a Docker image for the Spark container that will +run the tests. The exit code will be the same as the exit code of the Spark container. -## Run all tests +This is the flow used in CI and should be done locally before pushing to GitHub to ensure that no +environmental factors contribute to the outcome of the tests. -Polaris REST server must be running on localhost:8181 before running tests. +## Run Tests Locally -Running test harness will automatically run idempotent setup script. +Regression tests can be run locally using the test harness. +In this setup, a Polaris server must be running on localhost:8181 before running tests. The simplest +way to do this is to run the Polaris server in a separate terminal window in [Quarkus Dev +mode](https://quarkus.io/guides/dev-mode-differences): + +```shell +./gradlew polarisServerDev ``` -./run.sh + +Note: the regression tests expect Polaris to run in "dev" mode, with support for `FILE` storage and +with default realm `default-realm`; if you run the above command, this will be the case. If you run +Polaris in a different way, make sure that Polaris is configured with the following properties: + +```properties +polaris.persistence.type=in-memory +polaris.authentication.authenticator.type=test +polaris.authentication.token-service.type=test +polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["FILE","S3","GCS","AZURE"] +polaris.realm-context.realms=default-realm +quarkus.otel.sdk.disabled=true ``` -## Run in VERBOSE mode with test stdout printing to console +Running the test harness will automatically run the idempotent setup script. From the root of the +project, just run: +```shell +POLARIS_HOST=localhost ./regtests/run.sh ``` -VERBOSE=1 ./run.sh t_spark_sql/src/spark_sql_basic.sh + +To run the tests in verbose mode, with test stdout printing to console, set the `VERBOSE` +environment variable to `1`; you can also choose to run only a subset of tests by specifying the +test directories as arguments to `run.sh`. For example, to run only the `t_spark_sql` tests in +verbose mode: + +```shell +VERBOSE=1 POLARIS_HOST=localhost ./regtests/run.sh t_spark_sql/src/spark_sql_basic.sh ``` ## Run with Cloud resources @@ -78,15 +144,16 @@ into the `credentials` folder. Then specify the name of the file in your .env fi path, as `/tmp/credentials` is the folder on the container where the credentials file will be mounted. -## Experiment with failed test +## Fixing a failed test due to incorrect expected output -``` -rm t_hello_world/ref/hello_world.sh.ref -./run.sh -``` +If a test fails due to incorrect expected output, the test harness will generate a script to help +you compare the actual output with the expected output. The script will be located in the `output` +directory, and will have the same name as the test, with the extension `.fixdiffs.sh`. + +For example, if the test `t_hello_world` fails, the script to compare the actual and expected output +will be located at `output/t_hello_world/hello_world.sh.fixdiffs.sh`: ``` -Tue Apr 23 06:32:23 UTC 2024: Running all tests Tue Apr 23 06:32:23 UTC 2024: Starting test t_hello_world:hello_world.sh Tue Apr 23 06:32:23 UTC 2024: Test run concluded for t_hello_world:hello_world.sh Tue Apr 23 06:32:23 UTC 2024: Test FAILED: t_hello_world:hello_world.sh @@ -96,26 +163,41 @@ Tue Apr 23 06:32:32 UTC 2024: Test run concluded for t_spark_sql:spark_sql_basic Tue Apr 23 06:32:32 UTC 2024: Test SUCCEEDED: t_spark_sql:spark_sql_basic.sh ``` -Simply run the specified fixdiffs file to run `meld` and fix the ref file. +Simply execute the specified `fixdiffs.sh` file, which will in turn run `meld` and fix the ref file: ``` /tmp/polaris-regtests/t_hello_world/hello_world.sh.fixdiffs.sh ``` +Then commit the changes to the ref file. + ## Run a spark-sql interactive shell -With in-memory standalong Polaris running: +With a Polaris server running in "dev" mode (see above), you can run a spark-sql interactive shell +to test. From the root of the project: ``` -./run_spark_sql.sh +POLARIS_HOST=localhost ./regtests/run_spark_sql.sh +``` + +Some SQL commands that you can try: + +```sql +create database db1; +show databases; +create table db1.table1 (id int, name string); +insert into db1.table1 values (1, 'a'); +select * from db1.table1; ``` +Other commands are available in the `regtests/t_spark_sql/src` directory. + ## Python Tests Python tests are based on `pytest`. They rely on a python Polaris client, which is generated from the openapi spec. The client can be generated using two commands: -```bash +```shell # generate the management api client $ docker run --rm \ -v ${PWD}:/local openapitools/openapi-generator-cli generate \ @@ -134,7 +216,7 @@ $ docker run --rm \ Tests rely on Python 3.8 or higher. `pyenv` can be used to install a current version and mapped to the local directory by using -```bash +```shell pyenv install 3.8 pyenv local 3.8 ``` diff --git a/docker-compose.yml b/regtests/docker-compose.yml similarity index 81% rename from docker-compose.yml rename to regtests/docker-compose.yml index e2b3c352a..6e194bd15 100644 --- a/docker-compose.yml +++ b/regtests/docker-compose.yml @@ -18,11 +18,11 @@ # services: + polaris: - build: - context: . + image: apache/polaris:latest ports: - - "8181:8181" + - "8181" - "8182" environment: AWS_REGION: us-west-2 @@ -32,20 +32,23 @@ services: AZURE_TENANT_ID: $AZURE_TENANT_ID AZURE_CLIENT_ID: $AZURE_CLIENT_ID AZURE_CLIENT_SECRET: $AZURE_CLIENT_SECRET - POLARIS_AUTHENTICATION_AUTHENTICATOR_TYPE: test - POLARIS_AUTHENTICATION_TOKEN_SERVICE_TYPE: test - QUARKUS_OTEL_SDK_DISABLED: "true" + polaris.persistence.type: in-memory + polaris.authentication.authenticator.type: test + polaris.authentication.token-service.type: test + polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES": '["FILE","S3","GCS","AZURE"]' + polaris.realm-context.realms: default-realm + quarkus.otel.sdk.disabled: "true" volumes: - - ./regtests/credentials:/tmp/credentials/ - + - ./credentials:/tmp/credentials/ healthcheck: test: ["CMD", "curl", "http://localhost:8182/q/health"] interval: 10s timeout: 10s retries: 5 + regtest: build: - context: regtests + context: . args: POLARIS_HOST: polaris depends_on: @@ -69,5 +72,5 @@ services: AWS_CROSS_REGION_BUCKET: $AWS_CROSS_REGION_BUCKET AWS_ROLE_FOR_CROSS_REGION_BUCKET: $AWS_ROLE_FOR_CROSS_REGION_BUCKET volumes: - - ./regtests/output:/tmp/polaris-regtests/ - - ./regtests/credentials:/tmp/credentials/ + - ./output:/tmp/polaris-regtests/ + - ./credentials:/tmp/credentials/ diff --git a/run.sh b/run.sh index 5947b8985..0ce5804ab 100755 --- a/run.sh +++ b/run.sh @@ -21,13 +21,9 @@ # Runs Polaris as a mini-deployment locally. Creates two pods that bind themselves to port 8181. -# Initialize variables -BUILD_ARGS="" # Initialize an empty string to store Docker build arguments - # Function to display usage information usage() { - echo "Usage: $0 [-b build-arg1=value1;build-arg2=value2;...] [-h]" - echo " -b Pass a set of arbitrary build arguments to docker build, separated by semicolons" + echo "Usage: $0 [-h]" echo " -h Display this help message" exit 1 } @@ -35,12 +31,6 @@ usage() { # Parse command-line arguments while getopts "b:h" opt; do case ${opt} in - b) - IFS=';' read -ra ARGS <<< "${OPTARG}" # Split the semicolon-separated list into an array - for arg in "${ARGS[@]}"; do - BUILD_ARGS+=" --build-arg ${arg}" # Append each build argument to the list - done - ;; h) usage ;; @@ -57,20 +47,17 @@ shift $((OPTIND-1)) echo "Building Kind Registry..." sh ./kind-registry.sh -# Check if BUILD_ARGS is not empty and print the build arguments -if [[ -n "$BUILD_ARGS" ]]; then - echo "Building polaris image with build arguments:$BUILD_ARGS" -else - echo "Building polaris image without any additional build arguments." -fi - # Build and deploy the server image echo "Building polaris image..." -docker build -t localhost:5001/polaris $BUILD_ARGS -f Dockerfile . +./gradlew :polaris-quarkus-server:build \ + -Dquarkus.container-image.build=true \ + -Dquarkus.container-image.registry=localhost:5001 + echo "Pushing polaris image..." -docker push localhost:5001/polaris +docker push localhost:5001/apache/polaris + echo "Loading polaris image to kind..." -kind load docker-image localhost:5001/polaris:latest +kind load docker-image localhost:5001/apache/polaris:latest echo "Applying kubernetes manifests..." kubectl delete -f k8/deployment.yaml --ignore-not-found diff --git a/site/content/in-dev/unreleased/admin-tool.md b/site/content/in-dev/unreleased/admin-tool.md new file mode 100644 index 000000000..92d05ccb1 --- /dev/null +++ b/site/content/in-dev/unreleased/admin-tool.md @@ -0,0 +1,152 @@ +--- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +linkTitle: Polaris Admin Tool +title: Apache Polaris (incubating) Admin Tool +type: docs +weight: 300 +--- + +In order to help administrators manage their Polaris database, Polaris provides an administration +tool. + +The tool is built using [Quarkus](https://quarkus.io/). + +## How to Download the Admin Tool + +As of January 2025, there is currently no binary release or official Docker image available for +the tool. For now, you need to build the artifacts yourself, for example, by running the following +command: + +```shell +./gradlew :polaris-quarkus-admin:build -Dquarkus.container-image.build=true +``` + +The above command will generate: + +- One standalone JAR in `quarkus/admin/build/polaris-quarkus-admin-*-runner.jar` +- Two distribution archives in `quarkus/admin/build/distributions` +- Two Docker image named `apache/polaris-admin-tool:latest` and `apache/polaris-admin-tool:` + +## Usage + +To run the standalone JAR, use the following command: + +```shell +java -jar quarkus/admin/build/polaris-quarkus-admin-*-runner.jar --help +``` + +To unpack and run the distribution, you can use the following command: + +```shell +unzip polaris-quarkus-admin-.zip +cd polaris-quarkus-admin- +java -jar polaris-quarkus-admin--runner.jar +``` + +To run the Docker image, use the following command: + +```shell +docker run apache/polaris-server-admin-tool:latest --help +``` +` +The basic usage of the Polaris Admin Tool is outlined below: + +``` +Usage: polaris-quarkus-admin-runner.jar [-hV] [COMMAND] +Polaris Admin Tool + -h, --help Show this help message and exit. + -V, --version Print version information and exit. +Commands: + help Display help information about the specified command. + bootstrap Bootstraps realms and principal credentials. + purge Purge principal credentials. +``` + +## Configuration + +The Polaris Admin Tool must be executed with the same configuration as the Polaris server. The +configuration can be done via environment variables or system properties. + +At a minimum, it is necessary to configure the Polaris Admin Tool to connect to the same database +used by the Polaris server. This can be done by setting the following system properties: + +```shell +java -jar quarkus/admin/build/polaris-quarkus-admin-*-runner.jar \ + -Dpolaris.persistence.eclipselink.configuration-file=/path/to/persistence.xml + -Dpolaris.persistence.eclipselink.persistence-unit=polaris +``` + +See the [metastore documentation]({{% ref "metastores" %}}) for more information on configuring the +database connection. + +## Bootstrapping Principal Credentials + +The `bootstrap` command is used to bootstrap realms and create the necessary principal credentials for the Polaris +server. This command is idempotent and can be run multiple times without causing any issues. + +```shell +java -jar quarkus/admin/build/polaris-quarkus-admin-*-runner.jar bootstrap --help +``` + +The basic usage of the `bootstrap` command is outlined below: + +``` +Usage: polaris-quarkus-admin-runner.jar bootstrap [-hV] [-c=]... + -r= [-r=]... +Bootstraps realms and principal credentials. + -c, --credential= + Principal credentials to bootstrap. Must be of the + form 'realm,userName,clientId,clientSecret'. + -h, --help Show this help message and exit. + -r, --realm= The name of a realm to bootstrap. + -V, --version Print version information and exit. +``` + +For example, to bootstrap the `realm1` realm and create a principal credential for the user `root` +with the client ID `admin` and client secret `admin`, you can run the following command: + +```shell +java -jar quarkus/admin/build/polaris-quarkus-admin-*-runner.jar bootstrap -r realm1 -c realm1,root,admin,admin +``` + +## Purging Principal Credentials + +The `purge` command is used to remove realms and principal credentials from the Polaris server. This +command is idempotent and can be run multiple times without causing any issues. + +```shell +java -jar quarkus/admin/build/polaris-quarkus-admin-*-runner.jar purge --help +``` + +The basic usage of the `purge` command is outlined below: + +``` +Usage: polaris-quarkus-admin-runner.jar purge [-hV] -r= [-r=]... +Purge principal credentials. + -h, --help Show this help message and exit. + -r, --realm= The name of a realm to purge. + -V, --version Print version information and exit. +``` + +For example, to purge the `realm1` realm, you can run the following command: + +```shell +java -jar quarkus/admin/build/polaris-quarkus-admin-*-runner.jar purge -r realm1 +``` \ No newline at end of file diff --git a/site/content/in-dev/unreleased/configuring-polaris-for-production.md b/site/content/in-dev/unreleased/configuring-polaris-for-production.md index ebf4c9d4a..b5cb30b71 100644 --- a/site/content/in-dev/unreleased/configuring-polaris-for-production.md +++ b/site/content/in-dev/unreleased/configuring-polaris-for-production.md @@ -23,116 +23,205 @@ type: docs weight: 600 --- -The default `polaris-server.yml` configuration is intended for development and testing. When deploying Polaris in production, there are several best practices to keep in mind. +Polaris server runs on Quarkus. Please refer to the [Quarkus +documentation](https://quarkus.io/guides/) to get familiar with Quarkus. -## Security +## How to Download Polaris Distributions -### Configurations +As of January 2025, there is currently no binary release or official Docker image available for +Polaris. For now, you need to build the artifacts yourself, for example, by running the following +command: -Notable configuration used to secure a Polaris deployment are outlined below. +```bash +./gradlew assemble -Dquarkus.container-image.build=true +``` -#### oauth2 +The above command will generate: -> [!WARNING] -> Ensure that the `tokenBroker` setting reflects the token broker specified in `authenticator` below. +- One tar.gz distribution archive in `quarkus/server/build/distributions` +- One zip distribution archive in the same directory +- One Docker image `apache/polaris:latest` +- One Docker image `apache/polaris:` -* Configure [OAuth](https://oauth.net/2/) with this setting. Remove the `TestInlineBearerTokenPolarisAuthenticator` option and uncomment the `DefaultPolarisAuthenticator` authenticator option beneath it. -* Then, configure the token broker. You can configure the token broker to use either [asymmetric](https://github.com/apache/polaris/blob/b482617bf8cc508b37dbedf3ebc81a9408160a5e/polaris-service/src/main/java/io/polaris/service/auth/JWTRSAKeyPair.java#L24) or [symmetric](https://github.com/apache/polaris/blob/b482617bf8cc508b37dbedf3ebc81a9408160a5e/polaris-service/src/main/java/io/polaris/service/auth/JWTSymmetricKeyBroker.java#L23) keys. +Note: the above command will also generate artifacts for the [Polaris Admin Tool]({{% ref +"admin-tool" %}}). -#### authenticator.tokenBroker +## Server Configuration -> [!WARNING] -> Ensure that the `tokenBroker` setting reflects the token broker specified in `oauth2` above. +The default server configuration is ready for production, but you may need to customize it to your +needs. -#### callContextResolver & realmIdResolver -* Use these configurations to specify a service that can resolve a realm from bearer tokens. -* The service(s) used here must implement the relevant interfaces (i.e. [CallContextResolver](https://github.com/apache/polaris/blob/8290019c10290a600e40b35ddb1e2f54bf99e120/polaris-service/src/main/java/io/polaris/service/context/CallContextResolver.java#L27) and [RealmContextResolver](https://github.com/apache/polaris/blob/7ce86f10a68a3b56aed766235c88d6027c0de038/polaris-service/src/main/java/io/polaris/service/context/RealmContextResolver.java)). +Quarkus aggregates configuration properties from multiple sources, applying them in a specific order +of precedence. When a property is defined in multiple sources, the value from the source with the +higher priority overrides those from lower-priority sources. -## Metastore Management +The sources are listed below, from highest to lowest priority: -> [!IMPORTANT] -> The default `in-memory` implementation for `metastoreManager` is meant for testing and not suitable for production usage. Instead, consider an implementation such as `eclipse-link` which allows you to store metadata in a remote database. +1. System properties: Properties set via the Java command line using `-Dproperty.name=value`. +2. Environment variables. +3. Settings in `$PWD/config/application.properties` file. +4. The `application.properties` files packaged in Polaris. +5. Default values: hardcoded defaults within the application. -A Metastore Manger should be configured with an implementation that durably persists Polaris entities. Use the configuration `metaStoreManager` to configure a [MetastoreManager](https://github.com/apache/polaris/blob/627dc602eb15a3258dcc32babf8def34cf6de0e9/polaris-core/src/main/java/io/polaris/core/persistence/PolarisMetaStoreManager.java#L47) implementation where Polaris entities will be persisted. +Read Quarkus [configuration guide](https://quarkus.io/guides/config) for more information. -Be sure to secure your metastore backend since it will be storing credentials and catalog metadata. +## Security -### Configuring EclipseLink +Notable configuration options used to secure a Polaris deployment are outlined below. -To use EclipseLink for metastore management, specify the configuration `metaStoreManager.conf-file` to point to an EclipseLink `persistence.xml` file. This file, local to the Polaris service, contains details of the database used for metastore management and the connection settings. For more information, refer to the [metastore documentation]({{% ref "metastores" %}}). +Polaris authentication requires specifying a token broker factory type. Two types are supported: -> [!IMPORTANT] -> EclipseLink requires -> 1. Building the JAR for the EclipseLink extension -> 2. Setting the `eclipseLink` gradle property to `true`. -> -> This can be achieved by setting `eclipseLink=true` in the `gradle.properties` file, or by passing the property explicitly while building all JARs, e.g.: `./gradlew -PeclipseLink=true clean assemble` +- `rsa-key-pair` uses a pair of public and private keys +- `symmetric-key` uses a shared secret -### Bootstrapping +By default, Polaris uses `rsa-key-pair`, with randomly generated keys. -Before using Polaris when using a metastore manager other than `in-memory`, you must **bootstrap** the metastore manager. This is a manual operation that must be performed **only once** in order to prepare the metastore manager to integrate with Polaris. When the metastore manager is bootstrapped, any existing Polaris entities in the metastore manager may be **purged**. +> [!WARNING] +> The default `rsa-key-pair` configuration is not suitable when deploying many replicas of Polaris, +> as each replica will have its own set of keys. This will cause token validation to fail when a +> request is routed to a different replica than the one that issued the token. -By default, Polaris will create randomised `CLIENT_ID` and `CLIENT_SECRET` for the `root` principal and store their hashes in the metastore backend. In order to provide your own credentials for `root` principal (so you can request tokens via `api/catalog/v1/oauth/tokens`), set the `POLARIS_BOOTSTRAP_CREDENTIALS` environment variable as follows: +It is highly recommended to configure Polaris with previously-generated RSA keys. This can be done +by setting the following properties in `application.properties`: +```properties +polaris.authentication.token-broker.type=rsa-key-pair +polaris.authentication.token-broker.rsa-key-pair.public-key-file=/tmp/public.key +polaris.authentication.token-broker.rsa-key-pair.private-key-file=/tmp/private.key ``` -export POLARIS_BOOTSTRAP_CREDENTIALS=my_realm,root,my-client-id,my-client-secret + +Alternatively, you can use a symmetric key by setting the following properties in +`application.properties`: + +```properties +polaris.authentication.token-broker.type=symmetric-key +polaris.authentication.token-broker.symmetric-key.file=/tmp/symmetric.key ``` -The format of the environment variable is `realm,principal,client_id,client_secret`. You can provide multiple credentials separated by `;`. For example, to provide credentials for two realms `my_realm` and `my_realm2`: +Note: it is also possible to set the symmetric key directly in the configuration file, but that is +not recommended for production use, as the secret is stored in plain text: +```properties +polaris.authentication.token-broker.symmetric-key.secret=my-secret ``` -export POLARIS_BOOTSTRAP_CREDENTIALS=my_realm,root,my-client-id,my-client-secret;my_realm2,root,my-client-id2,my-client-secret2 + +Finally, you can also configure the token broker to use a maximum lifespan by setting the following +property in `application.properties`: + +```properties +polaris.authentication.token-broker.max-token-generation=PT1H ``` -You can also provide credentials for other users too. +## Realm Id Resolver -It is also possible to use system properties to provide the credentials: +By default, Polaris resolves realms based on incoming request headers. You can configure the realm +context resolver by setting the following properties in `application.properties`: -``` -java -Dpolaris.bootstrap.credentials=my_realm,root,my-client-id,my-client-secret -jar /path/to/jar/polaris-service-all.jar bootstrap polaris-server.yml +```properties +polaris.realm-context.realms=realm1,realm2,realm3 +polaris.realm-context.header-name=Polaris-Realm ``` -Now, to bootstrap Polaris, run: +Where: -```bash -java -jar /path/to/jar/polaris-service-all.jar bootstrap polaris-server.yml -``` +- `realms` is a comma-separated list of allowed realms. This setting _must_ be correctly configured. + At least one realm must be specified. +- `header-name` is the name of the header used to resolve the realm; by default, it is + `Polaris-Realm`. -or in a container: +If a request does not contain the specified header, Polaris will use the first realm in the list as +the default realm. -```bash -bin/polaris-service bootstrap config/polaris-server.yml +## Metastore Configuration + +A Metastore Manager should be configured with an implementation that durably persists Polaris +entities. By default, Polaris uses the EclipseLink implementation, with an in-memory H2 database. + +> [!WARNING] +> The default in-memory H2 database is not suitable for production use, as it will lose all data +> when the server is restarted; it is also unusable when multiple Polaris replicas are used. + +To use a different Metastore Manager, you need to provide your own `persistence.xml` file. This file +contains details of the database used for metastore management and the connection settings. For more +information, refer to the [metastore documentation]({{% ref "metastores" %}}). + +Then, configure Polaris to use it by setting the following properties in `application.properties`: + +```properties +polaris.persistence.eclipselink.configuration-file=/path/to/persistence.xml +polaris.persistence.eclipselink.persistence-unit=polaris ``` -Afterward, Polaris can be launched normally: +Where: -```bash -java -jar /path/to/jar/polaris-service-all.jar server polaris-server.yml +- `configuration-file` is the path to the `persistence.xml` file. It can also be a classpath + resource. +- `persistence-unit` is the name of the persistence unit to use (in case the configuration file + has many persistence units). + +Be sure to secure your metastore backend since it will be storing credentials and catalog metadata. + +### Bootstrapping + +Before using Polaris, you must **bootstrap** the metastore manager. This is a manual operation that +must be performed **only once** in order to prepare the metastore manager to integrate with Polaris. +When the metastore manager is bootstrapped, any existing Polaris entities in the metastore manager +may be **purged**. + +By default, Polaris will create randomised `CLIENT_ID` and `CLIENT_SECRET` for the `root` principal +and store their hashes in the metastore backend. + +In order to provide your own credentials for `root` principal (so you can request tokens via +`api/catalog/v1/oauth/tokens`), there are two approaches: + +1. Use the [Polaris Admin Tool]({{% ref "admin-tool" %}}) to set the `root` principal credentials. +2. Set the `root` principal credentials when deploying Polaris for the first time. + +The first approach is recommended as it is more flexible. The second approach is useful for testing +and development, but can be used in production as well. + +In order to bootstrap root credentials for a realm name `my-realm` when deploying Polaris, set the +following environment variables: + +``` +export POLARIS_BOOTSTRAP_CREDENTIALS=my-realm,root,my-client-id,my-client-secret ``` +You can also provide credentials for other users too. + +If the realm hasn't been bootstrapped yet, Polaris will create the realm and the `root` principal +with the provided credentials upon first usage of that realm. If the realm already exists, Polaris +will not attempt to update the `root` principal credentials. + You can verify the setup by attempting a token issue for the `root` principal: ```bash -curl -X POST http://localhost:8181/api/catalog/v1/oauth/tokens -d "grant_type=client_credentials&client_id=my-client-id&client_secret=my-client-secret&scope=PRINCIPAL_ROLE:ALL" +curl -X POST http://localhost:8181/api/catalog/v1/oauth/tokens \ + -d "grant_type=client_credentials" \ + -d "client_id=my-client-id" \ + -d "client_secret=my-client-secret" \ + -d "scope=PRINCIPAL_ROLE:ALL" ``` -which should return: +Which should return an access token: ```json -{"access_token":"...","token_type":"bearer","issued_token_type":"urn:ietf:params:oauth:token-type:access_token","expires_in":3600} +{ + "access_token": "...", + "token_type": "bearer", + "issued_token_type": "urn:ietf:params:oauth:token-type:access_token", + "expires_in": 3600 +} ``` -Note that if you used non-default realm name, for example, `iceberg` instead of `default-realm` in your `polaris-server.yml`, then you should add an appropriate request header: +Note that if you used a realm name that is not the default realm name, then you should add an +appropriate request header to the `curl` command, for example: + ```bash -curl -X POST -H 'realm: iceberg' http://localhost:8181/api/catalog/v1/oauth/tokens -d "grant_type=client_credentials&client_id=my-client-id&client_secret=my-client-secret&scope=PRINCIPAL_ROLE:ALL" +curl -X POST http://localhost:8181/api/catalog/v1/oauth/tokens \ + -H "Polaris-Realm: my-realm" \ + -d "grant_type=client_credentials" \ + -d "client_id=my-client-id" \ + -d "client_secret=my-client-secret" \ + -d "scope=PRINCIPAL_ROLE:ALL" ``` - -## Other Configurations - -When deploying Polaris in production, consider adjusting the following configurations: - -#### featureConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES - - By default Polaris catalogs are allowed to be located in local filesystem with the `FILE` storage type. This should be disabled for production systems. - - Use this configuration to additionally disable any other storage types that will not be in use. - - diff --git a/site/content/in-dev/unreleased/metastores.md b/site/content/in-dev/unreleased/metastores.md index 8b287ffb2..61eb8f663 100644 --- a/site/content/in-dev/unreleased/metastores.md +++ b/site/content/in-dev/unreleased/metastores.md @@ -26,32 +26,40 @@ weight: 700 This page documents important configurations for connecting to production database through [EclipseLink](https://eclipse.dev/eclipselink/). ## Polaris Server Configuration -Configure the `metaStoreManager` section in the Polaris configuration (`polaris-server.yml` by default) as follows: + +Configure the `polaris.persistence` section in the Polaris configuration (`application.properties`) as follows: + ``` -metaStoreManager: - type: eclipse-link - conf-file: META-INF/persistence.xml - persistence-unit: polaris +polaris.persistence.eclipselink.configuration-file=/path/to/persistence.xml +polaris.persistence.eclipselink.persistence-unit=polaris ``` -`conf-file` must point to an [EclipseLink configuration file](https://eclipse.dev/eclipselink/documentation/2.5/solutions/testingjpa002.htm) +`configuration-file` must point to an [EclipseLink configuration file](https://eclipse.dev/eclipselink/documentation/2.5/solutions/testingjpa002.htm). -By default, `conf-file` points to the embedded resource file `META-INF/persistence.xml` in the `polaris-eclipselink` module. +## EclipseLink Configuration - persistence.xml -In order to specify a configuration file outside the classpath, follow these steps. -1) Place `persistence.xml` into a jar file: `jar cvf /tmp/conf.jar persistence.xml` -2) Use `conf-file: /tmp/conf.jar!/persistence.xml` +The configuration file `persistence.xml` is used to set up the database connection properties, which +can differ depending on the type of database and its configuration. -## EclipseLink Configuration - persistence.xml -The configuration file `persistence.xml` is used to set up the database connection properties, which can differ depending on the type of database and its configuration. +The path to the `persistence.xml` file and the persistence unit name must be set in the Polaris +configuration: -Check out the default [persistence.xml](https://github.com/apache/polaris/blob/main/extension/persistence/eclipselink/src/main/resources/META-INF/persistence.xml) for a complete sample for connecting to the file-based H2 database. +```properties +polaris.persistence.eclipselink.configuration-file=/path/to/persistence.xml +polaris.persistence.eclipselink.persistence-unit=polaris +``` -Polaris creates and connects to a separate database for each realm. Specifically, the `{realm}` placeholder in `jakarta.persistence.jdbc.url` is substituted with the actual realm name, allowing the Polaris server to connect to different databases based on the realm. +Check out the default [persistence.xml] for a complete sample below. An in-memory H2 database is +used by default, but you can easily switch to a different database. + +[persistence.xml]: https://github.com/apache/polaris/blob/main/extension/persistence/eclipselink/src/main/resources/META-INF/persistence.xml -> Note: some database systems such as Postgres don't create databases automatically. Database admins need to create them manually before running Polaris server. ```xml - + + + org.eclipse.persistence.jpa.PersistenceProvider org.apache.polaris.jpa.models.ModelEntity org.apache.polaris.jpa.models.ModelEntityActive @@ -62,22 +70,23 @@ Polaris creates and connects to a separate database for each realm. Specifically org.apache.polaris.jpa.models.ModelSequenceId NONE - + + + + - + + ``` -A single `persistence.xml` can describe multiple [persistence units](https://eclipse.dev/eclipselink/documentation/2.6/concepts/app_dev001.htm). For example, with both a `polaris-dev` and `polaris` persistence unit defined, you could use a single `persistence.xml` to easily switch between development and production databases. Use `persistence-unit` in the Polaris server configuration to easily switch between persistence units. +Polaris creates and connects to a separate database for each realm. Specifically, the `{realm}` placeholder in `jakarta.persistence.jdbc.url` is substituted with the actual realm name, allowing the Polaris server to connect to different databases based on the realm. + +> Note: some database systems such as Postgres don't create databases automatically. Database admins need to create them manually before running Polaris server. -To build Polaris with the necessary H2 dependency and start the Polaris service, run the following: -```bash -polaris> ./gradlew --no-daemon --info -PeclipseLink=true -PeclipseLinkDeps=com.h2database:h2:2.3.232 clean shadowJar -polaris> java -jar quarkus/service/build/quarkus-app/quarkus-run.jar -``` +A single `persistence.xml` can describe multiple [persistence units](https://eclipse.dev/eclipselink/documentation/2.6/concepts/app_dev001.htm). For example, with both a `polaris-dev` and `polaris` persistence unit defined, you could use a single `persistence.xml` to easily switch between development and production databases. Use `polaris.persistence.eclipselink.persistence-unit` in the Polaris server configuration to easily switch between persistence units. ### Postgres @@ -104,9 +113,3 @@ The following shows a sample configuration for integrating Polaris with Postgres ``` - -To build Polaris with the necessary Postgres dependency and start the Polaris service, run the following: -```bash -polaris> ./gradlew --no-daemon --info -PeclipseLink=true -PeclipseLinkDeps=org.postgresql:postgresql:42.7.4 clean shadowJar -polaris> java -jar quarkus/service/build/quarkus-app/quarkus-run.jar -``` \ No newline at end of file diff --git a/site/content/in-dev/unreleased/quickstart.md b/site/content/in-dev/unreleased/quickstart.md index 1cdb7dab6..584f66363 100644 --- a/site/content/in-dev/unreleased/quickstart.md +++ b/site/content/in-dev/unreleased/quickstart.md @@ -97,42 +97,52 @@ To start using Polaris in Docker, launch Polaris while Docker is running: ```shell cd ~/polaris -docker compose -f docker-compose.yml up --build +./gradlew clean :polaris-quarkus-server:assemble -Dquarkus.container-image.build=true +docker run -p 8181:8181 -p 8182:8182 apache/polaris:latest ``` -Once the `polaris-polaris` container is up, you can continue to [Defining a Catalog](#defining-a-catalog). +You should see output for some time as Polaris builds and starts up. Eventually, you won’t see any more logs and should see messages that resemble the following: + +``` +INFO [io.quarkus] [,] [,,,] (Quarkus Main Thread) Apache Polaris Server on JVM (powered by Quarkus ) started in 2.656s. Listening on: http://localhost:8181. Management interface listening on http://0.0.0.0:8182. +INFO [io.quarkus] [,] [,,,] (Quarkus Main Thread) Profile prod activated. +INFO [io.quarkus] [,] [,,,] (Quarkus Main Thread) Installed features: [...] +``` + +Once the container is up, you can continue to [Defining a Catalog](#defining-a-catalog). ### Building Polaris -Run Polaris locally with: +Run Polaris locally in [Dev mode](https://quarkus.io/guides/dev-mode-differences) with: ```shell cd ~/polaris -./gradlew runApp +./gradlew polarisServerDev ``` You should see output for some time as Polaris builds and starts up. Eventually, you won’t see any more logs and should see messages that resemble the following: ``` -INFO [...] [main] [] o.e.j.s.handler.ContextHandler: Started i.d.j.MutableServletContextHandler@... -INFO [...] [main] [] o.e.j.server.AbstractConnector: Started application@... -INFO [...] [main] [] o.e.j.server.AbstractConnector: Started admin@... -INFO [...] [main] [] o.eclipse.jetty.server.Server: Started Server@... +realm: POLARIS root principal credentials: : +INFO [io.quarkus] [,] [,,,] (Quarkus Main Thread) polaris-quarkus-service 1.0.0-incubating-SNAPSHOT on JVM (powered by Quarkus 3.17.6) started in 2.656s. Listening on: http://localhost:8181. Management interface listening on http://0.0.0.0:8182. +INFO [io.quarkus] [,] [,,,] (Quarkus Main Thread) Profile dev activated. Live Coding activated. +INFO [io.quarkus] [,] [,,,] (Quarkus Main Thread) Installed features: [...] ``` -At this point, Polaris is running. +At this point, Polaris is running. Take note of the root principal credentials provided in the logs, as we'll use them in the next section. ## Bootstrapping Polaris For this tutorial, we'll launch an instance of Polaris that stores entities only in-memory. This means that any entities that you define will be destroyed when Polaris is shut down. It also means that Polaris will automatically bootstrap itself with root credentials. For more information on how to configure Polaris for production usage, see the [docs]({{% ref "configuring-polaris-for-production" %}}). -When Polaris is launched using in-memory mode the root principal credentials can be found in stdout on initial startup. For example: +When Polaris is launched using an in-memory metastore, such as when Dev mode is used, the root +principal credentials can be found in stdout on initial startup. For example: ``` -realm: default-realm root principal credentials: : +realm: POLARIS root principal credentials: : ``` -Be sure to note of these credentials as we'll be using them below. You can also set these credentials as environment variables for use with the Polaris CLI: +Be sure to take note of these credentials as we'll be using them below. You can also set these credentials as environment variables for use with the Polaris CLI: ```shell export CLIENT_ID=