From 11c341916dd4d3412512747dc3685903bdcd5ec6 Mon Sep 17 00:00:00 2001 From: Ed Espino Date: Fri, 21 Jun 2024 18:53:34 -0700 Subject: [PATCH 1/3] CBDB main branch with Rocky Linux 9.4 support Updated run.sh to support: * Building against the latest release as indicated by release zip file contained in configs directory (e.g. cloudberrydb-1.5.2.zip) and using centos7. The Dockerfile `Dockerfile.RELEASE.centos7` will be used. The docker file is now passed a docker variable (CODEBASE_VERSION_VAR) which is calculated dynamically based on the version string embeded in the release zip file. * Building against the latest main branch and using Rocky Linux 9. The Dockerfile `Dockerfile.main.rockylinux9` will be used. * The CBDB cluster now has three primaries and mirrors. This will allow developers and users to test segment failover scenarios. * As long as the `Dockerfile.RELEASE.centos7` supports the release zip/tar files, only the release zip file needs to be updated to support a new release. Here is the current usage for run.sh followed by several usage examples: ``` Usage: $0 [-o ] [-c ] [-b] -o OS version (valid values: centos7, rockylinux9; default: $DEFAULT_OS_VERSION, or set via OS_VERSION environment variable) -c Codebase version (valid values: main, or determined from release zip file name) -t Timezone (default: Asia/Shanghai, or set via TIMEZONE_VAR environment variable) -b Build only, do not run the container (default: false, or set via BUILD_ONLY environment variable) ``` **Example** Build and deploy container using latest release version specified in the zip file contained in the configs directory (e.g. cloudberrydb-1.5.2.zip). ``` ./run.sh ``` **Example** Only build container using latest release version specified in the zip file contained in the configs directory (e.g. cloudberrydb-1.5.2.zip). ``` ./run.sh -b ``` **Example** Only build container using main branch on Rocky Linux 9 setting the timezone "America/Los_Angeles" ``` ./run.sh -c main -o rockylinux9 -t "America/Los_Angeles" ``` **Example** Only build container using main branch on Rocky Linux 9 ``` ./run.sh -c main -o rockylinux9 -b ``` --- ..._Dockerfile => Dockerfile.RELEASE.centos7} | 19 ++- 000-cbdb-sandbox/Dockerfile.main.rockylinux9 | 132 ++++++++++++++++++ 000-cbdb-sandbox/README.md | 31 ++-- 000-cbdb-sandbox/configs/90-cbdb-limits.conf | 10 ++ .../{sysctl.conf.add => 90-cbdb-sysctl.conf} | 2 +- .../configs/gpinitsystem_singlenode | 28 ++-- 000-cbdb-sandbox/configs/init_system.sh | 84 +++++++++-- 000-cbdb-sandbox/configs/limits.conf.add | 9 -- .../configs/python-dependencies.txt | 3 - 000-cbdb-sandbox/run.sh | 126 ++++++++++++++++- 10 files changed, 385 insertions(+), 59 deletions(-) rename 000-cbdb-sandbox/{cbdb_Dockerfile => Dockerfile.RELEASE.centos7} (83%) create mode 100644 000-cbdb-sandbox/Dockerfile.main.rockylinux9 create mode 100644 000-cbdb-sandbox/configs/90-cbdb-limits.conf rename 000-cbdb-sandbox/configs/{sysctl.conf.add => 90-cbdb-sysctl.conf} (97%) mode change 100644 => 100755 000-cbdb-sandbox/configs/init_system.sh delete mode 100644 000-cbdb-sandbox/configs/limits.conf.add delete mode 100644 000-cbdb-sandbox/configs/python-dependencies.txt diff --git a/000-cbdb-sandbox/cbdb_Dockerfile b/000-cbdb-sandbox/Dockerfile.RELEASE.centos7 similarity index 83% rename from 000-cbdb-sandbox/cbdb_Dockerfile rename to 000-cbdb-sandbox/Dockerfile.RELEASE.centos7 index ebaa1f4..9299f73 100644 --- a/000-cbdb-sandbox/cbdb_Dockerfile +++ b/000-cbdb-sandbox/Dockerfile.RELEASE.centos7 @@ -1,4 +1,9 @@ FROM centos:7.9.2009 + +# Define a build argument with a default value +ARG CODEBASE_VERSION_VAR=${CODEBASE_VERSION_VAR} +ARG TIMEZONE_VAR="Asia/Shanghai" + RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == \ systemd-tmpfiles-setup.service ] || rm -f $i; done); \ rm -f /lib/systemd/system/multi-user.target.wants/*;\ @@ -18,10 +23,10 @@ RUN echo root:cbdb@123 | chpasswd \ && yum install -y centos-release-scl scl-utils \ && yum install -y devtoolset-10 -RUN cat /tmp/sysctl.conf.add >> /etc/sysctl.conf \ +RUN cat /tmp/90-cbdb-sysctl.conf >> /etc/sysctl.conf \ && sed -i "s/#UseDNS YES/UseDNS no/g" /etc/ssh/sshd_config \ - && cat /tmp/limits.conf.add >> /etc/security/limits.conf \ - && cat /usr/share/zoneinfo/Asia/Shanghai > /etc/localtime \ + && cat /tmp/90-cbdb-limits.conf >> /etc/security/limits.conf \ + && cat /usr/share/zoneinfo/${TIMEZONE_VAR} > /etc/localtime \ && echo "mdw" > /tmp/gpdb-hosts \ && echo "/usr/local/lib" >> /etc/ld.so.conf \ && echo "/usr/local/lib64" >> /etc/ld.so.conf \ @@ -47,11 +52,11 @@ RUN cat /tmp/sysctl.conf.add >> /etc/sysctl.conf \ RUN yum install -y python3-devel python3-pip; yum clean all RUN cd /tmp/ \ - && unzip -d /tmp /tmp/cloudberrydb-1.5.2.zip \ - && cd /tmp/cloudberrydb-1.5.2 \ + && unzip -d /tmp /tmp/cloudberrydb-${CODEBASE_VERSION_VAR}.zip \ + && cd /tmp/cloudberrydb-${CODEBASE_VERSION_VAR} \ && pip3 install -i https://mirrors.aliyun.com/pypi/simple -r python-dependencies.txt -RUN cd /tmp/cloudberrydb-1.5.2 \ +RUN cd /tmp/cloudberrydb-${CODEBASE_VERSION_VAR} \ && source scl_source enable devtoolset-10 \ && ./configure --with-perl --with-python --with-libxml --with-gssapi --prefix=/usr/local/cloudberry-db \ && make -j8 \ @@ -60,4 +65,4 @@ RUN cd /tmp/cloudberrydb-1.5.2 \ EXPOSE 5432 22 VOLUME [ "/sys/fs/cgroup" ] -CMD ["bash","-c","/tmp/init_system.sh"] \ No newline at end of file +CMD ["bash","-c","/tmp/init_system.sh"] diff --git a/000-cbdb-sandbox/Dockerfile.main.rockylinux9 b/000-cbdb-sandbox/Dockerfile.main.rockylinux9 new file mode 100644 index 0000000..71ba517 --- /dev/null +++ b/000-cbdb-sandbox/Dockerfile.main.rockylinux9 @@ -0,0 +1,132 @@ +FROM rockylinux/rockylinux:9 + +ARG TIMEZONE_VAR="Asia/Shanghai" + +ENV container docker + +RUN dnf update -y && \ + dnf install -y systemd \ + systemd-libs && \ + dnf clean all + +# Clean up unnecessary systemd units +RUN [ -d /lib/systemd/system/sysinit.target.wants ] && find /lib/systemd/system/sysinit.target.wants/ -type l -not -name 'systemd-tmpfiles-setup.service' -delete || echo "Directory /lib/systemd/system/sysinit.target.wants does not exist" && \ + [ -d /lib/systemd/system/multi-user.target.wants ] && find /lib/systemd/system/multi-user.target.wants/ -type l -delete || echo "Directory /lib/systemd/system/multi-user.target.wants does not exist" && \ + [ -d /etc/systemd/system/*.wants ] && find /etc/systemd/system/*.wants/ -type l -delete || echo "Directory /etc/systemd/system/*.wants does not exist" && \ + [ -d /lib/systemd/system/local-fs.target.wants ] && find /lib/systemd/system/local-fs.target.wants/ -type l -delete || echo "Directory /lib/systemd/system/local-fs.target.wants does not exist" && \ + [ -d /lib/systemd/system/sockets.target.wants ] && find /lib/systemd/system/sockets.target.wants/ -type l -not -name '*udev*' -delete || echo "Directory /lib/systemd/system/sockets.target.wants does not exist" && \ + [ -d /lib/systemd/system/basic.target.wants ] && find /lib/systemd/system/basic.target.wants/ -type l -delete || echo "Directory /lib/systemd/system/basic.target.wants does not exist" && \ + [ -d /lib/systemd/system/anaconda.target.wants ] && find /lib/systemd/system/anaconda.target.wants/ -type l -delete || echo "Directory /lib/systemd/system/anaconda.target.wants does not exist" + +COPY ./configs/* /tmp/ + +RUN echo root:cbdb@123 | chpasswd && \ + dnf makecache && \ + dnf install -y yum-utils \ + epel-release \ + git && \ + yum-config-manager --disable epel-cisco-openh264 && \ + dnf makecache && \ + yum-config-manager --disable epel && \ + dnf install -y --enablerepo=epel \ + the_silver_searcher \ + bat \ + htop && \ + dnf install -y bison \ + cmake3 \ + ed \ + flex \ + gcc \ + gcc-c++ \ + glibc-langpack-en \ + go \ + initscripts \ + iproute \ + less \ + m4 \ + net-tools \ + openssh-clients \ + openssh-server \ + passwd \ + perl \ + rsync \ + sudo \ + tar \ + unzip \ + util-linux-ng \ + wget \ + which && \ + dnf install -y apr-devel \ + bzip2-devel \ + krb5-devel \ + libcurl-devel \ + libevent-devel \ + libxml2-devel \ + libzstd-devel \ + openldap-devel \ + openssl-devel \ + pam-devel \ + perl-ExtUtils-Embed \ + perl-Test-Simple \ + perl-core \ + python3-devel \ + readline-devel \ + zlib-devel && \ + dnf install -y --enablerepo=crb \ + libuv-devel \ + libyaml-devel \ + perl-IPC-Run && \ + dnf install -y --enablerepo=epel \ + xerces-c-devel + +RUN cp /tmp/90-cbdb-sysctl.conf /etc/sysctl.conf && \ + cp /tmp/90-cbdb-limits.conf /etc/security/limits.d/90-cbdb-limits.conf && \ + cat /usr/share/zoneinfo/${TIMEZONE_VAR} > /etc/localtime && \ + echo "mdw" > /tmp/gpdb-hosts && \ + echo "/usr/local/lib" >> /etc/ld.so.conf && \ + echo "/usr/local/lib64" >> /etc/ld.so.conf && \ + ldconfig && \ + chmod 777 /tmp/gpinitsystem_singlenode && \ + chmod 777 /tmp/init_system.sh && \ + hostname > ~/orig_hostname && \ + /usr/sbin/groupadd gpadmin && \ + /usr/sbin/useradd gpadmin -g gpadmin -G wheel && \ + echo "cbdb@123"|passwd --stdin gpadmin && \ + echo "gpadmin ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \ + echo "root ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \ + echo "export COORDINATOR_DATA_DIRECTORY=/data0/database/master/gpseg-1" >> /home/gpadmin/.bashrc && \ + echo "source /usr/local/cloudberry-db/greenplum_path.sh" >> /home/gpadmin/.bashrc && \ + mkdir -p /data0/database/master /data0/database/primary /data0/database/mirror && \ + chown -R gpadmin:gpadmin /data0 && \ + ssh-keygen -A && \ + echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config + +RUN cd /tmp/ && \ + git clone --branch main --single-branch --depth=1 https://github.com/cloudberrydb/cloudberrydb.git + +RUN cd /tmp/cloudberrydb && \ + ./configure --prefix=/usr/local/cloudberry-db \ + --enable-cassert \ + --enable-debug-extensions \ + --enable-ic-proxy \ + --enable-mapreduce \ + --enable-orafce \ + --enable-orca \ + --enable-pxf \ + --enable-tap-tests \ + --with-gssapi \ + --with-ldap \ + --with-libxml \ + --with-openssl \ + --with-pam \ + --with-perl \ + --with-pgport=5432 \ + --with-python \ + --with-pythonsrc-ext && \ + make -j$(nproc) && \ + make install + +EXPOSE 5432 22 + +VOLUME [ "/sys/fs/cgroup" ] +CMD ["bash","-c","/tmp/init_system.sh"] diff --git a/000-cbdb-sandbox/README.md b/000-cbdb-sandbox/README.md index fd71c16..35b831b 100644 --- a/000-cbdb-sandbox/README.md +++ b/000-cbdb-sandbox/README.md @@ -19,7 +19,10 @@ Make sure that your environment meets the following requirements: ## Build the Sandbox -This section introduces how to set up the Docker container in which the source code of the latest Cloudberry Database (released in [Cloudberry Database Release Page](https://github.com/cloudberrydb/cloudberrydb/releases)) will be compiled. In this CentOS 7.9 Docker container, a single-node cluster will be initialized with one coordinator and two segments. Both x86 and ARM CPUs (including Apple chips) are supported. +This section introduces two methods to set up the Docker container. The container will host a CBDB single-node cluster intialized with one coordinator and three primary and mirror segments. Both x86 and ARM CPUs (including Apple chips) are supported. + +- Method 1 - Compile with the source code of the latest Cloudberry Database (released in [Cloudberry Database Release Page](https://github.com/cloudberrydb/cloudberrydb/releases)). The base OS will use CentOS 7.9 Docker container. +- Method 2 - Compile with the latest Cloudberry Database [main](https://github.com/cloudberrydb/cloudberrydb/tree/main) branch. The base OS will use Rocky Linux 9 Docker container. Build steps: @@ -31,15 +34,25 @@ Build steps: git clone https://github.com/cloudberrydb/bootcamp.git ``` -3. Enter the repository and run the `run.sh` script to start the Docker container. This will start the automatic installation process. +3. Enter the repository and run the `run.sh` script to start the Docker container. This will start the automatic installation process. Depending on your environment, you may need to run this with 'sudo' command. + + - For latest Cloudberry DB release + + ```shell + cd bootcamp/000-cbdb-sandbox + ./run.sh + ``` + + - For latest main branch ```shell cd bootcamp/000-cbdb-sandbox - chmod +x ./run.sh - sudo ./run.sh + ./run.sh -c main -o rockylinux9 ``` - Once the script finishes without error, the sandbox is built successfully. + Once the script finishes without error, the sandbox is built and running successfully. The `docker run` command uses --detach option allowing you to ssh or access the running CBDB instance remotely. + + Please review run.sh script for additional options (e.g. setting Timezone in running container, only building container) ## Connect to the database @@ -61,11 +74,11 @@ You can now connect to the database and try some basic operations. ```shell [root@mdw /] su - gpadmin # Switches to the gpadmin user. - + # Last login: Tue Oct 24 10:26:14 CST 2023 on pts/1 - + [gpadmin@mdw ~]$ psql # Connects to the database with the default database name "gpadmin". - + # psql (14.4, server 14.4) # Type "help" for help. ``` @@ -73,7 +86,7 @@ You can now connect to the database and try some basic operations. ```sql gpadmin=# SELECT VERSION(); -- Checks the database version. version - + ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ----- PostgreSQL 14.4 (Cloudberry Database 1.0.0 build dev) on aarch64-unknown-linux-gnu, compiled by gcc (GCC) 10.2.1 20210130 (Red Hat 10.2.1-11), 64-bit compiled on Oct 24 2023 10:24:28 diff --git a/000-cbdb-sandbox/configs/90-cbdb-limits.conf b/000-cbdb-sandbox/configs/90-cbdb-limits.conf new file mode 100644 index 0000000..d2bf601 --- /dev/null +++ b/000-cbdb-sandbox/configs/90-cbdb-limits.conf @@ -0,0 +1,10 @@ +###################### +# CBDB CONFIG PARAMS # +###################### + + * soft core unlimited + * hard core unlimited + * soft nofile 524288 + * hard nofile 524288 + * soft nproc 131072 + * hard nproc 131072 diff --git a/000-cbdb-sandbox/configs/sysctl.conf.add b/000-cbdb-sandbox/configs/90-cbdb-sysctl.conf similarity index 97% rename from 000-cbdb-sandbox/configs/sysctl.conf.add rename to 000-cbdb-sandbox/configs/90-cbdb-sysctl.conf index d0457da..ed806a5 100644 --- a/000-cbdb-sandbox/configs/sysctl.conf.add +++ b/000-cbdb-sandbox/configs/90-cbdb-sysctl.conf @@ -1,5 +1,5 @@ ###################### -# HAWQ CONFIG PARAMS # +# CBDB CONFIG PARAMS # ###################### kernel.shmmax = 1000000000 diff --git a/000-cbdb-sandbox/configs/gpinitsystem_singlenode b/000-cbdb-sandbox/configs/gpinitsystem_singlenode index 1416792..7c3e061 100644 --- a/000-cbdb-sandbox/configs/gpinitsystem_singlenode +++ b/000-cbdb-sandbox/configs/gpinitsystem_singlenode @@ -12,8 +12,7 @@ # A name for the array you are configuring. You can use any name you # like. Enclose the name in quotes if the name contains spaces. -ARRAY_NAME="GPDB SANDBOX" - +ARRAY_NAME="Sandbox: Cloudberry Database Cluster" # This specifies the file that contains the list of segment host names # that comprise the Greenplum system. For a single-node system, this @@ -24,7 +23,6 @@ ARRAY_NAME="GPDB SANDBOX" MACHINE_LIST_FILE=/tmp/gpdb-hosts - # This specifies a prefix that will be used to name the data directories # of the master and segment instances. The naming convention for data # directories in a Greenplum Database system is SEG_PREFIX @@ -35,14 +33,12 @@ MACHINE_LIST_FILE=/tmp/gpdb-hosts SEG_PREFIX=gpseg - # Base port number on which primary segment instances will be # started on a segment host. The base port number will be # incremented by one for each segment instance started on a host. PORT_BASE=40000 - # This specifies the data storage location(s) where the script will # create the primary segment data directories. The script creates a # unique data directory for each segment instance. If you want multiple @@ -57,15 +53,14 @@ PORT_BASE=40000 # may want to create these directories on the segment hosts before running # gpinitsystem and chown them to the appropriate user. -declare -a DATA_DIRECTORY=(/data0/database/primary /data0/database/primary) - +declare -a DATA_DIRECTORY=(/data0/database/primary \ + /data0/database/primary \ + /data0/database/primary) # The OS-configured hostname of the Greenplum Database master instance. MASTER_HOSTNAME=mdw - - # The location where the data directory will be created on the # Greenplum master host. # You must make sure that the user who runs gpinitsystem @@ -75,14 +70,12 @@ MASTER_HOSTNAME=mdw MASTER_DIRECTORY=/data0/database/master - # The port number for the master instance. This is the port number # that users and client connections will use when accessing the # Greenplum Database system. MASTER_PORT=5432 - # The shell the gpinitsystem script uses to execute # commands on remote hosts. Allowed value is ssh. You must set up # your trusted host environment before running the gpinitsystem @@ -90,7 +83,6 @@ MASTER_PORT=5432 TRUSTED_SHELL=ssh - # Maximum distance between automatic write ahead log (WAL) # checkpoints, in log file segments (each segment is normally 16 # megabytes). This will set the checkpoint_segments parameter @@ -99,7 +91,6 @@ TRUSTED_SHELL=ssh CHECK_POINT_SEGMENTS=8 - # The character set encoding to use. Greenplum supports the # same character sets as PostgreSQL. See 'Character Set Support' # in the PostgreSQL documentation for allowed character sets. @@ -108,7 +99,6 @@ CHECK_POINT_SEGMENTS=8 ENCODING=UNICODE - ################################################ # OPTIONAL PARAMETERS ################################################ @@ -119,7 +109,13 @@ ENCODING=UNICODE DATABASE_NAME=gpadmin -# MIRROR_PORT_BASE=50000 +# Mirror configuration + +MIRROR_PORT_BASE=50000 + +declare -a MIRROR_DATA_DIRECTORY=(/data0/database/mirror \ + /data0/database/mirror \ + /data0/database/mirror) + # REPLICATION_PORT_BASE=41000 # MIRROR_REPLICATION_PORT_BASE=51000 -# declare -a MIRROR_DATA_DIRECTORY=(/data0/database/mirror /data0/database/mirror) diff --git a/000-cbdb-sandbox/configs/init_system.sh b/000-cbdb-sandbox/configs/init_system.sh old mode 100644 new mode 100755 index 3d86a00..8650ecb --- a/000-cbdb-sandbox/configs/init_system.sh +++ b/000-cbdb-sandbox/configs/init_system.sh @@ -1,15 +1,79 @@ #!/bin/bash +## ====================================================================== +## Container initialization script +## ====================================================================== +## Start SSH daemon and setup for ssh access /usr/sbin/sshd -INSATALL_PATH="/usr/local/cloudberry-db" rm -rf /run/nologin -echo $(grep $(hostname) /etc/hosts | cut -f1) mdw >> /etc/hosts -echo "127.0.0.1 $(cat ~/orig_hostname)" >> /etc/hosts -chown -R gpadmin.gpadmin ${INSATALL_PATH} /tmp/gpinitsystem_singlenode /tmp/gpdb-hosts /tmp/gpdb-master -su gpadmin -l -c "source ${INSATALL_PATH}/greenplum_path.sh;gpssh-exkeys -f /tmp/gpdb-hosts" -su gpadmin -l -c "source ${INSATALL_PATH}/greenplum_path.sh;gpinitsystem -a -c /tmp/gpinitsystem_singlenode -h /tmp/gpdb-hosts --max_connections=100" -su gpadmin -l -c "export COORDINATOR_DATA_DIRECTORY=/data0/database/master/gpseg-1;source ${INSATALL_PATH}/greenplum_path.sh;psql -d template1 -c \"alter user gpadmin password 'cbdb@123'\"" -echo "host all all 0.0.0.0/0 trust" >> /data0/database/master/gpseg-1/pg_hba.conf -su gpadmin -l -c "source ${INSATALL_PATH}/greenplum_path.sh;gpstop -u" -/bin/bash \ No newline at end of file + +echo $(grep $(hostname) /etc/hosts | cut -f1) mdw >> /etc/hosts +echo "127.0.0.1 $(cat ~/orig_hostname)" >> /etc/hosts + +## Set gpadmin ownership - Clouberry install directory and supporting +## cluster creation files. +chown -R gpadmin.gpadmin /usr/local/cloudberry-db \ + /tmp/gpinitsystem_singlenode \ + /tmp/gpdb-hosts + +# Allow passwordless ssh access +su gpadmin -l \ + -c "mkdir -p /home/gpadmin/.ssh; chmod 700 /home/gpadmin/.ssh; \ + ssh-keygen -t rsa -b 4096 -C gpadmin -f /home/gpadmin/.ssh/id_rsa -P \"\" > /dev/null 2>&1; \ + cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys; \ + chmod 600 /home/gpadmin/.ssh/authorized_keys; \ + ssh-keyscan -t rsa mdw > /home/gpadmin/.ssh/known_hosts; \ + ssh mdw uptime" + +cat <<'EOF' + +====================================================================== + ____ _ _ _ ____ ____ + / ___| | ___ _ _ __| | |__ ___ _ __ _ __ _ _ | _ \| __ ) +| | | |/ _ \| | | |/ _` | '_ \ / _ \ '__| '__| | | | | | | | _ \ +| |___| | (_) | |_| | (_| | |_) | __/ | | | | |_| | | |_| | |_) | + \____|_|\___/ \__,_|\__,_|_.__/ \___|_| |_| \__, | |____/|____/ + |___/ +====================================================================== + +EOF + +# Initialize Cloudberry cluster +su gpadmin -l \ + -c "gpinitsystem -a \ + -c /tmp/gpinitsystem_singlenode \ + -h /tmp/gpdb-hosts \ + --max_connections=100" + +## Allow any host access the Cloudberry Cluster +su gpadmin -l \ + -c "echo 'host all all 0.0.0.0/0 trust' >> /data0/database/master/gpseg-1/pg_hba.conf; \ + gpstop -u" + +su gpadmin -l \ + -c "psql -d template1 \ + -c \"ALTER USER gpadmin PASSWORD 'cbdb@123'\"" + +cat <<'EOF' + +====================================================================== +Sandbox: Cloudberry Database Cluster details +====================================================================== + +EOF + +echo "Current time: $(date)" +source /etc/os-release +echo "OS Version: ${NAME} ${VERSION}" + +## Set gpadmin password, display version and cluster configuration +su gpadmin -l \ + -c "psql -d template1 \ + -c \"SELECT VERSION()\"; \ + psql -d template1 \ + -c \"SELECT * FROM gp_segment_configuration\"; \ + psql -d template1 \ + -c \"SHOW optimizer\"" + +/bin/bash diff --git a/000-cbdb-sandbox/configs/limits.conf.add b/000-cbdb-sandbox/configs/limits.conf.add deleted file mode 100644 index 5e31276..0000000 --- a/000-cbdb-sandbox/configs/limits.conf.add +++ /dev/null @@ -1,9 +0,0 @@ -###################### -# HAWQ CONFIG PARAMS # -###################### - -* soft nofile 65536 -* hard nofile 65536 -* soft nproc 131072 -* hard nproc 131072 -* soft core unlimited \ No newline at end of file diff --git a/000-cbdb-sandbox/configs/python-dependencies.txt b/000-cbdb-sandbox/configs/python-dependencies.txt deleted file mode 100644 index 9e63afe..0000000 --- a/000-cbdb-sandbox/configs/python-dependencies.txt +++ /dev/null @@ -1,3 +0,0 @@ -psutil==5.7.0 -pygresql==5.2 -pyyaml==5.3.1 diff --git a/000-cbdb-sandbox/run.sh b/000-cbdb-sandbox/run.sh index 5f3a8a0..eacd1dd 100755 --- a/000-cbdb-sandbox/run.sh +++ b/000-cbdb-sandbox/run.sh @@ -1,5 +1,123 @@ -#! /bin/bash +#!/bin/bash +set -eu -docker build -f cbdb_Dockerfile -t cbdb:centos7 . -#! docker run -ti -d -v /sys/fs/cgroup:/sys/fs/cgroup:ro -p 22:22 -p 5432:5432 -h mdw cbdb:centos8 -docker run -ti -d -v /sys/fs/cgroup:/sys/fs/cgroup:ro -p 122:22 -p 15432:5432 -h mdw cbdb:centos7 +# Default values +DEFAULT_OS_VERSION="centos7" +DEFAULT_TIMEZONE_VAR="Asia/Shanghai" +BUILD_ONLY="false" + +# Use environment variables if set, otherwise use default values +OS_VERSION="${OS_VERSION:-$DEFAULT_OS_VERSION}" +BUILD_ONLY="${BUILD_ONLY:-false}" +CODEBASE_VERSION="${CODEBASE_VERSION:-}" +TIMEZONE_VAR="${TIMEZONE_VAR:-$DEFAULT_TIMEZONE_VAR}" + +# Function to display help message +function usage() { + echo "Usage: $0 [-o ] [-c ] [-b]" + echo " -o OS version (valid values: centos7, rockylinux9; default: $DEFAULT_OS_VERSION, or set via OS_VERSION environment variable)" + echo " -c Codebase version (valid values: main, or determined from release zip file name)" + echo " -t Timezone (default: Asia/Shanghai, or set via TIMEZONE_VAR environment variable)" + echo " -b Build only, do not run the container (default: false, or set via BUILD_ONLY environment variable)" + exit 1 +} + +# Parse command-line options +while getopts "o:c:t:bh" opt; do + case "${opt}" in + o) + OS_VERSION=${OPTARG} + ;; + c) + CODEBASE_VERSION=${OPTARG} + ;; + t) + TIMEZONE_VAR=${OPTARG} + ;; + b) + BUILD_ONLY="true" + ;; + h) + usage + ;; + *) + usage + ;; + esac +done + +# If CODEBASE_VERSION is not specified, determine it from the file name +if [[ -z "$CODEBASE_VERSION" ]]; then + BASE_CODEBASE_FILE=$(ls configs/cloudberrydb-*.zip 2>/dev/null) + + if [[ -z "$BASE_CODEBASE_FILE" ]]; then + echo "Error: No configs/cloudberrydb-*.zip file found and codebase version not specified." + exit 1 + fi + + CODEBASE_FILE=$(basename ${BASE_CODEBASE_FILE}) + + if [[ $CODEBASE_FILE =~ cloudberrydb-([0-9]+\.[0-9]+\.[0-9]+)\.zip ]]; then + CODEBASE_VERSION="${BASH_REMATCH[1]}" + else + echo "Error: Cannot extract version from file name $CODEBASE_FILE" + exit 1 + fi +fi + +# Validate OS_VERSION and map to appropriate Docker image +case "${OS_VERSION}" in + centos7) + OS_DOCKER_IMAGE="centos7" + ;; + rockylinux9) + OS_DOCKER_IMAGE="rockylinux9" + ;; + *) + echo "Invalid OS version: ${OS_VERSION}" + usage + ;; +esac + +# Validate CODEBASE_VERSION +if [[ "${CODEBASE_VERSION}" != "main" && ! "${CODEBASE_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Invalid codebase version: ${CODEBASE_VERSION}" + usage +fi + +# Ensure centos7 is only used with codebase version "release" +if [[ "${OS_VERSION}" == "centos7" && "${CODEBASE_VERSION}" = "main" ]]; then + echo "Error: OS version 'centos7' cannot be used with codebase version 'main'." + usage +fi + +# Build Container +if [[ "${CODEBASE_VERSION}" = "main" ]]; then + DOCKERFILE=Dockerfile.${CODEBASE_VERSION}.${OS_VERSION} + + docker build --file ${DOCKERFILE} \ + --build-arg TIMEZONE_VAR="${TIMEZONE_VAR}" \ + --tag cbdb-${CODEBASE_VERSION}:${OS_VERSION} . +else + DOCKERFILE=Dockerfile.RELEASE.${OS_VERSION} + + docker build --file ${DOCKERFILE} \ + --build-arg TIMEZONE_VAR="${TIMEZONE_VAR}" \ + --build-arg CODEBASE_VERSION_VAR="${CODEBASE_VERSION}" \ + --tag cbdb-${CODEBASE_VERSION}:${OS_VERSION} . +fi + +# Check if build only flag is set +if [ "${BUILD_ONLY}" == "true" ]; then + echo "Docker image built successfully with OS version ${OS_VERSION} and codebase version ${CODEBASE_VERSION}. Build only mode, not running the container." + exit 0 +fi + +docker run --interactive \ + --tty \ + --detach \ + --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ + --publish 122:22 \ + --publish 15432:5432 \ + --hostname mdw \ + cbdb-${CODEBASE_VERSION}:${OS_VERSION} From 1457648f5c604237435519ab71e16a9fd27c62ce Mon Sep 17 00:00:00 2001 From: Ed Espino Date: Sat, 22 Jun 2024 14:26:47 -0700 Subject: [PATCH 2/3] Add enhanced 1.5.x release support * Install go package for 1.5.3 release support * Add support for alternate location of python-dependencies.txt * Split out configure, make and make install commands into separate RUN sections. --- 000-cbdb-sandbox/Dockerfile.RELEASE.centos7 | 21 ++++++++++++++++---- 000-cbdb-sandbox/Dockerfile.main.rockylinux9 | 8 ++++++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/000-cbdb-sandbox/Dockerfile.RELEASE.centos7 b/000-cbdb-sandbox/Dockerfile.RELEASE.centos7 index 9299f73..e07c53e 100644 --- a/000-cbdb-sandbox/Dockerfile.RELEASE.centos7 +++ b/000-cbdb-sandbox/Dockerfile.RELEASE.centos7 @@ -18,7 +18,7 @@ COPY ./configs/* /tmp/ RUN echo root:cbdb@123 | chpasswd \ && yum install -y epel-release \ && yum install -y initscripts unzip which tar less net-tools util-linux-ng passwd openssh-clients openssh-server perl ed m4 sudo rsync git wget \ - && yum install -y apr-devel bison bzip2-devel cmake3 flex gcc gcc-c++ krb5-devel libcurl-devel libevent-devel libkadm5 libxml2-devel libyaml-devel libzstd-devel openssl-devel perl-ExtUtils-Embed readline-devel xerces-c-devel zlib-devel \ + && yum install -y apr-devel bison bzip2-devel cmake3 flex gcc gcc-c++ go krb5-devel libcurl-devel libevent-devel libkadm5 libxml2-devel libyaml-devel libzstd-devel openssl-devel perl-ExtUtils-Embed readline-devel xerces-c-devel zlib-devel \ && yum install -y postgresql postgresql-devel \ && yum install -y centos-release-scl scl-utils \ && yum install -y devtoolset-10 @@ -54,12 +54,25 @@ RUN yum install -y python3-devel python3-pip; yum clean all RUN cd /tmp/ \ && unzip -d /tmp /tmp/cloudberrydb-${CODEBASE_VERSION_VAR}.zip \ && cd /tmp/cloudberrydb-${CODEBASE_VERSION_VAR} \ - && pip3 install -i https://mirrors.aliyun.com/pypi/simple -r python-dependencies.txt + && if [ -f python-dependencies.txt ]; then \ + pip3 install -i https://mirrors.aliyun.com/pypi/simple -r python-dependencies.txt; \ + elif [ -f readmes/python-dependencies.txt ]; then \ + pip3 install -i https://mirrors.aliyun.com/pypi/simple -r readmes/python-dependencies.txt; \ + else \ + echo "File does not exist, skipping additional commands"; \ + exit 2; \ + fi + +RUN cd /tmp/cloudberrydb-${CODEBASE_VERSION_VAR} \ + && source scl_source enable devtoolset-10 \ + && ./configure --with-perl --with-python --with-libxml --with-gssapi --prefix=/usr/local/cloudberry-db + +RUN cd /tmp/cloudberrydb-${CODEBASE_VERSION_VAR} \ + && source scl_source enable devtoolset-10 \ + && make -j$(nproc) RUN cd /tmp/cloudberrydb-${CODEBASE_VERSION_VAR} \ && source scl_source enable devtoolset-10 \ - && ./configure --with-perl --with-python --with-libxml --with-gssapi --prefix=/usr/local/cloudberry-db \ - && make -j8 \ && make install EXPOSE 5432 22 diff --git a/000-cbdb-sandbox/Dockerfile.main.rockylinux9 b/000-cbdb-sandbox/Dockerfile.main.rockylinux9 index 71ba517..2acb2f2 100644 --- a/000-cbdb-sandbox/Dockerfile.main.rockylinux9 +++ b/000-cbdb-sandbox/Dockerfile.main.rockylinux9 @@ -122,8 +122,12 @@ RUN cd /tmp/cloudberrydb && \ --with-perl \ --with-pgport=5432 \ --with-python \ - --with-pythonsrc-ext && \ - make -j$(nproc) && \ + --with-pythonsrc-ext + +RUN cd /tmp/cloudberrydb && \ + make -j$(nproc) + +RUN cd /tmp/cloudberrydb && \ make install EXPOSE 5432 22 From 7f52bd9d53c514b20da496bb61643275fef81969 Mon Sep 17 00:00:00 2001 From: Ed Espino Date: Wed, 26 Jun 2024 09:52:41 -0700 Subject: [PATCH 3/3] Parameterize which PIP Index to use. The pip install will default to using the default PIP Index (https://pypi.org/simple). A user can supply an alternate by using the run.sh `-p` option or set the PIP_INDEX_URL_VAR environment variable. Here is the run.sh -p option output: ``` -p Python Package Index (PyPI) (default: https://pypi.org/simple, or set via PIP_INDEX_URL_VAR environment variable) ``` --- 000-cbdb-sandbox/Dockerfile.RELEASE.centos7 | 5 +++-- 000-cbdb-sandbox/run.sh | 9 ++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/000-cbdb-sandbox/Dockerfile.RELEASE.centos7 b/000-cbdb-sandbox/Dockerfile.RELEASE.centos7 index e07c53e..e38da44 100644 --- a/000-cbdb-sandbox/Dockerfile.RELEASE.centos7 +++ b/000-cbdb-sandbox/Dockerfile.RELEASE.centos7 @@ -3,6 +3,7 @@ FROM centos:7.9.2009 # Define a build argument with a default value ARG CODEBASE_VERSION_VAR=${CODEBASE_VERSION_VAR} ARG TIMEZONE_VAR="Asia/Shanghai" +ARG PIP_INDEX_URL_VAR="https://pypi.org/simple" RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == \ systemd-tmpfiles-setup.service ] || rm -f $i; done); \ @@ -55,9 +56,9 @@ RUN cd /tmp/ \ && unzip -d /tmp /tmp/cloudberrydb-${CODEBASE_VERSION_VAR}.zip \ && cd /tmp/cloudberrydb-${CODEBASE_VERSION_VAR} \ && if [ -f python-dependencies.txt ]; then \ - pip3 install -i https://mirrors.aliyun.com/pypi/simple -r python-dependencies.txt; \ + pip3 install -i ${PIP_INDEX_URL_VAR} -r python-dependencies.txt; \ elif [ -f readmes/python-dependencies.txt ]; then \ - pip3 install -i https://mirrors.aliyun.com/pypi/simple -r readmes/python-dependencies.txt; \ + pip3 install -i ${PIP_INDEX_URL_VAR} -r readmes/python-dependencies.txt; \ else \ echo "File does not exist, skipping additional commands"; \ exit 2; \ diff --git a/000-cbdb-sandbox/run.sh b/000-cbdb-sandbox/run.sh index eacd1dd..6b77bd4 100755 --- a/000-cbdb-sandbox/run.sh +++ b/000-cbdb-sandbox/run.sh @@ -4,6 +4,7 @@ set -eu # Default values DEFAULT_OS_VERSION="centos7" DEFAULT_TIMEZONE_VAR="Asia/Shanghai" +DEFAULT_PIP_INDEX_URL_VAR="https://pypi.org/simple" BUILD_ONLY="false" # Use environment variables if set, otherwise use default values @@ -11,6 +12,7 @@ OS_VERSION="${OS_VERSION:-$DEFAULT_OS_VERSION}" BUILD_ONLY="${BUILD_ONLY:-false}" CODEBASE_VERSION="${CODEBASE_VERSION:-}" TIMEZONE_VAR="${TIMEZONE_VAR:-$DEFAULT_TIMEZONE_VAR}" +PIP_INDEX_URL_VAR="${PIP_INDEX_URL_VAR:-$DEFAULT_PIP_INDEX_URL_VAR}" # Function to display help message function usage() { @@ -18,12 +20,13 @@ function usage() { echo " -o OS version (valid values: centos7, rockylinux9; default: $DEFAULT_OS_VERSION, or set via OS_VERSION environment variable)" echo " -c Codebase version (valid values: main, or determined from release zip file name)" echo " -t Timezone (default: Asia/Shanghai, or set via TIMEZONE_VAR environment variable)" + echo " -p Python Package Index (PyPI) (default: https://pypi.org/simple, or set via PIP_INDEX_URL_VAR environment variable)" echo " -b Build only, do not run the container (default: false, or set via BUILD_ONLY environment variable)" exit 1 } # Parse command-line options -while getopts "o:c:t:bh" opt; do +while getopts "o:c:t:p:bh" opt; do case "${opt}" in o) OS_VERSION=${OPTARG} @@ -34,6 +37,9 @@ while getopts "o:c:t:bh" opt; do t) TIMEZONE_VAR=${OPTARG} ;; + p) + PIP_INDEX_URL_VAR=${OPTARG} + ;; b) BUILD_ONLY="true" ;; @@ -103,6 +109,7 @@ else docker build --file ${DOCKERFILE} \ --build-arg TIMEZONE_VAR="${TIMEZONE_VAR}" \ + --build-arg PIP_INDEX_URL_VAR="${PIP_INDEX_URL_VAR}" \ --build-arg CODEBASE_VERSION_VAR="${CODEBASE_VERSION}" \ --tag cbdb-${CODEBASE_VERSION}:${OS_VERSION} . fi