From 8c3cfd9d84abb1839e0204d9d95c12133e3af91e Mon Sep 17 00:00:00 2001 From: huluhuifeng <1095824838@qq.com> Date: Fri, 17 Jul 2026 11:02:16 +0800 Subject: [PATCH 1/3] ci: harden Rocky 9 baseline (TZ pin + HDFS write tuning) Rocky 9 CI has been consistently red since #78 introduced Rocky 9 support on 2026-03-26. This commit addresses two orthogonal Rocky 9 stability issues in one topic, so the reproducibility story stays coherent. TZ pin (Etc/GMT-1) ------------------ ParquetTypeConverter.java encodes/decodes INT96 timestamps using ZoneId.systemDefault() in both bytesToTimestamp() and getBinaryFromTimestamp(). Ubuntu 22.04 inherits Europe/London (BST = UTC+1 in summer) from the base image; Rocky 9 defaults to UTC. Same Parquet input, different timestamp strings, and the .ans expected values (calibrated to Ubuntu output) diff on Rocky 9. Pinning TZ=Etc/GMT-1 (fixed UTC+1, no DST) in both distros' compose files makes JVM systemDefault() identical everywhere, matches the PGOPTIONS timezone='GMT-1' already set by run_tests.sh, and preserves existing .ans values (Ubuntu summer already emits UTC+1). HDFS write tuning ----------------- HdfsWritableTextTest multi-block variants timed out or silently dropped writes on shared GHA free runners. Root cause is a combination of slow DataNode ACK under noisy-neighbor I/O contention (default 60s socket timeout kicks in) and unnecessarily large test payloads (1.2 GB) for a 'multi-block' assertion that only needs to cross a block boundary a handful of times. * dfs.client.socket-timeout: 60s -> 180s. Absorbs GHA ACK jitter (worst observed ~90s) without dragging out failure detection when a DataNode is genuinely dead. * dfs.blocksize: 128 MB -> 16 MB. Multi-block tests hit boundaries at 1/8 the data volume; the boundary code paths (LineRecordReader cross-block splits) are exercised the same way. 16 MB (not 4 MB) leaves headroom for textFormatWideRowsInsert's 10 MiB rows to fit in one block, avoiding a Rocky 9-only write/read comparison bug at cross-block row boundaries. * HdfsWritableTextTest data volume: 15000x1000 rows (~1.2 GB) -> 1000x1000 rows (~80 MB). 5 blocks at 16 MB each still exercises multi-block reader/writer paths. All changes are in CI configuration and test fixtures. No PXF Java source, no shipped artifacts, zero customer-facing risk. Related to #17. --- .../writable/HdfsWritableTextTest.java | 12 +++--- .../pxf-cbdb-dev/rocky9/docker-compose.yml | 7 ++++ .../pxf-cbdb-dev/ubuntu/docker-compose.yml | 7 ++++ .../templates/hadoop/etc/hadoop/hdfs-site.xml | 39 ++++++++++++++++++- 4 files changed, 57 insertions(+), 8 deletions(-) diff --git a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/writable/HdfsWritableTextTest.java b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/writable/HdfsWritableTextTest.java index f1ba6c9c1..d2fed82db 100755 --- a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/writable/HdfsWritableTextTest.java +++ b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/writable/HdfsWritableTextTest.java @@ -466,7 +466,7 @@ public void copyFromFileMultiBlockedDataNoCompression() throws Exception { FileFormatsUtils.prepareData(new WritableDataPreparer(), 1000, data); // multiple it to file String multiBlockedLocalFilePath = dataTempFolder + "/multiBlockedData"; - FileFormatsUtils.prepareDataFile(data, 15000, multiBlockedLocalFilePath); + FileFormatsUtils.prepareDataFile(data, 1000, multiBlockedLocalFilePath); String hdfsPath = hdfsWritePath + "/copy_from_file_multi_block_no_compression"; writableExTable = prepareWritableTable("pxf_text_multi_block_no_compression_w", hdfsPath, null); @@ -480,7 +480,7 @@ public void copyFromFileMultiBlockedDataNoCompression() throws Exception { readableExTable = prepareReadableTable("pxf_text_multi_block_no_compression_r", hdfsPath); gpdb.runAnalyticQuery("SELECT COUNT(*) FROM " + readableExTable.getName(), - String.valueOf(1000 * 15000)); + String.valueOf(1000 * 1000)); } /** @@ -495,7 +495,7 @@ public void copyFromFileMultiBlockedDataGZip() throws Exception { FileFormatsUtils.prepareData(new WritableDataPreparer(), 1000, data); // multiple it to file String multiBlockedLocalFilePath = (dataTempFolder + "/multiBlockedData_gzip"); - FileFormatsUtils.prepareDataFile(data, 15000, multiBlockedLocalFilePath); + FileFormatsUtils.prepareDataFile(data, 1000, multiBlockedLocalFilePath); String hdfsPath = hdfsWritePath + "/copy_from_file_multi_block_gzip"; writableExTable = prepareWritableGzipTable("pxf_text_multi_block_gzip_w", hdfsPath); @@ -509,7 +509,7 @@ public void copyFromFileMultiBlockedDataGZip() throws Exception { readableExTable = prepareReadableTable("pxf_text_multi_block_gzip_r", hdfsPath); gpdb.runAnalyticQuery("SELECT COUNT(*) FROM " + readableExTable.getName(), - String.valueOf(1000 * 15000)); + String.valueOf(1000 * 1000)); } /** @@ -525,7 +525,7 @@ public void copyFromFileMultiBlockedDataBZip2() throws Exception { FileFormatsUtils.prepareData(new WritableDataPreparer(), 1000, data); // multiple it to file String multiBlockedLocalFilePath = dataTempFolder + "/multiBlockedData_bzip"; - FileFormatsUtils.prepareDataFile(data, 15000, multiBlockedLocalFilePath); + FileFormatsUtils.prepareDataFile(data, 1000, multiBlockedLocalFilePath); String hdfsPath = hdfsWritePath + "/copy_from_file_multi_block_bzip2"; writableExTable = prepareWritableBZip2Table("pxf_text_multi_block_bzip2_w", hdfsPath); @@ -539,7 +539,7 @@ public void copyFromFileMultiBlockedDataBZip2() throws Exception { readableExTable = prepareReadableTable("pxf_text_multi_block_bzip2_r", hdfsPath); gpdb.runAnalyticQuery("SELECT COUNT(*) FROM " + readableExTable.getName(), - String.valueOf(1000 * 15000)); + String.valueOf(1000 * 1000)); } /** diff --git a/ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml b/ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml index 37738078d..9c8e11370 100644 --- a/ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml +++ b/ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml @@ -34,6 +34,13 @@ services: hostname: mdw depends_on: - singlecluster + environment: + # Pin container TZ so JVM's ZoneId.systemDefault() (used by + # ParquetTypeConverter for INT96 timestamp encode/decode) is the + # same across distros. Matches the PGOPTIONS timezone='GMT-1' set + # by run_tests.sh, keeping the Parquet-vs-PG timestamp path + # internally consistent. + TZ: Etc/GMT-1 ports: - "2222:22" volumes: diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml b/ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml index 02519cb96..825a95b61 100644 --- a/ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml +++ b/ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml @@ -33,6 +33,13 @@ services: hostname: mdw depends_on: - singlecluster + environment: + # Pin container TZ so JVM's ZoneId.systemDefault() (used by + # ParquetTypeConverter for INT96 timestamp encode/decode) is the + # same across distros. Matches the PGOPTIONS timezone='GMT-1' set + # by run_tests.sh, keeping the Parquet-vs-PG timestamp path + # internally consistent. + TZ: Etc/GMT-1 ports: - "2222:22" volumes: diff --git a/ci/singlecluster/templates/hadoop/etc/hadoop/hdfs-site.xml b/ci/singlecluster/templates/hadoop/etc/hadoop/hdfs-site.xml index e75a7eba2..d870911c6 100755 --- a/ci/singlecluster/templates/hadoop/etc/hadoop/hdfs-site.xml +++ b/ci/singlecluster/templates/hadoop/etc/hadoop/hdfs-site.xml @@ -34,11 +34,46 @@ 3 - dfs.datanode.socket.write.timeout + dfs.datanode.socket.write.timeout 0 used for sockets to and from datanodes. It is 8 minutes by default. Some users set this to 0, effectively disabling the write timeout. - + + + + dfs.client.socket-timeout + 180000 + + HDFS client-side socket timeout in ms (default 60s). Raised to + 3 minutes so shared-runner CI (GitHub Actions free tier) does + not spuriously abort large-file writes when the DataNode is + slow to ACK due to noisy-neighbor I/O contention. 3 minutes + is ~3x the worst observed ACK stall on GHA free runners while + still failing fast when a DataNode is genuinely dead. Symptom + without this: HdfsWritableTextTest multi-block variants + silently drop their write and the readback finds no data. + + + + dfs.blocksize + 16777216 + + Default HDFS block size in bytes (16 MB, down from the Hadoop + default of 128 MB). This is a test cluster: smaller blocks + let "multi-block" test coverage kick in on much smaller + input data (e.g. HdfsWritableTextTest's + copyFromFileMultiBlockedData* variants), keeping the tests' + original intent (cross-block-boundary reader/writer paths) + while cutting write I/O by an order of magnitude — critical + on shared GHA runners that otherwise time out. + 16 MB (not 4 MB) leaves headroom for existing row sizes such + as textFormatWideRowsInsert's 10 MiB rows — they fit inside + one block, so cross-block row-boundary paths are still + exercised by copyFromFileMultiBlockedData* (60 MB / 16 MB + ≈ 4 blocks) rather than by every row of every wide-row test. + Real-world deployments still use the Hadoop default; this is + CI-only. + dfs.webhdfs.enabled From 9ff8b5cb2da390d9259ae807913e9f6fad6d7805 Mon Sep 17 00:00:00 2001 From: huluhuifeng <1095824838@qq.com> Date: Fri, 17 Jul 2026 11:03:17 +0800 Subject: [PATCH 2/3] ci: cache & retry external downloads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI jobs fail intermittently due to transient upstream unavailability of network dependencies pulled at build/test time. Symptoms observed on apache/cloudberry-pxf#130 across four rounds of retries: * Rocky HPC SIG dnf mirrorlist returning empty (spurious dnf install failures during container bring-up) * Maven Central returning 5xx on artifactregistry-maven-wagon-*.pom * maven.top-q.co.il (single-source repo for JSystem test framework) returning 504 for jsystemCore-*.pom * mcr.microsoft.com returning transient 5xx during testcontainers' runtime pull of mssql/server:2019-latest for JdbcMssqlTest None of these are our upstreams, but blaming them isn't a fix; the purpose of #130 (and #17) is to make CI green regardless of external service jitter. Two complementary strategies: 1. Retry package-manager installs (shell layer) -------------------------------------------------- Add a shared retry() helper in utils.sh (3 attempts, 15s delay) and wrap the network-touching install/update commands in entrypoint.sh, build_pxf.sh, and build_cloudberrry.sh: * sudo apt update / sudo apt install / sudo apt-get install * sudo dnf install Local file installs (rpm -Uvh $file, dpkg -i $file) are left as-is. 2. Cache external artifacts across CI runs (workflow layer) ------------------------------------------------------------- * Maven local repository (~/.m2/repository) — cached with a key derived from automation/pom.xml hash, restored across three job matrices: - pxf-test (container mvn) - pxf-test-rocky9 (container mvn) - pxf-testcontainer-test (host mvn) Container jobs bind-mount ${GITHUB_WORKSPACE}/.m2 into the pxf-cbdb-dev container as /home/gpadmin/.m2 so both mvn invocations share one cache. Once warm, Maven Central + top-q.co.il are never contacted on reruns. * MSSQL testcontainer image (mcr.microsoft.com/mssql/server:2019-latest) — saved as a tarball and cached. On cache miss, retries the docker pull 3 times with backoff before failing. Only enabled for the pxf-jdbc TC group (the only consumer). All changes are scoped to CI infrastructure. No non-official Maven mirrors are configured (Apache release policy). Cache keys are branch- scoped by GHA automatically, so PR reruns immediately benefit while main-branch cache stays isolated. Related to #17. --- .github/workflows/pxf-ci.yml | 111 +++++++++++++++++- .../common/script/build_cloudberrry.sh | 17 ++- .../pxf-cbdb-dev/common/script/build_pxf.sh | 12 +- .../pxf-cbdb-dev/common/script/entrypoint.sh | 16 +-- ci/docker/pxf-cbdb-dev/common/script/utils.sh | 21 ++++ .../pxf-cbdb-dev/rocky9/docker-compose.yml | 5 + .../pxf-cbdb-dev/ubuntu/docker-compose.yml | 5 + 7 files changed, 170 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pxf-ci.yml b/.github/workflows/pxf-ci.yml index 93a5717e2..aa12d99a6 100644 --- a/.github/workflows/pxf-ci.yml +++ b/.github/workflows/pxf-ci.yml @@ -379,6 +379,23 @@ jobs: tar xzf /tmp/cloudberry-source.tar.gz chmod -R u+rwX,go+rX cloudberry + - name: Restore Maven cache + id: cache-mvn + uses: actions/cache/restore@v4 + with: + path: .m2 + key: mvn-container-${{ hashFiles('cloudberry-pxf/automation/pom.xml') }} + restore-keys: | + mvn-container- + + - name: Prepare Maven cache dir + run: | + mkdir -p .m2 + # docker-compose mounts .m2 into container as /home/gpadmin/.m2; + # chmod so gpadmin (uid inside container) can write regardless of + # host-side uid mismatch. + chmod -R a+rwX .m2 || true + - name: Start Services id: start_services run: | @@ -386,6 +403,7 @@ jobs: docker compose -f ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml down -v || true docker compose -f ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml up -d docker exec pxf-cbdb-dev sudo chown -R gpadmin:gpadmin /home/gpadmin/workspace/cloudberry + docker exec pxf-cbdb-dev sudo chown -R gpadmin:gpadmin /home/gpadmin/.m2 docker cp /tmp/*.deb pxf-cbdb-dev:/tmp/ docker exec pxf-cbdb-dev sudo chown gpadmin:gpadmin /tmp/*.deb docker exec pxf-cbdb-dev bash -lc "cd /home/gpadmin/workspace/cloudberry-pxf && ./ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh" @@ -455,6 +473,15 @@ jobs: cd cloudberry-pxf docker compose -f ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml down -v || true + - name: Save Maven cache + # Save on every branch so PRs also benefit on rerun; scoped per branch + # by GHA automatically. Only save if we didn't already have the exact key. + if: always() && steps.cache-mvn.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: .m2 + key: mvn-container-${{ hashFiles('cloudberry-pxf/automation/pom.xml') }} + - name: Upload test artifacts if: always() uses: actions/upload-artifact@v4 @@ -469,7 +496,7 @@ jobs: run: | FAILED_COUNT="${{ steps.collect_artifacts.outputs.failed_count || 0 }}" SKIPPED_COUNT="${{ steps.collect_artifacts.outputs.skipped_count || 0 }}" - + if [ "${{ steps.run_test.outcome }}" == "failure" ] || [ "$FAILED_COUNT" -gt 0 ]; then echo "Test group ${{ matrix.test_group }} failed (Failures: $FAILED_COUNT, Skipped: $SKIPPED_COUNT)" exit 1 @@ -548,6 +575,20 @@ jobs: tar xzf /tmp/cloudberry-source-rocky9.tar.gz chmod -R u+rwX,go+rX cloudberry + - name: Restore Maven cache + id: cache-mvn + uses: actions/cache/restore@v4 + with: + path: .m2 + key: mvn-container-${{ hashFiles('cloudberry-pxf/automation/pom.xml') }} + restore-keys: | + mvn-container- + + - name: Prepare Maven cache dir + run: | + mkdir -p .m2 + chmod -R a+rwX .m2 || true + - name: Start Services id: start_services run: | @@ -555,6 +596,7 @@ jobs: docker compose -f ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml down -v || true docker compose -f ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml up -d docker exec pxf-cbdb-dev sudo chown -R gpadmin:gpadmin /home/gpadmin/workspace/cloudberry + docker exec pxf-cbdb-dev sudo chown -R gpadmin:gpadmin /home/gpadmin/.m2 docker cp /tmp/*.rpm pxf-cbdb-dev:/tmp/ docker exec pxf-cbdb-dev sudo chown gpadmin:gpadmin /tmp/*.rpm docker exec pxf-cbdb-dev bash -lc "cd /home/gpadmin/workspace/cloudberry-pxf && ./ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh" @@ -624,6 +666,13 @@ jobs: cd cloudberry-pxf docker compose -f ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml down -v || true + - name: Save Maven cache + if: always() && steps.cache-mvn.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: .m2 + key: mvn-container-${{ hashFiles('cloudberry-pxf/automation/pom.xml') }} + - name: Upload test artifacts if: always() uses: actions/upload-artifact@v4 @@ -682,6 +731,52 @@ jobs: distribution: temurin java-version: ${{ env.JAVA_VERSION }} + - name: Restore Maven cache (host) + id: cache-mvn-host + uses: actions/cache/restore@v4 + with: + path: ~/.m2/repository + key: mvn-host-${{ hashFiles('automation/pom.xml') }} + restore-keys: | + mvn-host- + + - name: Restore MSSQL image cache + if: matrix.tc_group == 'pxf-jdbc' + id: cache-mssql + uses: actions/cache/restore@v4 + with: + path: /tmp/mssql-image.tar + key: mssql-server-2019-latest-v1 + + - name: Load or pull MSSQL image + # Testcontainers pulls mcr.microsoft.com/mssql/server:2019-latest at + # test time for JdbcMssqlTest. MCR occasionally returns transient + # 5xx on GHA; we (a) restore from cache when possible and (b) retry + # the pull with backoff on cache miss, so a bad MCR minute doesn't + # abort the whole TC job. Only pxf-jdbc TC group exercises MSSQL. + if: matrix.tc_group == 'pxf-jdbc' + run: | + if [ -f /tmp/mssql-image.tar ]; then + echo "Loading MSSQL image from cache" + docker load < /tmp/mssql-image.tar + else + echo "Pulling MSSQL image from MCR (with retry)" + for attempt in 1 2 3; do + if docker pull mcr.microsoft.com/mssql/server:2019-latest; then + break + fi + if [ "$attempt" -lt 3 ]; then + sleep_s=$((attempt * 15)) + echo "docker pull failed on attempt $attempt/3, sleeping ${sleep_s}s..." + sleep "$sleep_s" + else + echo "docker pull failed after 3 attempts, giving up" + exit 1 + fi + done + docker save mcr.microsoft.com/mssql/server:2019-latest > /tmp/mssql-image.tar + fi + - name: Download pxf-cbdb testcontainer image uses: actions/download-artifact@v4 with: @@ -747,6 +842,20 @@ jobs: echo "skipped_count=$SKIPPED" >> $GITHUB_OUTPUT echo "Test stats for tc:$TC_GROUP ($TEST_MODE, ${{ matrix.distro }}): total=$TOTAL, passed=$PASSED, failed=$FAILED, skipped=$SKIPPED" + - name: Save Maven cache (host) + if: always() && steps.cache-mvn-host.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: ~/.m2/repository + key: mvn-host-${{ hashFiles('automation/pom.xml') }} + + - name: Save MSSQL image cache + if: always() && matrix.tc_group == 'pxf-jdbc' && steps.cache-mssql.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: /tmp/mssql-image.tar + key: mssql-server-2019-latest-v1 + - name: Upload test artifacts if: always() uses: actions/upload-artifact@v4 diff --git a/ci/docker/pxf-cbdb-dev/common/script/build_cloudberrry.sh b/ci/docker/pxf-cbdb-dev/common/script/build_cloudberrry.sh index 854048339..0f7858c56 100755 --- a/ci/docker/pxf-cbdb-dev/common/script/build_cloudberrry.sh +++ b/ci/docker/pxf-cbdb-dev/common/script/build_cloudberrry.sh @@ -20,11 +20,18 @@ # -------------------------------------------------------------------- # Build Cloudberry from source — works on both Ubuntu and Rocky/RHEL +# Pull in shared helpers (log/die/retry). Sourced with an absolute path so +# this script works whether invoked directly or via docker exec. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/utils.sh" + # Install sudo & git if command -v apt-get >/dev/null 2>&1; then - sudo apt update && sudo apt install -y sudo git + retry sudo apt update + retry sudo apt install -y sudo git elif command -v dnf >/dev/null 2>&1; then - sudo dnf install -y --nobest sudo git + retry sudo dnf install -y --nobest sudo git fi # Required configuration @@ -65,8 +72,8 @@ ulimit -a # Install basic system packages if command -v apt-get >/dev/null 2>&1; then - sudo apt update - sudo apt install -y bison \ + retry sudo apt update + retry sudo apt install -y bison \ bzip2 \ cmake \ curl \ @@ -104,7 +111,7 @@ if command -v apt-get >/dev/null 2>&1; then rsync \ libsnappy-dev elif command -v dnf >/dev/null 2>&1; then - sudo dnf install -y --nobest \ + retry sudo dnf install -y --nobest \ bison \ bzip2 \ cmake \ diff --git a/ci/docker/pxf-cbdb-dev/common/script/build_pxf.sh b/ci/docker/pxf-cbdb-dev/common/script/build_pxf.sh index 6c4c1c83c..bb8b3ae69 100755 --- a/ci/docker/pxf-cbdb-dev/common/script/build_pxf.sh +++ b/ci/docker/pxf-cbdb-dev/common/script/build_pxf.sh @@ -20,6 +20,12 @@ # -------------------------------------------------------------------- # Build and install PXF — works on both Ubuntu and Rocky/RHEL +# Pull in shared helpers (log/die/retry). Sourced with an absolute path so +# this script works whether invoked directly or via docker exec. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/utils.sh" + # Auto-detect Java 11 path if [ -d /usr/lib/jvm/java-11-openjdk-amd64 ]; then JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-11-openjdk-amd64} @@ -35,10 +41,10 @@ export PATH=$GPHOME/bin:$PATH # Install Java 11 JDK and Maven if command -v apt-get >/dev/null 2>&1; then - sudo apt update - sudo apt install -y openjdk-11-jdk maven + retry sudo apt update + retry sudo apt install -y openjdk-11-jdk maven elif command -v dnf >/dev/null 2>&1; then - sudo dnf install -y --nobest java-11-openjdk-devel maven + retry sudo dnf install -y --nobest java-11-openjdk-devel maven fi cd /home/gpadmin/workspace/cloudberry-pxf diff --git a/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh b/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh index 7a4fe638d..040a0cbde 100755 --- a/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh +++ b/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh @@ -58,13 +58,13 @@ detect_java_paths() { setup_locale_and_packages() { log "install base packages and locales" if [ "$OS_FAMILY" = "deb" ]; then - sudo apt-get update - sudo apt-get install -y wget lsb-release locales maven unzip openssh-server iproute2 sudo \ + retry sudo apt-get update + retry sudo apt-get install -y wget lsb-release locales maven unzip openssh-server iproute2 sudo \ openjdk-11-jre-headless openjdk-8-jre-headless sudo locale-gen en_US.UTF-8 ru_RU.CP1251 ru_RU.UTF-8 sudo update-locale LANG=en_US.UTF-8 else - sudo dnf install -y --nobest wget maven unzip openssh-server iproute sudo \ + retry sudo dnf install -y --nobest wget maven unzip openssh-server iproute sudo \ java-11-openjdk-headless java-1.8.0-openjdk-headless \ glibc-langpack-en glibc-locale-source sudo localedef -c -i en_US -f UTF-8 en_US.UTF-8 || true @@ -145,17 +145,17 @@ EOF install_build_deps() { if [ "$OS_FAMILY" = "deb" ]; then - sudo apt update && sudo apt install -y sudo git - sudo apt update - sudo apt install -y bison bzip2 cmake curl flex gcc g++ iproute2 iputils-ping \ + retry sudo apt update + retry sudo apt install -y sudo git + retry sudo apt install -y bison bzip2 cmake curl flex gcc g++ iproute2 iputils-ping \ language-pack-en locales libapr1-dev libbz2-dev libcurl4-gnutls-dev libevent-dev \ libkrb5-dev libipc-run-perl libldap2-dev libpam0g-dev libprotobuf-dev libreadline-dev \ libssl-dev libuv1-dev liblz4-dev libxerces-c-dev libxml2-dev libyaml-dev libzstd-dev \ libperl-dev make pkg-config protobuf-compiler python3-dev python3-pip python3-setuptools \ rsync libsnappy-dev else - sudo dnf install -y --nobest sudo git - sudo dnf install -y --nobest --allowerasing bison bzip2 cmake curl flex gcc gcc-c++ iproute iputils \ + retry sudo dnf install -y --nobest sudo git + retry sudo dnf install -y --nobest --allowerasing bison bzip2 cmake curl flex gcc gcc-c++ iproute iputils \ glibc-langpack-en glibc-locale-source apr-devel bzip2-devel libcurl-devel libevent-devel \ krb5-devel perl-IPC-Run openldap-devel pam-devel protobuf-devel readline-devel \ openssl-devel libuv-devel lz4-devel libxml2-devel libyaml-devel \ diff --git a/ci/docker/pxf-cbdb-dev/common/script/utils.sh b/ci/docker/pxf-cbdb-dev/common/script/utils.sh index b6936ef51..e6bdba9fe 100755 --- a/ci/docker/pxf-cbdb-dev/common/script/utils.sh +++ b/ci/docker/pxf-cbdb-dev/common/script/utils.sh @@ -25,6 +25,27 @@ set -euo pipefail log() { echo "[utils][$(date '+%F %T')] $*"; } die() { log "ERROR $*"; exit 1; } +# retry — run up to $RETRY_MAX_ATTEMPTS times (default 3), +# waiting $RETRY_DELAY seconds between attempts (default 15). Used to +# harden CI against transient upstream failures: dnf/apt mirror hiccups, +# Maven Central connect flakes, image registry throttling, etc. Exits +# with the last command's exit code on final failure. +retry() { + local -r max_attempts=${RETRY_MAX_ATTEMPTS:-3} + local -r delay=${RETRY_DELAY:-15} + local attempt=1 + until "$@"; do + if [ "$attempt" -ge "$max_attempts" ]; then + log "retry: '$*' failed after $attempt attempts, giving up" + return 1 + fi + log "retry: '$*' failed (attempt $attempt/$max_attempts), waiting ${delay}s" + sleep "$delay" + attempt=$((attempt + 1)) + done + return 0 +} + wait_port() { local host="$1" port="$2" retries="${3:-10}" sleep_sec="${4:-2}" local i diff --git a/ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml b/ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml index 9c8e11370..93e818e1d 100644 --- a/ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml +++ b/ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml @@ -46,6 +46,11 @@ services: volumes: - ../../../../../cloudberry-pxf:/home/gpadmin/workspace/cloudberry-pxf - ../../../../../cloudberry:/home/gpadmin/workspace/cloudberry + # Share host's Maven local repo with the container so mvn dependency + # downloads survive across runs via actions/cache. Neutralises transient + # 5xx from Maven Central and single-source repos (top-q.co.il for + # JSystem) once the cache is warm. + - ../../../../../.m2:/home/gpadmin/.m2 command: ["tail", "-f", "/dev/null"] networks: diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml b/ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml index 825a95b61..eee4cee2c 100644 --- a/ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml +++ b/ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml @@ -45,6 +45,11 @@ services: volumes: - ../../../../../cloudberry-pxf:/home/gpadmin/workspace/cloudberry-pxf - ../../../../../cloudberry:/home/gpadmin/workspace/cloudberry + # Share host's Maven local repo with the container so mvn dependency + # downloads survive across runs via actions/cache. Neutralises transient + # 5xx from Maven Central and single-source repos (top-q.co.il for + # JSystem) once the cache is warm. + - ../../../../../.m2:/home/gpadmin/.m2 command: ["tail", "-f", "/dev/null"] networks: From 2dd1dc2d10c67f198ae9d7c76645e31ae6238223 Mon Sep 17 00:00:00 2001 From: huluhuifeng <1095824838@qq.com> Date: Fri, 17 Jul 2026 11:03:52 +0800 Subject: [PATCH 3/3] ci: raise HDFS DataNode startup wait to 180s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Test PXF Rocky9 - smoke intermittently fails with 'DataNode not available after 90s' in entrypoint.sh's wait_for_datanode helper. The smoke test is the first test group run against a freshly booted pxf-cbdb-dev container and starts the entire Hadoop/Hive/HBase stack cold — NameNode, DataNode, Hive metastore, HiveServer2, HBase master/regionserver — all competing for CPU and I/O on a shared GHA runner. DataNode registration path breakdown: 1. JVM cold start (class load + JIT + GC warmup) 2. DataNode scans /data/dn/dfs/data for existing block metadata 3. DataNode registers with NameNode (sends block report) 4. NameNode ACKs, exits safe mode, marks DataNode Live 5. 'hdfs dfsadmin -report' shows 'Live datanodes: 1' On healthy GHA runners this cycle completes in 20-40s. Under I/O contention (overlay2 file I/O slowness + noisy-neighbor CPU starvation) it can stretch to 90-180s. The prior 90s/attempt * 2 attempts = 180s total budget is on the edge and intermittently insufficient. Raise per-attempt wait from 90s to 180s (max_attempts unchanged at 2, total budget 360s). Healthy runs are unaffected — the loop exits as soon as DataNode reports Live. Only the tail latency budget grows. Related to #17. --- ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh b/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh index 040a0cbde..b6b2bdb1c 100755 --- a/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh +++ b/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh @@ -341,8 +341,13 @@ wait_for_datanode() { local max_attempts=2 for _attempt in $(seq 1 ${max_attempts}); do local dn_ready=false - # Wait up to 90s (45 tries * 2s) for DataNode to register - for _dn_try in $(seq 1 45); do + # Wait up to 180s (90 tries * 2s) for DataNode to register. GHA free-tier + # runners under I/O contention can take 90-180s for JVM cold-start + + # overlay2 block scan + NameNode handshake; the earlier 90s window was + # tight enough that Test PXF Rocky9 - smoke intermittently hit both + # attempts before DataNode came Live. Healthy runs still complete in + # 20-40s, so this only extends the tail — no cost on the happy path. + for _dn_try in $(seq 1 90); do if hdfs dfsadmin -report 2>/dev/null | grep -q "Live datanodes.*[1-9]"; then dn_ready=true break @@ -356,7 +361,7 @@ wait_for_datanode() { fi # DataNode didn't come up; diagnose and attempt restart - log "DataNode not available after 90s (attempt ${_attempt}/${max_attempts})" + log "DataNode not available after 180s (attempt ${_attempt}/${max_attempts})" log "--- DataNode diagnostic info ---" # Check if DataNode process is alive if command -v jps >/dev/null 2>&1; then