From a807963788532f5b62ff988e9eb6665a30276685 Mon Sep 17 00:00:00 2001 From: woblerr Date: Sun, 15 Mar 2026 23:10:13 +0300 Subject: [PATCH 01/39] Add gpbackman in the cloudberry-backup. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrate gpbackman (https://github.com/woblerr/gpbackman) — a CLI utility for managing gpbackup backups — into the cloudberry-backup repo. YAML functionality (history-migrate, converters, utils_file) is not ported. Key changes: - Add gpbackman.go entry point with //go:build gpbackman tag - Create gpbackman/cmd/, gpbackman/gpbckpconfig/, gpbackman/textmsg/ - Migrate imports: greenplum-db/* → apache/cloudberry-* - Remove converter layer — use history.BackupConfig directly - Extract 11 BackupConfig methods into standalone functions (helpers.go) - Replace go-pretty/v6 with olekukonko/tablewriter - Add gpbackman to all Makefile targets (build/install/clean/package) - Version via ldflags: -X gpbackman/cmd.version - Rewrite unit tests from stdlib testing to Ginkgo/Gomega CLI subcommands: backup-info, backup-delete, backup-clean, history-clean, report-info. --- Makefile | 13 +- go.mod | 2 +- go.sum | 2 + gpbackman.go | 10 + gpbackman/cmd/backup_clean.go | 275 +++++++++ gpbackman/cmd/backup_delete.go | 573 ++++++++++++++++++ gpbackman/cmd/backup_info.go | 336 ++++++++++ gpbackman/cmd/cmd_suite_test.go | 18 + gpbackman/cmd/constants.go | 56 ++ gpbackman/cmd/history_clean.go | 124 ++++ gpbackman/cmd/interfaces.go | 29 + gpbackman/cmd/report_info.go | 277 +++++++++ gpbackman/cmd/root.go | 98 +++ gpbackman/cmd/wrappers.go | 244 ++++++++ gpbackman/cmd/wrappers_test.go | 373 ++++++++++++ gpbackman/gpbckpconfig/cluster.go | 79 +++ .../gpbckpconfig/gpbckpconfig_suite_test.go | 13 + gpbackman/gpbckpconfig/helpers.go | 234 +++++++ gpbackman/gpbckpconfig/helpers_test.go | 558 +++++++++++++++++ gpbackman/gpbckpconfig/struct.go | 22 + gpbackman/gpbckpconfig/utils.go | 164 +++++ gpbackman/gpbckpconfig/utils_db.go | 191 ++++++ gpbackman/gpbckpconfig/utils_db_test.go | 118 ++++ gpbackman/gpbckpconfig/utils_test.go | 253 ++++++++ gpbackman/textmsg/error.go | 253 ++++++++ gpbackman/textmsg/error_test.go | 145 +++++ gpbackman/textmsg/info.go | 54 ++ gpbackman/textmsg/info_test.go | 102 ++++ gpbackman/textmsg/textmsg_suite_test.go | 13 + gpbackman/textmsg/warn.go | 7 + gpbackman/textmsg/warn_test.go | 24 + 31 files changed, 4656 insertions(+), 4 deletions(-) create mode 100644 gpbackman.go create mode 100644 gpbackman/cmd/backup_clean.go create mode 100644 gpbackman/cmd/backup_delete.go create mode 100644 gpbackman/cmd/backup_info.go create mode 100644 gpbackman/cmd/cmd_suite_test.go create mode 100644 gpbackman/cmd/constants.go create mode 100644 gpbackman/cmd/history_clean.go create mode 100644 gpbackman/cmd/interfaces.go create mode 100644 gpbackman/cmd/report_info.go create mode 100644 gpbackman/cmd/root.go create mode 100644 gpbackman/cmd/wrappers.go create mode 100644 gpbackman/cmd/wrappers_test.go create mode 100644 gpbackman/gpbckpconfig/cluster.go create mode 100644 gpbackman/gpbckpconfig/gpbckpconfig_suite_test.go create mode 100644 gpbackman/gpbckpconfig/helpers.go create mode 100644 gpbackman/gpbckpconfig/helpers_test.go create mode 100644 gpbackman/gpbckpconfig/struct.go create mode 100644 gpbackman/gpbckpconfig/utils.go create mode 100644 gpbackman/gpbckpconfig/utils_db.go create mode 100644 gpbackman/gpbckpconfig/utils_db_test.go create mode 100644 gpbackman/gpbckpconfig/utils_test.go create mode 100644 gpbackman/textmsg/error.go create mode 100644 gpbackman/textmsg/error_test.go create mode 100644 gpbackman/textmsg/info.go create mode 100644 gpbackman/textmsg/info_test.go create mode 100644 gpbackman/textmsg/textmsg_suite_test.go create mode 100644 gpbackman/textmsg/warn.go create mode 100644 gpbackman/textmsg/warn_test.go diff --git a/Makefile b/Makefile index a5c3138a..c1f8cf05 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ BACKUP=gpbackup RESTORE=gprestore HELPER=gpbackup_helper S3PLUGIN=gpbackup_s3_plugin +GPBACKMAN=gpbackman BIN_DIR=$(shell echo $${GOPATH:-~/go} | awk -F':' '{ print $$1 "/bin"}') GINKGO_FLAGS := -r --keep-going --randomize-suites --randomize-all --no-color GIT_VERSION := $(shell v=$$(git describe --tags 2>/dev/null); if [ -n "$$v" ]; then echo $$v | perl -pe 's/(.*)-([0-9]*)-(g[0-9a-f]*)/\1+dev.\2.\3/'; else cat VERSION 2>/dev/null || echo "dev"; fi) @@ -16,9 +17,10 @@ BACKUP_VERSION_STR=github.com/apache/cloudberry-backup/backup.version=$(GIT_VERS RESTORE_VERSION_STR=github.com/apache/cloudberry-backup/restore.version=$(GIT_VERSION) HELPER_VERSION_STR=github.com/apache/cloudberry-backup/helper.version=$(GIT_VERSION) S3PLUGIN_VERSION_STR=github.com/apache/cloudberry-backup/plugins/s3plugin.version=$(GIT_VERSION) +GPBACKMAN_VERSION_STR=github.com/apache/cloudberry-backup/gpbackman/cmd.version=$(GIT_VERSION) # note that /testutils is not a production directory, but has unit tests to validate testing tools -SUBDIRS_HAS_UNIT=backup/ filepath/ history/ helper/ options/ report/ restore/ toc/ utils/ testutils/ plugins/s3plugin/ +SUBDIRS_HAS_UNIT=backup/ filepath/ history/ helper/ options/ report/ restore/ toc/ utils/ testutils/ plugins/s3plugin/ gpbackman/cmd/ gpbackman/gpbckpconfig/ gpbackman/textmsg/ SUBDIRS_ALL=$(SUBDIRS_HAS_UNIT) integration/ end_to_end/ GOLANG_LINTER=$(GOPATH)/bin/golangci-lint GINKGO=$(GOPATH)/bin/ginkgo @@ -87,21 +89,24 @@ build : $(GOSQLITE) CGO_ENABLED=1 $(GO_BUILD) -tags '$(RESTORE)' -o $(BIN_DIR)/$(RESTORE) --ldflags '-X $(RESTORE_VERSION_STR)' CGO_ENABLED=1 $(GO_BUILD) -tags '$(HELPER)' -o $(BIN_DIR)/$(HELPER) --ldflags '-X $(HELPER_VERSION_STR)' CGO_ENABLED=1 $(GO_BUILD) -tags '$(S3PLUGIN)' -o $(BIN_DIR)/$(S3PLUGIN) --ldflags '-X $(S3PLUGIN_VERSION_STR)' + CGO_ENABLED=1 $(GO_BUILD) -tags '$(GPBACKMAN)' -o $(BIN_DIR)/$(GPBACKMAN) --ldflags '-X $(GPBACKMAN_VERSION_STR)' debug : CGO_ENABLED=1 $(GO_BUILD) -tags '$(BACKUP)' -o $(BIN_DIR)/$(BACKUP) -ldflags "-X $(BACKUP_VERSION_STR)" $(DEBUG) CGO_ENABLED=1 $(GO_BUILD) -tags '$(RESTORE)' -o $(BIN_DIR)/$(RESTORE) -ldflags "-X $(RESTORE_VERSION_STR)" $(DEBUG) CGO_ENABLED=1 $(GO_BUILD) -tags '$(HELPER)' -o $(BIN_DIR)/$(HELPER) -ldflags "-X $(HELPER_VERSION_STR)" $(DEBUG) CGO_ENABLED=1 $(GO_BUILD) -tags '$(S3PLUGIN)' -o $(BIN_DIR)/$(S3PLUGIN) -ldflags "-X $(S3PLUGIN_VERSION_STR)" $(DEBUG) + CGO_ENABLED=1 $(GO_BUILD) -tags '$(GPBACKMAN)' -o $(BIN_DIR)/$(GPBACKMAN) -ldflags "-X $(GPBACKMAN_VERSION_STR)" $(DEBUG) build_linux : env GOOS=linux GOARCH=amd64 $(GO_BUILD) -tags '$(BACKUP)' -o $(BACKUP) -ldflags "-X $(BACKUP_VERSION_STR)" env GOOS=linux GOARCH=amd64 $(GO_BUILD) -tags '$(RESTORE)' -o $(RESTORE) -ldflags "-X $(RESTORE_VERSION_STR)" env GOOS=linux GOARCH=amd64 $(GO_BUILD) -tags '$(HELPER)' -o $(HELPER) -ldflags "-X $(HELPER_VERSION_STR)" env GOOS=linux GOARCH=amd64 $(GO_BUILD) -tags '$(S3PLUGIN)' -o $(S3PLUGIN) -ldflags "-X $(S3PLUGIN_VERSION_STR)" + env GOOS=linux GOARCH=amd64 $(GO_BUILD) -tags '$(GPBACKMAN)' -o $(GPBACKMAN) -ldflags "-X $(GPBACKMAN_VERSION_STR)" install : - cp $(BIN_DIR)/$(BACKUP) $(BIN_DIR)/$(RESTORE) $(GPHOME)/bin + cp $(BIN_DIR)/$(BACKUP) $(BIN_DIR)/$(RESTORE) $(BIN_DIR)/$(GPBACKMAN) $(GPHOME)/bin @psql -X -t -d template1 -c 'select distinct hostname from gp_segment_configuration where content != -1' > /tmp/seg_hosts 2>/dev/null; \ if [ $$? -eq 0 ]; then \ $(COPYUTIL) -f /tmp/seg_hosts $(helper_path) $(s3plugin_path) =:$(GPHOME)/bin/; \ @@ -119,7 +124,7 @@ install : clean : # Build artifacts - rm -f $(BIN_DIR)/$(BACKUP) $(BACKUP) $(BIN_DIR)/$(RESTORE) $(RESTORE) $(BIN_DIR)/$(HELPER) $(HELPER) $(BIN_DIR)/$(S3PLUGIN) $(S3PLUGIN) + rm -f $(BIN_DIR)/$(BACKUP) $(BACKUP) $(BIN_DIR)/$(RESTORE) $(RESTORE) $(BIN_DIR)/$(HELPER) $(HELPER) $(BIN_DIR)/$(S3PLUGIN) $(S3PLUGIN) $(BIN_DIR)/$(GPBACKMAN) $(GPBACKMAN) # Test artifacts rm -rf /tmp/go-build* /tmp/gexec_artifacts* /tmp/ginkgo* docker stop s3-minio # stop minio before removing its data directories @@ -178,6 +183,7 @@ package: @GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=$(CGO) go build -tags '$(RESTORE)' -o $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/bin/$(RESTORE) --ldflags '-X $(RESTORE_VERSION_STR)' @GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=$(CGO) go build -tags '$(HELPER)' -o $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/bin/$(HELPER) --ldflags '-X $(HELPER_VERSION_STR)' @GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=$(CGO) go build -tags '$(S3PLUGIN)' -o $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/bin/$(S3PLUGIN) --ldflags '-X $(S3PLUGIN_VERSION_STR)' + @GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=$(CGO) go build -tags '$(GPBACKMAN)' -o $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/bin/$(GPBACKMAN) --ldflags '-X $(GPBACKMAN_VERSION_STR)' @echo "Creating install script..." @echo '#!/bin/bash' > $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh @echo 'set -e' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh @@ -203,6 +209,7 @@ package: @echo 'sudo chmod 755 "$${INSTALL_DIR}/bin/$(RESTORE)"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh @echo 'sudo chmod 755 "$${INSTALL_DIR}/bin/$(HELPER)"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh @echo 'sudo chmod 755 "$${INSTALL_DIR}/bin/$(S3PLUGIN)"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh + @echo 'sudo chmod 755 "$${INSTALL_DIR}/bin/$(GPBACKMAN)"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh @echo '' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh @echo 'echo "Installation complete!"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh @echo 'echo "$(PACKAGE_NAME) binaries installed to $${INSTALL_DIR}/bin/"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh diff --git a/go.mod b/go.mod index 3adaf23e..463ee9e4 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 github.com/urfave/cli v1.22.13 + golang.org/x/crypto v0.21.0 golang.org/x/sys v0.18.0 golang.org/x/tools v0.12.0 gopkg.in/cheggaaa/pb.v1 v1.0.28 @@ -49,7 +50,6 @@ require ( github.com/mattn/go-runewidth v0.0.13 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - golang.org/x/crypto v0.21.0 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index ca12ca24..7495cb68 100644 --- a/go.sum +++ b/go.sum @@ -252,6 +252,8 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/gpbackman.go b/gpbackman.go new file mode 100644 index 00000000..cc1cbc8e --- /dev/null +++ b/gpbackman.go @@ -0,0 +1,10 @@ +//go:build gpbackman +// +build gpbackman + +package main + +import "github.com/apache/cloudberry-backup/gpbackman/cmd" + +func main() { + cmd.Execute() +} diff --git a/gpbackman/cmd/backup_clean.go b/gpbackman/cmd/backup_clean.go new file mode 100644 index 00000000..1e610320 --- /dev/null +++ b/gpbackman/cmd/backup_clean.go @@ -0,0 +1,275 @@ +package cmd + +import ( + "database/sql" + "strconv" + + "github.com/apache/cloudberry-go-libs/gplog" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/apache/cloudberry-backup/gpbackman/gpbckpconfig" + "github.com/apache/cloudberry-backup/gpbackman/textmsg" + "github.com/apache/cloudberry-backup/utils" +) + +// Flags for the gpbackman backup-clean command (backupCleanCmd) +var ( + backupCleanBeforeTimestamp string + backupCleanAfterTimestamp string + backupCleanPluginConfigFile string + backupCleanBackupDir string + backupCleanOlderThenDays uint + backupCleanParallelProcesses int + backupCleanCascade bool +) + +var backupCleanCmd = &cobra.Command{ + Use: "backup-clean", + Short: "Delete all existing backups older than the specified time condition", + Long: `Delete all existing backups older than the specified time condition. + +To delete backup sets older than the given timestamp, use the --before-timestamp option. +To delete backup sets older than the given number of days, use the --older-than-day option. +To delete backup sets newer than the given timestamp, use the --after-timestamp option. +Only --older-than-days, --before-timestamp or --after-timestamp option must be specified. + +By default, the existence of dependent backups is checked and deletion process is not performed, +unless the --cascade option is passed in. + +By default, the deletion will be performed for local backup. + +The full path to the backup directory can be set using the --backup-dir option. + +For local backups the following logic are applied: + * If the --backup-dir option is specified, the deletion will be performed in provided path. + * If the --backup-dir option is not specified, but the backup was made with --backup-dir flag for gpbackup, the deletion will be performed in the backup manifest path. + * If the --backup-dir option is not specified and backup directory is not specified in backup manifest, the deletion will be performed in backup folder in the master and segments data directories. + * If backup is not local, the error will be returned. + +For control over the number of parallel processes and ssh connections to delete local backups, the --parallel-processes option can be used. + +The storage plugin config file location can be set using the --plugin-config option. +The full path to the file is required. In this case, the deletion will be performed using the storage plugin. + +For non local backups the following logic are applied: + * If the --plugin-config option is specified, the deletion will be performed using the storage plugin. + * If backup is local, the error will be returned. + +The gpbackup_history.db file location can be set using the --history-db option. +Can be specified only once. The full path to the file is required. +If the --history-db option is not specified, the history database will be searched in the current directory.`, + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + doRootFlagValidation(cmd.Flags(), checkFileExistsConst) + doCleanBackupFlagValidation(cmd.Flags()) + doCleanBackup() + }, +} + +func init() { + rootCmd.AddCommand(backupCleanCmd) + backupCleanCmd.PersistentFlags().StringVar( + &backupCleanPluginConfigFile, + pluginConfigFileFlagName, + "", + "the full path to plugin config file", + ) + backupCleanCmd.PersistentFlags().BoolVar( + &backupCleanCascade, + cascadeFlagName, + false, + "delete all dependent backups", + ) + backupCleanCmd.PersistentFlags().UintVar( + &backupCleanOlderThenDays, + olderThenDaysFlagName, + 0, + "delete backup sets older than the given number of days", + ) + backupCleanCmd.PersistentFlags().StringVar( + &backupCleanBeforeTimestamp, + beforeTimestampFlagName, + "", + "delete backup sets older than the given timestamp", + ) + backupCleanCmd.PersistentFlags().StringVar( + &backupCleanAfterTimestamp, + afterTimestampFlagName, + "", + "delete backup sets newer than the given timestamp", + ) + backupCleanCmd.PersistentFlags().StringVar( + &backupCleanBackupDir, + backupDirFlagName, + "", + "the full path to backup directory for local backups", + ) + backupCleanCmd.PersistentFlags().IntVar( + &backupCleanParallelProcesses, + parallelProcessesFlagName, + 1, + "the number of parallel processes to delete local backups", + ) + backupCleanCmd.MarkFlagsMutuallyExclusive(beforeTimestampFlagName, olderThenDaysFlagName, afterTimestampFlagName) +} + +// These flag checks are applied only for backup-clean command. +func doCleanBackupFlagValidation(flags *pflag.FlagSet) { + var err error + // If before-timestamp flag is specified and have correct values. + if flags.Changed(beforeTimestampFlagName) { + err = gpbckpconfig.CheckTimestamp(backupCleanBeforeTimestamp) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(backupCleanBeforeTimestamp, beforeTimestampFlagName, err)) + execOSExit(exitErrorCode) + } + beforeTimestamp = backupCleanBeforeTimestamp + } + if flags.Changed(olderThenDaysFlagName) { + beforeTimestamp = gpbckpconfig.GetTimestampOlderThen(backupCleanOlderThenDays) + } + // If after-timestamp flag is specified and have correct values. + if flags.Changed(afterTimestampFlagName) { + err = gpbckpconfig.CheckTimestamp(backupCleanAfterTimestamp) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(backupCleanAfterTimestamp, afterTimestampFlagName, err)) + execOSExit(exitErrorCode) + } + afterTimestamp = backupCleanAfterTimestamp + } + // backup-dir anf plugin-config flags cannot be used together. + err = checkCompatibleFlags(flags, backupDirFlagName, pluginConfigFileFlagName) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableCompatibleFlags(err, backupDirFlagName, pluginConfigFileFlagName)) + execOSExit(exitErrorCode) + } + // If parallel-processes flag is specified and have correct values. + if flags.Changed(parallelProcessesFlagName) && !gpbckpconfig.IsPositiveValue(backupCleanParallelProcesses) { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(strconv.Itoa(backupCleanParallelProcesses), parallelProcessesFlagName, err)) + execOSExit(exitErrorCode) + } + // plugin-config and parallel-precesses flags cannot be used together. + err = checkCompatibleFlags(flags, parallelProcessesFlagName, pluginConfigFileFlagName) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableCompatibleFlags(err, parallelProcessesFlagName, pluginConfigFileFlagName)) + execOSExit(exitErrorCode) + } + // If backup-dir flag is specified and it exists and the full path is specified. + if flags.Changed(backupDirFlagName) { + err = gpbckpconfig.CheckFullPath(backupCleanBackupDir, checkFileExistsConst) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(backupCleanBackupDir, backupDirFlagName, err)) + execOSExit(exitErrorCode) + } + } + // If plugin-config flag is specified and it exists and the full path is specified. + if flags.Changed(pluginConfigFileFlagName) { + err = gpbckpconfig.CheckFullPath(backupCleanPluginConfigFile, checkFileExistsConst) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(backupCleanPluginConfigFile, pluginConfigFileFlagName, err)) + execOSExit(exitErrorCode) + } + } + if beforeTimestamp == "" && afterTimestamp == "" { + gplog.Error("%s", textmsg.ErrorTextUnableValidateValue(textmsg.ErrorValidationValue(), olderThenDaysFlagName, beforeTimestampFlagName, afterTimestampFlagName)) + execOSExit(exitErrorCode) + } +} + +func doCleanBackup() { + logHeadersDebug() + err := cleanBackup() + if err != nil { + execOSExit(exitErrorCode) + } +} + +func cleanBackup() error { + hDB, err := gpbckpconfig.OpenHistoryDB(getHistoryDBPath(rootHistoryDB)) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("open", err)) + return err + } + defer func() { + closeErr := hDB.Close() + if closeErr != nil { + gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("close", closeErr)) + } + }() + if backupCleanPluginConfigFile != "" { + pluginConfig, err := utils.ReadPluginConfig(backupCleanPluginConfigFile) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableReadPluginConfigFile(err)) + return err + } + err = backupCleanDBPlugin(backupCleanCascade, beforeTimestamp, afterTimestamp, backupCleanPluginConfigFile, pluginConfig, hDB) + if err != nil { + return err + } + } else { + err := backupCleanDBLocal(backupCleanCascade, beforeTimestamp, afterTimestamp, backupCleanBackupDir, backupCleanParallelProcesses, hDB) + if err != nil { + return err + } + } + return nil +} + +func backupCleanDBPlugin(deleteCascade bool, cutOffTimestamp, cutOffAfterTimestamp, pluginConfigPath string, pluginConfig *utils.PluginConfig, hDB *sql.DB) error { + backupList, err := fetchBackupNamesForDeletion(cutOffTimestamp, cutOffAfterTimestamp, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableReadHistoryDB(err)) + return err + } + if len(backupList) > 0 { + gplog.Debug("%s", textmsg.InfoTextBackupDeleteList(backupList)) + // Execute deletion for each backup. + // Use backupDeleteDBPlugin function from backup-delete command. + // Don't use force deletes and ignore errors for mass deletion. + err = backupDeleteDBPlugin(backupList, deleteCascade, false, false, pluginConfigPath, pluginConfig, hDB) + if err != nil { + return err + } + } else { + gplog.Info("%s", textmsg.InfoTextNothingToDo()) + } + return nil +} + +func backupCleanDBLocal(deleteCascade bool, cutOffTimestamp, cutOffAfterTimestamp, backupDir string, maxParallelProcesses int, hDB *sql.DB) error { + backupList, err := fetchBackupNamesForDeletion(cutOffTimestamp, cutOffAfterTimestamp, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableReadHistoryDB(err)) + return err + } + if len(backupList) > 0 { + gplog.Debug("%s", textmsg.InfoTextBackupDeleteList(backupList)) + err = backupDeleteDBLocal(backupList, backupDir, deleteCascade, false, false, maxParallelProcesses, hDB) + if err != nil { + return err + } + } else { + gplog.Info("%s", textmsg.InfoTextNothingToDo()) + } + return nil +} + +// Get the list of backup names for deletion. +func fetchBackupNamesForDeletion(cutOffTimestamp, cutOffAfterTimestamp string, hDB *sql.DB) ([]string, error) { + var backupList []string + var err error + if cutOffTimestamp != "" { + backupList, err = gpbckpconfig.GetBackupNamesBeforeTimestamp(cutOffTimestamp, hDB) + if err != nil { + return nil, err + } + } + if cutOffAfterTimestamp != "" { + backupList, err = gpbckpconfig.GetBackupNamesAfterTimestamp(cutOffAfterTimestamp, hDB) + if err != nil { + return nil, err + } + } + return backupList, nil +} diff --git a/gpbackman/cmd/backup_delete.go b/gpbackman/cmd/backup_delete.go new file mode 100644 index 00000000..bd673a96 --- /dev/null +++ b/gpbackman/cmd/backup_delete.go @@ -0,0 +1,573 @@ +package cmd + +import ( + "bytes" + "database/sql" + "fmt" + "os" + "os/exec" + "strconv" + "sync" + "time" + + "golang.org/x/crypto/ssh" + + "github.com/apache/cloudberry-go-libs/gplog" + "github.com/apache/cloudberry-go-libs/operating" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/apache/cloudberry-backup/gpbackman/gpbckpconfig" + "github.com/apache/cloudberry-backup/gpbackman/textmsg" + "github.com/apache/cloudberry-backup/utils" +) + +// Flags for the gpbackman backup-delete command (backupDeleteCmd) +var ( + backupDeleteTimestamp []string + backupDeletePluginConfigFile string + backupDeleteBackupDir string + backupDeleteCascade bool + backupDeleteForce bool + backupDeleteIgnoreErrors bool + backupDeleteParallelProcesses int +) +var backupDeleteCmd = &cobra.Command{ + Use: "backup-delete", + Short: "Delete a specific existing backup", + Long: `Delete a specific existing backup. + +The --timestamp option must be specified. It could be specified multiple times. + +By default, the existence of dependent backups is checked and deletion process is not performed, +unless the --cascade option is passed in. + +If backup already deleted, the deletion process is skipped, unless --force option is specified. +If errors occur during the deletion process, the errors can be ignored using the --ignore-errors option. +The --ignore-errors option can be used only with --force option. + +By default, the deletion will be performed for local backup. + +The full path to the backup directory can be set using the --backup-dir option. + +For local backups the following logic are applied: + * If the --backup-dir option is specified, the deletion will be performed in provided path. + * If the --backup-dir option is not specified, but the backup was made with --backup-dir flag for gpbackup, the deletion will be performed in the backup manifest path. + * If the --backup-dir option is not specified and backup directory is not specified in backup manifest, the deletion will be performed in backup folder in the master and segments data directories. + * If backup is not local, the error will be returned. + +For control over the number of parallel processes and ssh connections to delete local backups, the --parallel-processes option can be used. + +The storage plugin config file location can be set using the --plugin-config option. +The full path to the file is required. In this case, the deletion will be performed using the storage plugin. + +For non local backups the following logic are applied: + * If the --plugin-config option is specified, the deletion will be performed using the storage plugin. + * If backup is local, the error will be returned. + +The gpbackup_history.db file location can be set using the --history-db option. +Can be specified only once. The full path to the file is required. +If the --history-db option is not specified, the history database will be searched in the current directory.`, + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + doRootFlagValidation(cmd.Flags(), checkFileExistsConst) + doDeleteBackupFlagValidation(cmd.Flags()) + doDeleteBackup() + }, +} + +var execCommand = exec.Command + +func init() { + rootCmd.AddCommand(backupDeleteCmd) + backupDeleteCmd.PersistentFlags().StringArrayVar( + &backupDeleteTimestamp, + timestampFlagName, + []string{""}, + "the backup timestamp for deleting, could be specified multiple times", + ) + backupDeleteCmd.PersistentFlags().StringVar( + &backupDeletePluginConfigFile, + pluginConfigFileFlagName, + "", + "the full path to plugin config file", + ) + backupDeleteCmd.PersistentFlags().BoolVar( + &backupDeleteCascade, + cascadeFlagName, + false, + "delete all dependent backups for the specified backup timestamp", + ) + backupDeleteCmd.PersistentFlags().BoolVar( + &backupDeleteForce, + forceFlagName, + false, + "try to delete, even if the backup already mark as deleted", + ) + backupDeleteCmd.PersistentFlags().StringVar( + &backupDeleteBackupDir, + backupDirFlagName, + "", + "the full path to backup directory for local backups", + ) + backupDeleteCmd.PersistentFlags().IntVar( + &backupDeleteParallelProcesses, + parallelProcessesFlagName, + 1, + "the number of parallel processes to delete local backups", + ) + backupDeleteCmd.PersistentFlags().BoolVar( + &backupDeleteIgnoreErrors, + ignoreErrorsFlagName, + false, + "ignore errors when deleting backups", + ) + _ = backupDeleteCmd.MarkPersistentFlagRequired(timestampFlagName) +} + +// These flag checks are applied only for backup-delete command. +func doDeleteBackupFlagValidation(flags *pflag.FlagSet) { + var err error + // If timestamps are specified and have correct values. + if flags.Changed(timestampFlagName) { + for _, timestamp := range backupDeleteTimestamp { + err = gpbckpconfig.CheckTimestamp(timestamp) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(timestamp, timestampFlagName, err)) + execOSExit(exitErrorCode) + } + } + } + // backup-dir anf plugin-config flags cannot be used together. + err = checkCompatibleFlags(flags, backupDirFlagName, pluginConfigFileFlagName) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableCompatibleFlags(err, backupDirFlagName, pluginConfigFileFlagName)) + execOSExit(exitErrorCode) + } + // If parallel-processes flag is specified and have correct values. + if flags.Changed(parallelProcessesFlagName) && !gpbckpconfig.IsPositiveValue(backupDeleteParallelProcesses) { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(strconv.Itoa(backupDeleteParallelProcesses), parallelProcessesFlagName, err)) + execOSExit(exitErrorCode) + } + // plugin-config and parallel-precesses flags cannot be used together. + err = checkCompatibleFlags(flags, parallelProcessesFlagName, pluginConfigFileFlagName) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableCompatibleFlags(err, parallelProcessesFlagName, pluginConfigFileFlagName)) + execOSExit(exitErrorCode) + } + // If backup-dir flag is specified and it exists and the full path is specified. + if flags.Changed(backupDirFlagName) { + err = gpbckpconfig.CheckFullPath(backupDeleteBackupDir, checkFileExistsConst) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(backupDeleteBackupDir, backupDirFlagName, err)) + execOSExit(exitErrorCode) + } + } + // If the plugin-config flag is specified and it exists and the full path is specified. + if flags.Changed(pluginConfigFileFlagName) { + err = gpbckpconfig.CheckFullPath(backupDeletePluginConfigFile, checkFileExistsConst) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(backupDeletePluginConfigFile, pluginConfigFileFlagName, err)) + execOSExit(exitErrorCode) + } + } + // If ignore-errors flag is specified, but force flag is not. + if flags.Changed(ignoreErrorsFlagName) && !flags.Changed(forceFlagName) { + gplog.Error("%s", textmsg.ErrorTextUnableValidateValue(textmsg.ErrorNotIndependentFlagsError(), ignoreErrorsFlagName, forceFlagName)) + execOSExit(exitErrorCode) + } + +} + +func doDeleteBackup() { + logHeadersDebug() + err := deleteBackup() + if err != nil { + execOSExit(exitErrorCode) + } +} + +func deleteBackup() error { + hDB, err := gpbckpconfig.OpenHistoryDB(getHistoryDBPath(rootHistoryDB)) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("open", err)) + return err + } + defer func() { + closeErr := hDB.Close() + if closeErr != nil { + gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("close", closeErr)) + } + }() + if backupDeletePluginConfigFile != "" { + pluginConfig, err := utils.ReadPluginConfig(backupDeletePluginConfigFile) + if err != nil { + return err + } + err = backupDeleteDBPlugin(backupDeleteTimestamp, backupDeleteCascade, backupDeleteForce, backupDeleteIgnoreErrors, backupDeletePluginConfigFile, pluginConfig, hDB) + if err != nil { + return err + } + } else { + err := backupDeleteDBLocal(backupDeleteTimestamp, backupDeleteBackupDir, backupDeleteCascade, backupDeleteForce, backupDeleteIgnoreErrors, backupDeleteParallelProcesses, hDB) + if err != nil { + return err + } + } + return nil +} + +func backupDeleteDBPlugin(backupListForDeletion []string, deleteCascade, deleteForce, ignoreErrors bool, pluginConfigPath string, pluginConfig *utils.PluginConfig, hDB *sql.DB) error { + deleter := &backupPluginDeleter{ + pluginConfigPath: pluginConfigPath, + pluginConfig: pluginConfig} + // Skip local backups. + skipLocalBackup := true + return backupDeleteDB(backupListForDeletion, deleteCascade, deleteForce, ignoreErrors, skipLocalBackup, deleter, hDB) +} + +func backupDeleteDBLocal(backupListForDeletion []string, backupDir string, deleteCascade, deleteForce, ignoreErrors bool, maxParallelProcesses int, hDB *sql.DB) error { + deleter := &backupLocalDeleter{ + backupDir: backupDir, + maxParallelProcesses: maxParallelProcesses} + // Include local backups. + skipLocalBackups := false + return backupDeleteDB(backupListForDeletion, deleteCascade, deleteForce, ignoreErrors, skipLocalBackups, deleter, hDB) +} + +func backupDeleteDB(backupListForDeletion []string, deleteCascade, deleteForce, ignoreErrors, skipLocalBackup bool, deleter backupDeleteInterface, hDB *sql.DB) error { + for _, backupName := range backupListForDeletion { + backupData, err := gpbckpconfig.GetBackupDataDB(backupName, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupInfo(backupName, err)) + return err + } + canBeDeleted, err := checkBackupCanBeUsed(deleteForce, skipLocalBackup, backupData) + if err != nil { + return err + } + if canBeDeleted { + backupDependencies, err := gpbckpconfig.GetBackupDependencies(backupName, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupValue("dependencies", backupName, err)) + return err + } + if len(backupDependencies) > 0 { + gplog.Info("%s", textmsg.InfoTextBackupDependenciesList(backupName, backupDependencies)) + if deleteCascade { + gplog.Debug("%s", textmsg.InfoTextBackupDeleteList(backupDependencies)) + // If the deletion of at least one dependent backup fails, we fail full entire chain. + err = backupDeleteDBCascade(backupDependencies, deleteForce, ignoreErrors, skipLocalBackup, deleter, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableDeleteBackupCascade(backupName, err)) + return err + } + } else { + gplog.Error("%s", textmsg.ErrorTextUnableDeleteBackupUseCascade(backupName, textmsg.ErrorBackupDeleteCascadeOptionError())) + return textmsg.ErrorBackupDeleteCascadeOptionError() + } + } + err = deleter.backupDeleteDB(backupName, hDB, ignoreErrors) + if err != nil { + return err + } + } + } + return nil +} + +func backupDeleteDBCascade(backupList []string, deleteForce, ignoreErrors, skipLocalBackup bool, deleter backupDeleteInterface, hDB *sql.DB) error { + for _, backup := range backupList { + backupData, err := gpbckpconfig.GetBackupDataDB(backup, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupInfo(backup, err)) + return err + } + // Skip local backup. + canBeDeleted, err := checkBackupCanBeUsed(deleteForce, skipLocalBackup, backupData) + if err != nil { + return err + } + if canBeDeleted { + err = deleter.backupDeleteDB(backup, hDB, ignoreErrors) + if err != nil { + return err + } + } + } + return nil +} + +func backupDeleteDBPluginFunc(backupName, pluginConfigPath string, pluginConfig *utils.PluginConfig, hDB *sql.DB, ignoreErrors bool) error { + var err error + dateDeleted := getCurrentTimestamp() + gplog.Info("%s", textmsg.InfoTextBackupDeleteStart(backupName)) + err = gpbckpconfig.UpdateDeleteStatus(backupName, gpbckpconfig.DateDeletedInProgress, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableSetBackupStatus(gpbckpconfig.DateDeletedInProgress, backupName, err)) + return err + } + gplog.Debug("%s", textmsg.InfoTextCommandExecution(pluginConfig.ExecutablePath, deleteBackupPluginCommand, pluginConfigPath, backupName)) + stdout, stderr, errdel := execDeleteBackupPlugin(pluginConfig.ExecutablePath, deleteBackupPluginCommand, pluginConfigPath, backupName) + if stderr != "" { + gplog.Error("%s", stderr) + } + if errdel != nil { + handleErrorDB(backupName, textmsg.ErrorTextUnableDeleteBackup(backupName, errdel), gpbckpconfig.DateDeletedPluginFailed, hDB) + if !ignoreErrors { + return errdel + } + } + gplog.Info("%s", stdout) + backupData, err := gpbckpconfig.GetBackupDataDB(backupName, hDB) + if err != nil { + handleErrorDB(backupName, textmsg.ErrorTextUnableGetBackupInfo(backupName, err), gpbckpconfig.DateDeletedPluginFailed, hDB) + if !ignoreErrors { + return err + } + } + bckpDir, _, _, err := getBackupMasterDir("", backupData.BackupDir, backupData.DatabaseName) + if err != nil { + handleErrorDB(backupName, textmsg.ErrorTextUnableGetBackupPath("backup directory", backupName, err), gpbckpconfig.DateDeletedPluginFailed, hDB) + if !ignoreErrors { + return err + } + } + gplog.Debug("%s", textmsg.InfoTextCommandExecution("delete directory", gpbckpconfig.BackupDirPath(bckpDir, backupName))) + // Delete local files on master. + err = os.RemoveAll(gpbckpconfig.BackupDirPath(bckpDir, backupName)) + if err != nil { + handleErrorDB(backupName, textmsg.ErrorTextUnableDeleteBackup(backupName, err), gpbckpconfig.DateDeletedPluginFailed, hDB) + if !ignoreErrors { + return err + } + } + err = gpbckpconfig.UpdateDeleteStatus(backupName, dateDeleted, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableSetBackupStatus(dateDeleted, backupName, err)) + return err + } + gplog.Info("%s", textmsg.InfoTextBackupDeleteSuccess(backupName)) + return nil +} + +func backupDeleteDBLocalFunc(backupName, backupDir string, maxParallelProcesses int, hDB *sql.DB, ignoreErrors bool) error { + var err, errUpdate error + dateDeleted := getCurrentTimestamp() + gplog.Info("%s", textmsg.InfoTextBackupDeleteStart(backupName)) + errUpdate = gpbckpconfig.UpdateDeleteStatus(backupName, gpbckpconfig.DateDeletedInProgress, hDB) + if errUpdate != nil { + gplog.Error("%s", textmsg.ErrorTextUnableSetBackupStatus(gpbckpconfig.DateDeletedInProgress, backupName, errUpdate)) + return errUpdate + } + backupData, err := gpbckpconfig.GetBackupDataDB(backupName, hDB) + if err != nil { + handleErrorDB(backupName, textmsg.ErrorTextUnableGetBackupInfo(backupName, err), gpbckpconfig.DateDeletedLocalFailed, hDB) + return err + } + bckpDir, segPrefix, isSingleBackupDir, err := getBackupMasterDir(backupDir, backupData.BackupDir, backupData.DatabaseName) + if err != nil { + handleErrorDB(backupName, textmsg.ErrorTextUnableGetBackupPath("backup directory", backupName, err), gpbckpconfig.DateDeletedLocalFailed, hDB) + return err + } + gplog.Debug("%s", textmsg.InfoTextBackupDirPath(bckpDir)) + gplog.Debug("%s", textmsg.InfoTextSegmentPrefix(segPrefix)) + backupType, err := gpbckpconfig.GetBackupType(backupData) + if err != nil { + handleErrorDB(backupName, textmsg.ErrorTextUnableGetBackupValue("type", backupName, err), gpbckpconfig.DateDeletedLocalFailed, hDB) + return err + } + // If backup type is not "metadata-only", we should delete files on segments and master. + // If backup type is "metadata-only", we should not delete files only on master. + if backupType != gpbckpconfig.BackupTypeMetadataOnly { + var errSeg error + segConfig, errSeg := getSegmentConfigurationClusterInfo(backupData.DatabaseName) + if errSeg != nil { + handleErrorDB(backupName, textmsg.ErrorTextUnableGetBackupPath("segment configuration", backupName, errSeg), gpbckpconfig.DateDeletedLocalFailed, hDB) + if !ignoreErrors { + return errSeg + } + } + // Execute on segments. + errSeg = executeDeleteBackupOnSegments(backupDir, backupData.BackupDir, backupName, segPrefix, isSingleBackupDir, ignoreErrors, segConfig, maxParallelProcesses) + if errSeg != nil { + handleErrorDB(backupName, textmsg.ErrorTextUnableDeleteBackup(backupName, errSeg), gpbckpconfig.DateDeletedLocalFailed, hDB) + if !ignoreErrors { + return errSeg + } + } + } + // Delete files on master. + gplog.Debug("%s", textmsg.InfoTextCommandExecution("delete directory", gpbckpconfig.BackupDirPath(bckpDir, backupName))) + err = os.RemoveAll(gpbckpconfig.BackupDirPath(bckpDir, backupName)) + if err != nil { + handleErrorDB(backupName, textmsg.ErrorTextUnableDeleteBackup(backupName, err), gpbckpconfig.DateDeletedLocalFailed, hDB) + if !ignoreErrors { + return err + } + } + errUpdate = gpbckpconfig.UpdateDeleteStatus(backupName, dateDeleted, hDB) + if errUpdate != nil { + gplog.Error("%s", textmsg.ErrorTextUnableSetBackupStatus(dateDeleted, backupName, errUpdate)) + return errUpdate + } + gplog.Info("%s", textmsg.InfoTextBackupDeleteSuccess(backupName)) + return nil +} + +func execDeleteBackupPlugin(executablePath, deleteBackupPluginCommand, pluginConfigFile, timestamp string) (string, string, error) { + cmd := execCommand(executablePath, deleteBackupPluginCommand, pluginConfigFile, timestamp) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + return stdout.String(), stderr.String(), err +} + +// ExecuteCommandsOnHosts Delete backup dir on all segment hosts in parallel. +// The function checks that the directories exists on all segment hosts before deletion. +func executeDeleteBackupOnSegments(backupDir, backupDataBackupDir, backupName, segPrefix string, isSingleBackupDir, ignoreErrors bool, configs []gpbckpconfig.SegmentConfig, maxParallelProcesses int) error { + var once sync.Once + limit := make(chan bool, maxParallelProcesses) + wg := &sync.WaitGroup{} + errCh := make(chan error, len(configs)) + sshClientConf, err := getSSHConfig() + if err != nil { + return err + } + // Check that the directory exists on all segment hosts. + for _, config := range configs { + wg.Add(1) + limit <- true + backupPath, err := getBackupSegmentDir(backupDir, backupDataBackupDir, config.DataDir, segPrefix, config.ContentID, isSingleBackupDir) + if err != nil { + return err + } + go func(backupPath, host string) { + defer func() { <-limit }() + defer wg.Done() + checkBackupDirExistsOnSegments(gpbckpconfig.BackupDirPath(backupPath, backupName), host, sshClientConf, errCh) + }(backupPath, config.Hostname) + } + // We should block the main function and wait for the WaitGroup to complete. + // It is necessary to strictly verify that all checks are performed for a specific backup + // and this particular backup will be deleted. + // Only after deleting backups can we move on to the next one. + // It is necessary to avoid situations where checks are performed simultaneously for one backup, + // and deletion occurs for another. + // + // Don't use code like + // go func() { wg.Wait(); once.Do(func() { close(errCh) }) }() + // + wg.Wait() + // Fix error like "panic: close of closed channel". + once.Do(func() { + close(errCh) + }) + for err := range errCh { + if err != nil && !ignoreErrors { + return err + } + } + // If all checks passed, delete the directory on all segment hosts. + // Reset the wait group. + wg = &sync.WaitGroup{} + for _, config := range configs { + wg.Add(1) + limit <- true + backupPath, err := getBackupSegmentDir(backupDir, backupDataBackupDir, config.DataDir, segPrefix, config.ContentID, isSingleBackupDir) + if err != nil { + return err + } + go func(backupPath, host string) { + defer func() { <-limit }() + defer wg.Done() + deleteBackupDirOnSegments(gpbckpconfig.BackupDirPath(backupPath, backupName), host, sshClientConf, errCh) + }(backupPath, config.Hostname) + } + wg.Wait() + // Fix error like "panic: close of closed channel". + once.Do(func() { + close(errCh) + }) + for err := range errCh { + if err != nil && !ignoreErrors { + return err + } + } + return nil +} +func checkBackupDirExistsOnSegments(path, host string, sshConf *ssh.ClientConfig, errCh chan error) { + connection, err := ssh.Dial("tcp", host+":22", sshConf) + if err != nil { + errCh <- err + return + } + defer connection.Close() + + session, err := connection.NewSession() + if err != nil { + errCh <- err + return + } + defer session.Close() + command := fmt.Sprintf("test -d %s", path) + gplog.Debug("%s", textmsg.InfoTextCommandExecution(command, "on host", host)) + if err := session.Run(command); err != nil { + gplog.Error("%s", textmsg.ErrorTextCommandExecutionFailed(err, command, "on host", host)) + errCh <- textmsg.ErrorNotFoundBackupDirIn(fmt.Sprintf("%s on host %s", path, host)) + return + } + gplog.Debug("%s", textmsg.InfoTextCommandExecutionSucceeded(command, "on host", host)) +} + +func deleteBackupDirOnSegments(path, host string, sshConf *ssh.ClientConfig, errCh chan error) { + connection, err := ssh.Dial("tcp", host+":22", sshConf) + if err != nil { + errCh <- err + return + } + defer connection.Close() + + session, err := connection.NewSession() + if err != nil { + errCh <- err + return + } + defer session.Close() + command := fmt.Sprintf("rm -rf %s", path) + gplog.Debug("%s", textmsg.InfoTextCommandExecution(command, "on host", host)) + if err := session.Run(command); err != nil { + gplog.Error("%s", textmsg.ErrorTextCommandExecutionFailed(err, command, "on host", host)) + errCh <- err + return + } + gplog.Debug("%s", textmsg.InfoTextCommandExecutionSucceeded(command, "on host", host)) +} + +func getSSHConfig() (*ssh.ClientConfig, error) { + currentUser, _ := operating.System.CurrentUser() + key, err := os.ReadFile(currentUser.HomeDir + "/.ssh/id_rsa") + if err != nil { + return nil, err + } + signer, err := ssh.ParsePrivateKey(key) + if err != nil { + return nil, err + } + // sshConfig is a configuration object for establishing an SSH connection. + // It contains the user's username, authentication method using public keys, + // and a host key callback that ignores insecure host keys. + sshConfig := &ssh.ClientConfig{ + User: currentUser.Username, + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(signer), + }, + // Disable known_hosts check. + // This check also disables in gpbackup utility. + // #nosec G106 + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: 30 * time.Second, + } + return sshConfig, nil +} diff --git a/gpbackman/cmd/backup_info.go b/gpbackman/cmd/backup_info.go new file mode 100644 index 00000000..f7a2595f --- /dev/null +++ b/gpbackman/cmd/backup_info.go @@ -0,0 +1,336 @@ +package cmd + +import ( + "database/sql" + "os" + + "github.com/apache/cloudberry-go-libs/gplog" + "github.com/olekukonko/tablewriter" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/apache/cloudberry-backup/gpbackman/gpbckpconfig" + "github.com/apache/cloudberry-backup/gpbackman/textmsg" + "github.com/apache/cloudberry-backup/history" +) + +// Flags for the gpbackman backup-info command (backupInfoCmd) +var ( + backupInfoShowDeleted bool + backupInfoShowFailed bool + backupInfoBackupTypeFilter string + backupInfoTableNameFilter string + backupInfoSchemaNameFilter string + backupInfoExcludeFilter bool + backupInfoTimestamp string + backupInfoShowDetails bool +) + +// Options for the backup-info command. +type BackupInfoOptions struct { + ShowDeleted bool + ShowFailed bool + BackupTypeFilter string + TableNameFilter string + SchemaNameFilter string + ExcludeFilter bool + Timestamp string + ShowDetails bool +} + +var backupInfoCmd = &cobra.Command{ + Use: "backup-info", + Short: "Display information about backups", + Long: `Display information about backups. + +By default, only active backups or backups with deletion status "In progress" from gpbackup_history.db are displayed. + +To display deleted backups, use the --deleted option. +To display failed backups, use the --failed option. +To display all backups, use --deleted and --failed options together. + +To display backups of a specific type, use the --type option. + +To display backups that include the specified table, use the --table option. +The formatting rules for . match those of the --include-table option in gpbackup. + +To display backups that include the specified schema, use the --schema option. +The formatting rules for match those of the --include-schema option in gpbackup. + +To display backups that exclude the specified table, use the --table and --exclude options. +The formatting rules for .
match those of the --exclude-table option in gpbackup. + +To display backups that exclude the specified schema, use the --schema and --exclude options. +The formatting rules for match those of the --exclude-schema option in gpbackup. + +To display details about object filtering, use the --detail option. +The details are presented as follows, depending on the active filtering type: + * include-table / exclude-table: a comma-separated list of fully-qualified table names in the format .
; + * include-schema / exclude-schema: a comma-separated list of schema names; + * if no object filtering was used, the value is empty. + +To display a backup chain for a specific backup, use the --timestamp option. +In this mode, the backup with the specified timestamp and all of its dependent backups will be displayed. +The deleted and failed backups are always included in this mode. +To display object filtering details in this mode, use the --detail option. +When --timestamp is set, the following options cannot be used: --type, --table, --schema, --exclude, --failed, --deleted. + +To display the "object filtering details" column for all backups without using --timestamp, use the --detail option. + +The gpbackup_history.db file location can be set using the --history-db option. +Can be specified only once. The full path to the file is required. +If the --history-db option is not specified, the history database will be searched in the current directory.`, + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + doRootFlagValidation(cmd.Flags(), checkFileExistsConst) + doBackupInfoFlagValidation(cmd.Flags()) + doBackupInfo() + }, +} + +func init() { + rootCmd.AddCommand(backupInfoCmd) + backupInfoCmd.Flags().StringVar( + &backupInfoTimestamp, + timestampFlagName, + "", + "show backup info and its dependent backups for the specified timestamp", + ) + backupInfoCmd.Flags().BoolVar( + &backupInfoShowDeleted, + deletedFlagName, + false, + "show deleted backups", + ) + backupInfoCmd.Flags().BoolVar( + &backupInfoShowFailed, + failedFlagName, + false, + "show failed backups", + ) + backupInfoCmd.Flags().StringVar( + &backupInfoBackupTypeFilter, + typeFlagName, + "", + "backup type filter (full, incremental, data-only, metadata-only)", + ) + backupInfoCmd.Flags().StringVar( + &backupInfoTableNameFilter, + tableFlagName, + "", + "show backups that include the specified table (format .
)", + ) + backupInfoCmd.Flags().StringVar( + &backupInfoSchemaNameFilter, + schemaFlagName, + "", + "show backups that include the specified schema", + ) + backupInfoCmd.Flags().BoolVar( + &backupInfoExcludeFilter, + excludeFlagName, + false, + "show backups that exclude the specific table (format .
) or schema", + ) + backupInfoCmd.Flags().BoolVar( + &backupInfoShowDetails, + detailFlagName, + false, + "show object filtering details", + ) +} + +// These flag checks are applied only for backup-info commands. +func doBackupInfoFlagValidation(flags *pflag.FlagSet) { + var err error + if flags.Changed(timestampFlagName) { + err = gpbckpconfig.CheckTimestamp(backupInfoTimestamp) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(backupInfoTimestamp, timestampFlagName, err)) + execOSExit(exitErrorCode) + } + // --timestamp is not compatible with --type, --table, --schema, --exclude, --failed, --deleted + err = checkCompatibleFlags(flags, timestampFlagName, + typeFlagName, tableFlagName, schemaFlagName, excludeFlagName, failedFlagName, deletedFlagName) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableCompatibleFlags(err, timestampFlagName, typeFlagName, tableFlagName, schemaFlagName, excludeFlagName, failedFlagName, deletedFlagName)) + execOSExit(exitErrorCode) + } + } + // If type is specified and have correct values. + if flags.Changed(typeFlagName) { + err = checkBackupType(backupInfoBackupTypeFilter) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(backupInfoBackupTypeFilter, typeFlagName, err)) + execOSExit(exitErrorCode) + } + } + // table flag and schema flags cannot be used together. + err = checkCompatibleFlags(flags, tableFlagName, schemaFlagName) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableCompatibleFlags(err, tableFlagName, schemaFlagName)) + execOSExit(exitErrorCode) + } + // If table is specified and have correct values. + if flags.Changed(tableFlagName) { + err = gpbckpconfig.CheckTableFQN(backupInfoTableNameFilter) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(backupInfoTableNameFilter, tableFlagName, err)) + execOSExit(exitErrorCode) + } + } + // If exclude flag is specified, but table or schema flag is not. + if flags.Changed(excludeFlagName) && !flags.Changed(tableFlagName) && !flags.Changed(schemaFlagName) { + gplog.Error("%s", textmsg.ErrorTextUnableValidateValue(textmsg.ErrorNotIndependentFlagsError(), tableFlagName, schemaFlagName)) + execOSExit(exitErrorCode) + } +} + +func doBackupInfo() { + logHeadersDebug() + err := backupInfo() + if err != nil { + execOSExit(exitErrorCode) + } +} + +func backupInfo() error { + opts := BackupInfoOptions{ + ShowDeleted: backupInfoShowDeleted, + ShowFailed: backupInfoShowFailed, + BackupTypeFilter: backupInfoBackupTypeFilter, + TableNameFilter: backupInfoTableNameFilter, + SchemaNameFilter: backupInfoSchemaNameFilter, + ExcludeFilter: backupInfoExcludeFilter, + Timestamp: backupInfoTimestamp, + ShowDetails: backupInfoShowDetails, + } + t := tablewriter.NewWriter(os.Stdout) + initTable(t, opts.ShowDetails) + hDB, err := gpbckpconfig.OpenHistoryDB(getHistoryDBPath(rootHistoryDB)) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("open", err)) + return err + } + defer func() { + closeErr := hDB.Close() + if closeErr != nil { + gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("close", closeErr)) + } + }() + err = backupInfoDB(opts, hDB, t) + if err != nil { + return err + } + t.Render() + return nil +} + +func backupInfoDB(opts BackupInfoOptions, hDB *sql.DB, t *tablewriter.Table) error { + // List all according to showDeleted/showFailed + if opts.Timestamp == "" { + backupList, err := gpbckpconfig.GetBackupNamesDB(opts.ShowDeleted, opts.ShowFailed, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableReadHistoryDB(err)) + return err + } + for _, backupName := range backupList { + backupData, err := gpbckpconfig.GetBackupDataDB(backupName, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupInfo(backupName, err)) + return err + } + addBackupToTable(opts.BackupTypeFilter, opts.TableNameFilter, opts.SchemaNameFilter, opts.ExcludeFilter, opts.ShowDetails, backupData, t) + } + return nil + } + // Timestamp mode: show base backup and only its dependent backups + // Verify base backup exists + baseBackupData, err := gpbckpconfig.GetBackupDataDB(opts.Timestamp, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupInfo(opts.Timestamp, err)) + return err + } + addBackupToTable("", "", "", false, opts.ShowDetails, baseBackupData, t) + backupDependenciesList, err := gpbckpconfig.GetBackupDependencies(opts.Timestamp, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableReadHistoryDB(err)) + return err + } + for _, depTimestamp := range backupDependenciesList { + backupData, err := gpbckpconfig.GetBackupDataDB(depTimestamp, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupInfo(depTimestamp, err)) + return err + } + addBackupToTable("", "", "", false, opts.ShowDetails, backupData, t) + } + return nil +} + +func initTable(t *tablewriter.Table, includeDetails bool) { + t.SetBorder(false) + t.SetAutoFormatHeaders(false) + header := []string{ + "timestamp", + "date", + "status", + "database", + "type", + "object filtering", + "plugin", + "duration", + "date deleted", + } + if includeDetails { + header = append(header, "object filtering details") + } + t.SetHeader(header) +} + +// addBackupToTable adds a backup to the table for displaying. +// +// If errors occur, they are logged, but they are not returned. +// The main idea is to show the maximum available information and display all errors that occur. +// But do not fall when errors occur. So, display anyway. +func addBackupToTable(backupTypeFilter, backupTableFilter, backupSchemaFilter string, backupExcludeFilter, includeDetails bool, backupData *history.BackupConfig, t *tablewriter.Table) { + var matchToObjectFilter bool + backupDate, err := gpbckpconfig.GetBackupDate(backupData) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupValue("date", backupData.Timestamp, err)) + } + backupType, err := gpbckpconfig.GetBackupType(backupData) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupValue("type", backupData.Timestamp, err)) + } + backupFilter, err := gpbckpconfig.GetObjectFilteringInfo(backupData) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupValue("object filtering", backupData.Timestamp, err)) + } + backupDuration, err := gpbckpconfig.GetBackupDuration(backupData) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupValue("duration", backupData.Timestamp, err)) + } + backupDateDeleted, err := gpbckpconfig.GetBackupDateDeleted(backupData) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupValue("date deletion", backupData.Timestamp, err)) + } + matchToObjectFilter = gpbckpconfig.CheckObjectFilteringExists(backupData, backupTableFilter, backupSchemaFilter, backupFilter, backupExcludeFilter) + if (backupTypeFilter == "" || backupTypeFilter == backupType) && matchToObjectFilter { + row := []string{ + backupData.Timestamp, + backupDate, + backupData.Status, + backupData.DatabaseName, + backupType, + backupFilter, + backupData.Plugin, + formatBackupDuration(backupDuration), + backupDateDeleted, + } + if includeDetails { + row = append(row, gpbckpconfig.GetObjectFilteringDetails(backupData)) + } + t.Append(row) + } +} diff --git a/gpbackman/cmd/cmd_suite_test.go b/gpbackman/cmd/cmd_suite_test.go new file mode 100644 index 00000000..d84a7680 --- /dev/null +++ b/gpbackman/cmd/cmd_suite_test.go @@ -0,0 +1,18 @@ +package cmd + +import ( + "testing" + + "github.com/apache/cloudberry-go-libs/testhelper" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestCmd(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Cmd Suite") +} + +var _ = BeforeSuite(func() { + _, _, _ = testhelper.SetupTestLogger() +}) diff --git a/gpbackman/cmd/constants.go b/gpbackman/cmd/constants.go new file mode 100644 index 00000000..28a3b209 --- /dev/null +++ b/gpbackman/cmd/constants.go @@ -0,0 +1,56 @@ +package cmd + +const ( + commandName = "gpbackman" + + // Plugin commands. + // To be able to work with various plugins, + // it is highly desirable to use the commands from the plugin specification. + // See https://github.com/greenplum-db/gpbackup/blob/710fe53305958c1faed2e6008b894b4923bed253/plugins/README.md + deleteBackupPluginCommand = "delete_backup" + restoreDataPluginCommand = "restore_data" + + historyFileNameBaseConst = "gpbackup_history" + historyFileDBSuffixConst = ".db" + historyDBNameConst = historyFileNameBaseConst + historyFileDBSuffixConst + + // Flags. + historyDBFlagName = "history-db" + logFileFlagName = "log-file" + logLevelConsoleFlagName = "log-level-console" + logLevelFileFlagName = "log-level-file" + timestampFlagName = "timestamp" + pluginConfigFileFlagName = "plugin-config" + reportFilePluginPathFlagName = "plugin-report-file-path" + deletedFlagName = "deleted" + failedFlagName = "failed" + cascadeFlagName = "cascade" + forceFlagName = "force" + olderThenDaysFlagName = "older-than-days" + beforeTimestampFlagName = "before-timestamp" + afterTimestampFlagName = "after-timestamp" + typeFlagName = "type" + tableFlagName = "table" + schemaFlagName = "schema" + excludeFlagName = "exclude" + backupDirFlagName = "backup-dir" + parallelProcessesFlagName = "parallel-processes" + ignoreErrorsFlagName = "ignore-errors" + detailFlagName = "detail" + + exitErrorCode = 1 + + // Default for checking the existence of the file. + checkFileExistsConst = true + + // Batch size for deleting from sqlite3. + // This is to prevent problem with sqlite3. + sqliteDeleteBatchSize = 1000 +) + +var ( + // Timestamp to delete all backups before. + beforeTimestamp string + // Timestamp to delete all backups after. + afterTimestamp string +) diff --git a/gpbackman/cmd/history_clean.go b/gpbackman/cmd/history_clean.go new file mode 100644 index 00000000..dc62d5d6 --- /dev/null +++ b/gpbackman/cmd/history_clean.go @@ -0,0 +1,124 @@ +package cmd + +import ( + "database/sql" + + "github.com/apache/cloudberry-go-libs/gplog" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/apache/cloudberry-backup/gpbackman/gpbckpconfig" + "github.com/apache/cloudberry-backup/gpbackman/textmsg" +) + +// Flags for the gpbackman history-clean command (historyCleanCmd) +var ( + historyCleanBeforeTimestamp string + historyCleanOlderThenDays uint +) + +var historyCleanCmd = &cobra.Command{ + Use: "history-clean", + Short: "Clean deleted backups from the history database", + Long: `Clean deleted backups from the history database. +Only the database is being cleaned up. + +Information is deleted only about deleted backups from gpbackup_history.db. Each backup must be deleted first. + +To delete information about backups older than the given timestamp, use the --before-timestamp option. +To delete information about backups older than the given number of days, use the --older-than-day option. +Only --older-than-days or --before-timestamp option must be specified, not both. + +The gpbackup_history.db file location can be set using the --history-db option. +Can be specified only once. The full path to the file is required. +If the --history-db option is not specified, the history database will be searched in the current directory.`, + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + doRootFlagValidation(cmd.Flags(), checkFileExistsConst) + doCleanHistoryFlagValidation(cmd.Flags()) + doCleanHistory() + }, +} + +func init() { + rootCmd.AddCommand(historyCleanCmd) + historyCleanCmd.PersistentFlags().UintVar( + &historyCleanOlderThenDays, + olderThenDaysFlagName, + 0, + "delete information about backups older than the given number of days", + ) + historyCleanCmd.PersistentFlags().StringVar( + &historyCleanBeforeTimestamp, + beforeTimestampFlagName, + "", + "delete information about backups older than the given timestamp", + ) + historyCleanCmd.MarkFlagsMutuallyExclusive(beforeTimestampFlagName, olderThenDaysFlagName) +} + +// These flag checks are applied only for backup-clean command. +func doCleanHistoryFlagValidation(flags *pflag.FlagSet) { + var err error + // If before-timestamp are specified and have correct values. + if flags.Changed(beforeTimestampFlagName) { + err = gpbckpconfig.CheckTimestamp(historyCleanBeforeTimestamp) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(historyCleanBeforeTimestamp, beforeTimestampFlagName, err)) + execOSExit(exitErrorCode) + } + beforeTimestamp = historyCleanBeforeTimestamp + } + if flags.Changed(olderThenDaysFlagName) { + beforeTimestamp = gpbckpconfig.GetTimestampOlderThen(historyCleanOlderThenDays) + } + if beforeTimestamp == "" { + gplog.Error("%s", textmsg.ErrorTextUnableValidateValue(textmsg.ErrorValidationValue(), olderThenDaysFlagName, beforeTimestampFlagName)) + execOSExit(exitErrorCode) + } +} + +func doCleanHistory() { + logHeadersDebug() + err := cleanHistory() + if err != nil { + execOSExit(exitErrorCode) + } +} + +func cleanHistory() error { + hDB, err := gpbckpconfig.OpenHistoryDB(getHistoryDBPath(rootHistoryDB)) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("open", err)) + return err + } + defer func() { + closeErr := hDB.Close() + if closeErr != nil { + gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("close", closeErr)) + } + }() + err = historyCleanDB(beforeTimestamp, hDB) + if err != nil { + return err + } + return nil +} + +func historyCleanDB(cutOffTimestamp string, hDB *sql.DB) error { + backupList, err := gpbckpconfig.GetBackupNamesForCleanBeforeTimestamp(cutOffTimestamp, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableReadHistoryDB(err)) + return err + } + if len(backupList) > 0 { + gplog.Debug("%s", textmsg.InfoTextBackupDeleteListFromHistory(backupList)) + err := gpbckpconfig.CleanBackupsDB(backupList, sqliteDeleteBatchSize, hDB) + if err != nil { + return err + } + } else { + gplog.Info("%s", textmsg.InfoTextNothingToDo()) + } + return nil +} diff --git a/gpbackman/cmd/interfaces.go b/gpbackman/cmd/interfaces.go new file mode 100644 index 00000000..91608514 --- /dev/null +++ b/gpbackman/cmd/interfaces.go @@ -0,0 +1,29 @@ +package cmd + +import ( + "database/sql" + + "github.com/apache/cloudberry-backup/utils" +) + +type backupDeleteInterface interface { + backupDeleteDB(backupName string, hDB *sql.DB, ignoreErrors bool) error +} + +type backupPluginDeleter struct { + pluginConfigPath string + pluginConfig *utils.PluginConfig +} + +func (bpd *backupPluginDeleter) backupDeleteDB(backupName string, hDB *sql.DB, ignoreErrors bool) error { + return backupDeleteDBPluginFunc(backupName, bpd.pluginConfigPath, bpd.pluginConfig, hDB, ignoreErrors) +} + +type backupLocalDeleter struct { + backupDir string + maxParallelProcesses int +} + +func (bld *backupLocalDeleter) backupDeleteDB(backupName string, hDB *sql.DB, ignoreErrors bool) error { + return backupDeleteDBLocalFunc(backupName, bld.backupDir, bld.maxParallelProcesses, hDB, ignoreErrors) +} diff --git a/gpbackman/cmd/report_info.go b/gpbackman/cmd/report_info.go new file mode 100644 index 00000000..b8a2cd21 --- /dev/null +++ b/gpbackman/cmd/report_info.go @@ -0,0 +1,277 @@ +package cmd + +import ( + "bytes" + "database/sql" + "fmt" + "os" + "path/filepath" + + "github.com/apache/cloudberry-go-libs/gplog" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/apache/cloudberry-backup/gpbackman/gpbckpconfig" + "github.com/apache/cloudberry-backup/gpbackman/textmsg" + "github.com/apache/cloudberry-backup/history" + "github.com/apache/cloudberry-backup/utils" +) + +// Flags for the gpbackman report-info command (reportInfoCmd) +var ( + reportInfoTimestamp string + reportInfoPluginConfigFile string + reportInfoReportFilePluginPath string + reportInfoBackupDir string +) + +var reportInfoCmd = &cobra.Command{ + Use: "report-info", + Short: "Display the report for a specific backup", + Long: `Display the report for a specific backup. + +The --timestamp option must be specified. + +The report could be displayed only for active backups. + +The full path to the backup directory can be set using the --backup-dir option. +The full path to the data directory is required. + +For local backups the following logic are applied: + * If the --backup-dir option is specified, the report will be searched in provided path. + * If the --backup-dir option is not specified, but the backup was made with --backup-dir flag for gpbackup, the report will be searched in provided path from backup manifest. + * If the --backup-dir option is not specified and backup directory is not specified in backup manifest, the utility try to connect to local cluster and get master data directory. + If this information is available, the report will be in master data directory. + * If backup is not local, the error will be returned. + +The storage plugin config file location can be set using the --plugin-config option. +The full path to the file is required. + +For non local backups the following logic are applied: + * If the --plugin-config option is specified, the report will be searched in provided location. + * If backup is local, the error will be returned. + +Only --backup-dir or --plugin-config option can be specified, not both. + +If a custom plugin is used, it is required to specify the path to the directory with the repo file using the --plugin-report-file-path option. +It is not necessary to use the --plugin-report-file-path flag for the following plugins (the path is generated automatically): + * gpbackup_s3_plugin. + +The gpbackup_history.db file location can be set using the --history-db option. +Can be specified only once. The full path to the file is required. +If the --history-db option is not specified, the history database will be searched in the current directory.`, + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + doRootFlagValidation(cmd.Flags(), checkFileExistsConst) + doReportInfoFlagValidation(cmd.Flags()) + doReportInfo() + }, +} + +func init() { + rootCmd.AddCommand(reportInfoCmd) + reportInfoCmd.PersistentFlags().StringVar( + &reportInfoTimestamp, + timestampFlagName, + "", + "the backup timestamp for report displaying", + ) + reportInfoCmd.PersistentFlags().StringVar( + &reportInfoPluginConfigFile, + pluginConfigFileFlagName, + "", + "the full path to plugin config file", + ) + reportInfoCmd.PersistentFlags().StringVar( + &reportInfoReportFilePluginPath, + reportFilePluginPathFlagName, + "", + "the full path to plugin report file", + ) + reportInfoCmd.PersistentFlags().StringVar( + &reportInfoBackupDir, + backupDirFlagName, + "", + "the full path to backup directory", + ) + _ = reportInfoCmd.MarkPersistentFlagRequired(timestampFlagName) +} + +// These flag checks are applied only for report-info command. +func doReportInfoFlagValidation(flags *pflag.FlagSet) { + var err error + // If timestamps are specified and have correct values. + if flags.Changed(timestampFlagName) { + err = gpbckpconfig.CheckTimestamp(reportInfoTimestamp) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(reportInfoTimestamp, timestampFlagName, err)) + execOSExit(exitErrorCode) + } + } + // backup-dir anf plugin-config flags cannot be used together. + err = checkCompatibleFlags(flags, backupDirFlagName, pluginConfigFileFlagName) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableCompatibleFlags(err, backupDirFlagName, pluginConfigFileFlagName)) + execOSExit(exitErrorCode) + } + // If backup-dir flag is specified and it exists and the full path is specified. + if flags.Changed(backupDirFlagName) { + err = gpbckpconfig.CheckFullPath(reportInfoBackupDir, checkFileExistsConst) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(reportInfoBackupDir, backupDirFlagName, err)) + execOSExit(exitErrorCode) + } + } + // If plugin-config flag is specified and it exists and the full path is specified. + if flags.Changed(pluginConfigFileFlagName) { + err = gpbckpconfig.CheckFullPath(reportInfoPluginConfigFile, checkFileExistsConst) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(reportInfoPluginConfigFile, pluginConfigFileFlagName, err)) + execOSExit(exitErrorCode) + } + } + // If plugin-report-file-path flag is specified. + if flags.Changed(reportFilePluginPathFlagName) { + // But plugin-config flag is not specified. + if !flags.Changed(pluginConfigFileFlagName) { + gplog.Error("%s", textmsg.ErrorTextUnableValidateValue(textmsg.ErrorNotIndependentFlagsError(), reportFilePluginPathFlagName, pluginConfigFileFlagName)) + execOSExit(exitErrorCode) + } + // Check full path. + err = gpbckpconfig.CheckFullPath(reportInfoReportFilePluginPath, false) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(reportInfoReportFilePluginPath, reportFilePluginPathFlagName, err)) + execOSExit(exitErrorCode) + } + } +} + +func doReportInfo() { + logHeadersDebug() + err := reportInfo() + if err != nil { + execOSExit(exitErrorCode) + } +} + +func reportInfo() error { + hDB, err := gpbckpconfig.OpenHistoryDB(getHistoryDBPath(rootHistoryDB)) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("open", err)) + return err + } + defer func() { + closeErr := hDB.Close() + if closeErr != nil { + gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("close", closeErr)) + } + }() + if reportInfoPluginConfigFile != "" { + pluginConfig, err := utils.ReadPluginConfig(reportInfoPluginConfigFile) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableReadPluginConfigFile(err)) + return err + } + err = reportInfoDBPlugin(reportInfoTimestamp, reportInfoPluginConfigFile, pluginConfig, hDB) + if err != nil { + return err + } + } else { + err := reportInfoDBLocal(reportInfoTimestamp, reportInfoBackupDir, hDB) + if err != nil { + return err + } + } + return nil +} + +func reportInfoDBPlugin(backupName, pluginConfigPath string, pluginConfig *utils.PluginConfig, hDB *sql.DB) error { + backupData, err := gpbckpconfig.GetBackupDataDB(backupName, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupInfo(backupName, err)) + return err + } + err = reportInfoPluginFunc(backupData, pluginConfigPath, pluginConfig) + if err != nil { + return err + } + return nil +} + +func reportInfoPluginFunc(backupData *history.BackupConfig, pluginConfigPath string, pluginConfig *utils.PluginConfig) error { + // Skip local backup. + canGetReport, err := checkBackupCanBeUsed(false, true, backupData) + if err != nil { + return err + } + if canGetReport { + reportFile, err := gpbckpconfig.GetReportFilePathPlugin(backupData, reportInfoReportFilePluginPath, pluginConfig.Options) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupPath("report", backupData.Timestamp, err)) + return err + } + gplog.Debug("%s", textmsg.InfoTextCommandExecution(pluginConfig.ExecutablePath, restoreDataPluginCommand, pluginConfigPath, reportFile)) + stdout, stderr, err := execReportInfo(pluginConfig.ExecutablePath, restoreDataPluginCommand, pluginConfigPath, reportFile) + if stderr != "" { + gplog.Error("%s", stderr) + } + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupReport(backupData.Timestamp, err)) + return err + } + // Display the report. + fmt.Println(stdout) + } + return nil +} + +func reportInfoDBLocal(backupName, backupDir string, hDB *sql.DB) error { + backupData, err := gpbckpconfig.GetBackupDataDB(backupName, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupInfo(backupName, err)) + return err + } + err = reportInfoFileLocalFunc(backupData, backupDir) + if err != nil { + return err + } + return nil +} + +func reportInfoFileLocalFunc(backupData *history.BackupConfig, backupDir string) error { + // Include local backup. + canGetReport, err := checkBackupCanBeUsed(false, false, backupData) + if err != nil { + return err + } + if canGetReport { + timestamp := backupData.Timestamp + bckpDir, segPrefix, _, err := getBackupMasterDir(backupDir, backupData.BackupDir, backupData.DatabaseName) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupPath("backup directory", timestamp, err)) + return err + } + gplog.Debug("%s", textmsg.InfoTextBackupDirPath(bckpDir)) + gplog.Debug("%s", textmsg.InfoTextSegmentPrefix(segPrefix)) + reportFile := gpbckpconfig.ReportFilePath(bckpDir, timestamp) + // Sanitize the file path + reportFile = filepath.Clean(reportFile) + gplog.Debug("%s", textmsg.InfoTextCommandExecution("read file", reportFile)) + content, err := os.ReadFile(reportFile) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupReport(backupData.Timestamp, err)) + return err + } + fmt.Println(string(content)) + } + return nil +} + +func execReportInfo(executablePath, reportInfoPluginCommand, pluginConfigFile, file string) (string, string, error) { + cmd := execCommand(executablePath, reportInfoPluginCommand, pluginConfigFile, file) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + return stdout.String(), stderr.String(), err +} diff --git a/gpbackman/cmd/root.go b/gpbackman/cmd/root.go new file mode 100644 index 00000000..15f59f48 --- /dev/null +++ b/gpbackman/cmd/root.go @@ -0,0 +1,98 @@ +package cmd + +import ( + "fmt" + + "github.com/apache/cloudberry-backup/gpbackman/gpbckpconfig" + "github.com/apache/cloudberry-backup/gpbackman/textmsg" + "github.com/apache/cloudberry-go-libs/gplog" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +var version string + +// Flags for the gpbackman command (rootCmd) +var ( + rootHistoryDB string + rootLogFile string + rootLogLevelConsole string + rootLogLevelFile string +) + +var rootCmd = &cobra.Command{ + Use: commandName, + Short: "gpBackMan - utility for managing backups created by gpbackup", + Args: cobra.NoArgs, +} + +func init() { + rootCmd.PersistentFlags().StringVar( + &rootHistoryDB, + historyDBFlagName, + "", + "full path to the gpbackup_history.db file", + ) + rootCmd.PersistentFlags().StringVar( + &rootLogFile, + logFileFlagName, + "", + "full path to log file directory, if not specified, the log file will be created in the $HOME/gpAdminLogs directory", + ) + rootCmd.PersistentFlags().StringVar( + &rootLogLevelConsole, + logLevelConsoleFlagName, + "info", + "level for console logging (error, info, debug, verbose)", + ) + rootCmd.PersistentFlags().StringVar( + &rootLogLevelFile, + logLevelFileFlagName, + "info", + "level for file logging (error, info, debug, verbose)", + ) +} + +func doInit() { + rootCmd.Version = version + // If log-file flag is specified the log file will be created in the specified directory + gplog.InitializeLogging(commandName, rootLogFile) +} + +func getVersion() string { + return rootCmd.Version +} + +// These flag checks are applied for all commands: +func doRootFlagValidation(flags *pflag.FlagSet, checkFileExists bool) { + var err error + // If history-db flag is specified and full path. + // The existence of the file is checked by condition from each specific command. + // Not all commands require a history db file to exist. + if flags.Changed(historyDBFlagName) { + err = gpbckpconfig.CheckFullPath(rootHistoryDB, checkFileExists) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(rootHistoryDB, historyDBFlagName, err)) + execOSExit(exitErrorCode) + } + } + // Check, that the log level is correct. + err = setLogLevelConsole(rootLogLevelConsole) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(rootLogLevelConsole, logLevelConsoleFlagName, err)) + execOSExit(exitErrorCode) + } + err = setLogLevelFile(rootLogLevelFile) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(rootLogLevelFile, logLevelFileFlagName, err)) + execOSExit(exitErrorCode) + } +} + +func Execute() { + doInit() + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + execOSExit(exitErrorCode) + } +} diff --git a/gpbackman/cmd/wrappers.go b/gpbackman/cmd/wrappers.go new file mode 100644 index 00000000..81c8bec0 --- /dev/null +++ b/gpbackman/cmd/wrappers.go @@ -0,0 +1,244 @@ +package cmd + +import ( + "database/sql" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/apache/cloudberry-go-libs/gplog" + "github.com/spf13/pflag" + + "github.com/apache/cloudberry-backup/gpbackman/gpbckpconfig" + "github.com/apache/cloudberry-backup/gpbackman/textmsg" + "github.com/apache/cloudberry-backup/history" +) + +var execOSExit = os.Exit + +func logHeadersDebug() { + gplog.Debug("Start %s version %s", commandName, getVersion()) + gplog.Debug("Use console log level: %s", rootLogLevelConsole) + gplog.Debug("Use file log level: %s", rootLogLevelFile) + gplog.Debug("%s command: %s", commandName, os.Args) +} + +// Sets the log levels for the console and file loggers. +// Uppercase or lowercase letters are accepted. +// If an incorrect value is specified, an error is returned. +func setLogLevelConsole(level string) error { + switch strings.ToLower(level) { + case "info": + gplog.SetVerbosity(gplog.LOGINFO) + case "error": + gplog.SetVerbosity(gplog.LOGERROR) + case "debug": + gplog.SetVerbosity(gplog.LOGDEBUG) + case "verbose": + gplog.SetVerbosity(gplog.LOGVERBOSE) + default: + return textmsg.ErrorInvalidValueError() + } + return nil +} + +// Sets the log levels for the console and file loggers. +// Uppercase or lowercase letters are accepted. +// If an incorrect value is specified, an error is returned. +func setLogLevelFile(level string) error { + switch strings.ToLower(level) { + case "info": + gplog.SetLogFileVerbosity(gplog.LOGINFO) + case "error": + gplog.SetLogFileVerbosity(gplog.LOGERROR) + case "debug": + gplog.SetLogFileVerbosity(gplog.LOGDEBUG) + case "verbose": + gplog.SetLogFileVerbosity(gplog.LOGVERBOSE) + default: + return textmsg.ErrorInvalidValueError() + } + return nil +} + +func getHistoryDBPath(historyDBPath string) string { + var historyDBName = historyDBNameConst + if historyDBPath != "" { + return historyDBPath + } + return historyDBName +} + +func getCurrentTimestamp() string { + return time.Now().Format(gpbckpconfig.Layout) +} + +func checkCompatibleFlags(flags *pflag.FlagSet, flagNames ...string) error { + n := 0 + for _, name := range flagNames { + if flags.Changed(name) { + n++ + } + } + if n > 1 { + return textmsg.ErrorIncompatibleFlagsError() + } + return nil +} + +func formatBackupDuration(value float64) string { + hours := int(value / 3600) + minutes := (int(value) % 3600) / 60 + seconds := int(value) % 60 + return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds) +} + +// The backup can be used in one of the cases for local and plugin backups: +// - backup is active +// - backup is not active, but the --force flag is set. +// Returns: +// - true, if backup can be used; +// - false, if backup can't be used. +// Errors and warnings will also returned and logged. +func checkBackupCanBeUsed(deleteForce, skipLocalBackup bool, backupData *history.BackupConfig) (bool, error) { + result := false + err := checkLocalBackupStatus(skipLocalBackup, gpbckpconfig.IsLocal(backupData)) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableWorkBackup(backupData.Timestamp, err)) + return result, err + } + if gpbckpconfig.IsInProgress(backupData) && !deleteForce { + gplog.Error("%s", textmsg.InfoTextBackupStatus(backupData.Timestamp, backupData.Status)) + return result, nil + } + backupDateDeleted, errDateDeleted := gpbckpconfig.GetBackupDateDeleted(backupData) + if errDateDeleted != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupValue("date deletion", backupData.Timestamp, errDateDeleted)) + } + // If the backup date deletion has invalid value, try to delete the backup. + if gpbckpconfig.IsBackupActive(backupDateDeleted) || errDateDeleted != nil { + result = true + } else { + if backupDateDeleted == gpbckpconfig.DateDeletedInProgress { + // We do not return the error here, + // because it is necessary to leave the possibility of starting the process + // of deleting backups that are stuck in the "In Progress" status using the --force flag. + gplog.Error("%s", textmsg.ErrorTextBackupDeleteInProgress(backupData.Timestamp, textmsg.ErrorBackupDeleteInProgressError())) + } else { + gplog.Debug("%s", textmsg.InfoTextBackupAlreadyDeleted(backupData.Timestamp)) + } + } + // If flag --force is set. + if deleteForce { + result = true + } + return result, nil +} + +// Check that specified backup type is supported. +func checkBackupType(backupType string) error { + var validVType = map[string]bool{ + gpbckpconfig.BackupTypeFull: true, + gpbckpconfig.BackupTypeIncremental: true, + gpbckpconfig.BackupTypeMetadataOnly: true, + gpbckpconfig.BackupTypeDataOnly: true, + } + if !validVType[backupType] { + return textmsg.ErrorInvalidValueError() + } + return nil +} + +// Check skip flag and local backup status. +// SkipLocalBackup - true, local backup - true, returns "is a local backup" error. +// SkipLocalBackup - false,local backup - false, returns "is not a local backup" error. +func checkLocalBackupStatus(skipLocalBackup, isLocalBackup bool) error { + if skipLocalBackup && isLocalBackup { + return textmsg.ErrorBackupLocalStorageError() + } + if !skipLocalBackup && !isLocalBackup { + return textmsg.ErrorBackupNotLocalStorageError() + } + return nil +} + +func getBackupMasterDir(backupDir, backupDataBackupDir, backupDataDBName string) (string, string, bool, error) { + if backupDir != "" { + return gpbckpconfig.CheckMasterBackupDir(backupDir) + } + if backupDataBackupDir != "" { + return gpbckpconfig.CheckMasterBackupDir(backupDataBackupDir) + } + // Try to get the backup directory from the cluster configuration. + // If the script executed not on the master host, the backup directory will not be found. + // And we return "value not set" error. + backupDirClusterInfo := getBackupMasterDirClusterInfo(backupDataDBName) + if backupDirClusterInfo != "" { + return backupDirClusterInfo, gpbckpconfig.GetSegPrefix(filepath.Join(backupDirClusterInfo, "backups")), false, nil + } + return "", "", false, textmsg.ErrorValidationValue() +} + +func getBackupSegmentDir(backupDir, backupDataBackupDir, backupDataDir, segPrefix, segID string, isSingleBackupDir bool) (string, error) { + if backupDir != "" { + return checkSingleBackupDir(backupDir, segPrefix, segID, isSingleBackupDir), nil + } + if backupDataBackupDir != "" { + return checkSingleBackupDir(backupDataBackupDir, segPrefix, segID, isSingleBackupDir), nil + } + if backupDataDir != "" { + return backupDataDir, nil + } + return "", textmsg.ErrorValidationValue() +} + +func checkSingleBackupDir(backupDir, segPrefix, segID string, isSingleBackupDir bool) string { + if isSingleBackupDir { + return backupDir + } + return filepath.Join(backupDir, fmt.Sprintf("%s%s", segPrefix, segID)) +} + +func getBackupMasterDirClusterInfo(dbName string) string { + db, err := gpbckpconfig.NewClusterLocalClusterConn(dbName) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableConnectLocalCluster(err)) + return "" + } + defer db.Close() + sqlQuery := "SELECT datadir FROM gp_segment_configuration WHERE content = -1 AND role = 'p';" + queryResult, err := gpbckpconfig.ExecuteQueryLocalClusterConn[string](db, sqlQuery) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupDirLocalClusterConn(err)) + return "" + } + gplog.Debug("Master data directory: %s", queryResult) + return queryResult +} + +func getSegmentConfigurationClusterInfo(dbName string) ([]gpbckpconfig.SegmentConfig, error) { + queryResult := make([]gpbckpconfig.SegmentConfig, 0) + db, err := gpbckpconfig.NewClusterLocalClusterConn(dbName) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableConnectLocalCluster(err)) + return queryResult, err + } + defer db.Close() + sqlQuery := "SELECT content as contentid, hostname, datadir FROM gp_segment_configuration WHERE role = 'p' and content != -1 ORDER BY content;" + queryResult, err = gpbckpconfig.ExecuteQueryLocalClusterConn[[]gpbckpconfig.SegmentConfig](db, sqlQuery) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableGetBackupDirLocalClusterConn(err)) + return queryResult, err + } + return queryResult, nil +} + +func handleErrorDB(backupName, errorMessage, backupStatus string, hDB *sql.DB) { + gplog.Error("%s", errorMessage) + err := gpbckpconfig.UpdateDeleteStatus(backupName, backupStatus, hDB) + if err != nil { + gplog.Error("%s", textmsg.ErrorTextUnableSetBackupStatus(backupStatus, backupName, err)) + } +} diff --git a/gpbackman/cmd/wrappers_test.go b/gpbackman/cmd/wrappers_test.go new file mode 100644 index 00000000..ddd4634a --- /dev/null +++ b/gpbackman/cmd/wrappers_test.go @@ -0,0 +1,373 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/apache/cloudberry-backup/gpbackman/gpbckpconfig" + "github.com/apache/cloudberry-backup/history" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/spf13/pflag" +) + +var _ = Describe("wrappers tests", func() { + Describe("getHistoryDBPath", func() { + It("returns default path when input is empty", func() { + Expect(getHistoryDBPath("")).To(Equal(historyDBNameConst)) + }) + + It("returns input path when not empty", func() { + Expect(getHistoryDBPath("path/to/" + historyDBNameConst)).To(Equal("path/to/" + historyDBNameConst)) + }) + }) + + Describe("formatBackupDuration", func() { + It("formats duration correctly", func() { + tests := []struct { + name string + value float64 + want string + }{ + {"01:00:00", 3600, "01:00:00"}, + {"01:01:01", 3661, "01:01:01"}, + {"00:00:00", 0, "00:00:00"}, + } + for _, tt := range tests { + Expect(formatBackupDuration(tt.value)).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("getCurrentTimestamp", func() { + It("returns valid timestamp", func() { + result := getCurrentTimestamp() + _, err := time.Parse(gpbckpconfig.Layout, result) + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Describe("checkCompatibleFlags", func() { + It("does not return error when no flags changed", func() { + flags := pflag.NewFlagSet("test", pflag.ContinueOnError) + Expect(checkCompatibleFlags(flags)).To(Succeed()) + }) + + It("does not return error when one flag changed", func() { + flags := pflag.NewFlagSet("test", pflag.ContinueOnError) + flags.String("flag1", "", "") + flags.Set("flag1", "") + Expect(checkCompatibleFlags(flags, "flag1")).To(Succeed()) + }) + + It("returns error when multiple flags changed", func() { + flags := pflag.NewFlagSet("test", pflag.ContinueOnError) + flags.String("flag1", "", "") + flags.String("flag2", "", "") + flags.Set("flag1", "") + flags.Set("flag2", "") + err := checkCompatibleFlags(flags, "flag1", "flag2") + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("checkBackupCanBeUsed", func() { + It("returns correct result for various backup configurations", func() { + tests := []struct { + name string + deleteForce bool + skipLocalBackup bool + backupConfig history.BackupConfig + want bool + wantErr bool + }{ + { + name: "successful backup with plugin and force, skipLocalBackup true", + deleteForce: true, + skipLocalBackup: true, + backupConfig: history.BackupConfig{ + Status: history.BackupStatusSucceed, + Plugin: gpbckpconfig.BackupS3Plugin, + }, + want: true, + }, + { + name: "successful backup with plugin and without force", + skipLocalBackup: true, + backupConfig: history.BackupConfig{ + Status: history.BackupStatusSucceed, + Plugin: gpbckpconfig.BackupS3Plugin, + }, + want: true, + }, + { + name: "failed backup with plugin and force", + deleteForce: true, + skipLocalBackup: true, + backupConfig: history.BackupConfig{ + Status: history.BackupStatusFailed, + Plugin: gpbckpconfig.BackupS3Plugin, + }, + want: true, + }, + { + name: "failed backup with plugin and without force", + skipLocalBackup: true, + backupConfig: history.BackupConfig{ + Status: history.BackupStatusFailed, + Plugin: gpbckpconfig.BackupS3Plugin, + }, + want: true, + }, + { + name: "successful backup without plugin and force", + deleteForce: true, + backupConfig: history.BackupConfig{ + Status: history.BackupStatusSucceed, + }, + want: true, + }, + { + name: "successful backup without plugin and without force", + backupConfig: history.BackupConfig{ + Status: history.BackupStatusSucceed, + }, + want: true, + }, + { + name: "successful deleted backup with plugin and force", + deleteForce: true, + skipLocalBackup: true, + backupConfig: history.BackupConfig{ + Status: history.BackupStatusSucceed, + Plugin: gpbckpconfig.BackupS3Plugin, + DateDeleted: "20240113210000", + }, + want: true, + }, + { + name: "successful deleted backup with plugin and without force", + skipLocalBackup: true, + backupConfig: history.BackupConfig{ + Status: history.BackupStatusSucceed, + Plugin: gpbckpconfig.BackupS3Plugin, + DateDeleted: "20240113210000", + }, + want: false, + }, + { + name: "invalid backup status with plugin and without force", + skipLocalBackup: true, + backupConfig: history.BackupConfig{ + Status: "some_status", + Plugin: gpbckpconfig.BackupS3Plugin, + }, + want: true, + }, + { + name: "successful backup with plugin with deletion in progress and force", + deleteForce: true, + skipLocalBackup: true, + backupConfig: history.BackupConfig{ + Status: history.BackupStatusSucceed, + Plugin: gpbckpconfig.BackupS3Plugin, + DateDeleted: gpbckpconfig.DateDeletedInProgress, + }, + want: true, + }, + { + name: "successful backup with plugin with deletion in progress and without force", + skipLocalBackup: true, + backupConfig: history.BackupConfig{ + Status: history.BackupStatusSucceed, + Plugin: gpbckpconfig.BackupS3Plugin, + DateDeleted: gpbckpconfig.DateDeletedInProgress, + }, + want: false, + }, + { + name: "successful backup with plugin with invalid deletion date and without force", + skipLocalBackup: true, + backupConfig: history.BackupConfig{ + Status: history.BackupStatusSucceed, + Plugin: gpbckpconfig.BackupS3Plugin, + DateDeleted: "some date", + }, + want: true, + }, + { + name: "successful backup with plugin with invalid skipLocalBackup variable", + backupConfig: history.BackupConfig{ + Status: history.BackupStatusSucceed, + Plugin: gpbckpconfig.BackupS3Plugin, + DateDeleted: "some date", + }, + wantErr: true, + }, + { + name: "successful backup without plugin with invalid skipLocalBackup variable", + skipLocalBackup: true, + backupConfig: history.BackupConfig{ + Status: history.BackupStatusSucceed, + DateDeleted: "some date", + }, + wantErr: true, + }, + } + for _, tt := range tests { + got, err := checkBackupCanBeUsed(tt.deleteForce, tt.skipLocalBackup, &tt.backupConfig) + if tt.wantErr { + Expect(err).To(HaveOccurred(), tt.name) + } else { + Expect(err).ToNot(HaveOccurred(), tt.name) + Expect(got).To(Equal(tt.want), tt.name) + } + } + }) + }) + + Describe("checkBackupType", func() { + It("accepts valid backup type", func() { + Expect(checkBackupType(gpbckpconfig.BackupTypeFull)).To(Succeed()) + }) + + It("rejects invalid backup type", func() { + Expect(checkBackupType("InvalidType")).To(HaveOccurred()) + }) + }) + + Describe("getBackupMasterDir", func() { + It("returns correct values for various backup dirs", func() { + tempDir, err := os.MkdirTemp("", "gpbackman-test-") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tempDir) + + tests := []struct { + name string + testDir string + backupDir string + backupDataBackupDir string + backupDataDBName string + wantBackupMasterDir string + wantSegPrefix string + wantIsSingleBackupDir bool + wantErr bool + }{ + { + name: "backupDir is set and valid", + testDir: filepath.Join(tempDir, "segPrefix", "segment-1", "backups"), + backupDir: filepath.Join(tempDir, "segPrefix"), + wantBackupMasterDir: filepath.Join(tempDir, "segPrefix", "segment-1"), + wantSegPrefix: "segment", + wantIsSingleBackupDir: false, + }, + { + name: "backupDataBackupDir is set and valid", + testDir: filepath.Join(tempDir, "segPrefix2", "segment-1", "backups"), + backupDataBackupDir: filepath.Join(tempDir, "segPrefix2"), + wantBackupMasterDir: filepath.Join(tempDir, "segPrefix2", "segment-1"), + wantSegPrefix: "segment", + wantIsSingleBackupDir: false, + }, + } + for _, tt := range tests { + err := os.MkdirAll(tt.testDir, 0o755) + Expect(err).ToNot(HaveOccurred()) + gotBackupMasterDir, gotSegPrefix, gotIsSingleBackupDir, err := getBackupMasterDir(tt.backupDir, tt.backupDataBackupDir, tt.backupDataDBName) + if tt.wantErr { + Expect(err).To(HaveOccurred(), tt.name) + } else { + Expect(err).ToNot(HaveOccurred(), tt.name) + Expect(gotBackupMasterDir).To(Equal(tt.wantBackupMasterDir), tt.name) + Expect(gotSegPrefix).To(Equal(tt.wantSegPrefix), tt.name) + Expect(gotIsSingleBackupDir).To(Equal(tt.wantIsSingleBackupDir), tt.name) + } + } + }) + }) + + Describe("checkSingleBackupDir", func() { + It("returns backupDir when isSingleBackupDir is true", func() { + got := checkSingleBackupDir("/path/to/backup", "seg", "1", true) + Expect(got).To(Equal("/path/to/backup")) + }) + + It("returns composed path when isSingleBackupDir is false", func() { + got := checkSingleBackupDir("/path/to/backup", "seg", "1", false) + Expect(got).To(Equal(filepath.Join("/path/to/backup", fmt.Sprintf("%s%s", "seg", "1")))) + }) + }) + + Describe("getBackupSegmentDir", func() { + It("returns correct segment dir for various inputs", func() { + tests := []struct { + name string + backupDir string + backupDataBackupDir string + backupDataDir string + isSingleBackupDir bool + want string + wantErr bool + }{ + { + name: "backupDir is not empty", + backupDir: "/path/to/backupDir", + isSingleBackupDir: true, + want: "/path/to/backupDir", + }, + { + name: "backupDataBackupDir is not empty", + backupDataBackupDir: "/path/to/backupDataBackupDir", + isSingleBackupDir: true, + want: "/path/to/backupDataBackupDir", + }, + { + name: "backupDataDir is not empty", + backupDataDir: "/path/to/backupDataDir", + isSingleBackupDir: true, + want: "/path/to/backupDataDir", + }, + { + name: "all backup directories are empty", + isSingleBackupDir: true, + wantErr: true, + }, + } + for _, tt := range tests { + got, err := getBackupSegmentDir(tt.backupDir, tt.backupDataBackupDir, tt.backupDataDir, "seg", "1", tt.isSingleBackupDir) + if tt.wantErr { + Expect(err).To(HaveOccurred(), tt.name) + } else { + Expect(err).ToNot(HaveOccurred(), tt.name) + Expect(got).To(Equal(tt.want), tt.name) + } + } + }) + }) + + Describe("checkLocalBackupStatus", func() { + It("returns correct result for various inputs", func() { + tests := []struct { + name string + skipLocalBackup bool + isLocalBackup bool + wantErr bool + }{ + {"skip local and local backup", true, true, true}, + {"skip local and plugin backup", true, false, false}, + {"do not skip local and local backup", false, true, false}, + {"do not skip local and plugin backup", false, false, true}, + } + for _, tt := range tests { + err := checkLocalBackupStatus(tt.skipLocalBackup, tt.isLocalBackup) + if tt.wantErr { + Expect(err).To(HaveOccurred(), tt.name) + } else { + Expect(err).ToNot(HaveOccurred(), tt.name) + } + } + }) + }) +}) diff --git a/gpbackman/gpbckpconfig/cluster.go b/gpbackman/gpbckpconfig/cluster.go new file mode 100644 index 00000000..ed59370f --- /dev/null +++ b/gpbackman/gpbckpconfig/cluster.go @@ -0,0 +1,79 @@ +package gpbckpconfig + +import ( + "fmt" + "strconv" + + "github.com/apache/cloudberry-backup/gpbackman/textmsg" + "github.com/apache/cloudberry-go-libs/operating" + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" +) + +type SegmentConfig struct { + ContentID string + Hostname string + DataDir string +} + +// NewClusterLocalClusterConn creates a new connection to the local postgres database. +// Returns an error if the connection could not be established. +func NewClusterLocalClusterConn(dbName string) (*sqlx.DB, error) { + if dbName == "" { + return nil, textmsg.ErrorEmptyDatabase() + } + username := operating.System.Getenv("PGUSER") + if username == "" { + currentUser, _ := operating.System.CurrentUser() + username = currentUser.Username + } + host := operating.System.Getenv("PGHOST") + if host == "" { + host, _ = operating.System.Hostname() + } + port, err := strconv.Atoi(operating.System.Getenv("PGPORT")) + if err != nil { + port = 5432 + } + connStr := fmt.Sprintf("postgres://%s@%s:%d/%s?sslmode=disable&connect_timeout=60", username, host, port, dbName) + return sqlx.Connect("postgres", connStr) +} + +// ExecuteQueryLocalClusterConn executes a query on the local cluster connection and returns the result. +// The function is generic and can handle different types of results based on the type parameter T. +// +// Parameters: +// - conn: A pointer to the sqlx.DB connection object. +// - query: A string containing the SQL query to be executed. +// +// Returns: +// - T: The result of the query, which can be of any type specified by the caller. +// - error: An error object if the query execution fails or if the type is unsupported. +// +// The function supports the following types for T: +// - string: The result will be a single string value. +// - []SegmentConfig: The result will be a slice of SegmentConfig structs. +// +// If the type T is not supported, the function returns an error indicating the unsupported type. +func ExecuteQueryLocalClusterConn[T any](conn *sqlx.DB, query string) (T, error) { + var result T + switch any(result).(type) { + case string: + var data string + err := conn.Get(&data, query) + if err != nil { + return result, err + } + result = any(data).(T) + case []SegmentConfig: + var segConfigs []SegmentConfig + err := conn.Select(&segConfigs, query) + if err != nil { + return result, err + } + result = any(segConfigs).(T) + default: + return result, fmt.Errorf("unsupported type") + } + return result, nil +} diff --git a/gpbackman/gpbckpconfig/gpbckpconfig_suite_test.go b/gpbackman/gpbckpconfig/gpbckpconfig_suite_test.go new file mode 100644 index 00000000..d8602978 --- /dev/null +++ b/gpbackman/gpbckpconfig/gpbckpconfig_suite_test.go @@ -0,0 +1,13 @@ +package gpbckpconfig + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestGpbckpconfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Gpbckpconfig Suite") +} diff --git a/gpbackman/gpbckpconfig/helpers.go b/gpbackman/gpbckpconfig/helpers.go new file mode 100644 index 00000000..81c1da6b --- /dev/null +++ b/gpbackman/gpbckpconfig/helpers.go @@ -0,0 +1,234 @@ +package gpbckpconfig + +import ( + "errors" + "strings" + "time" + + "github.com/apache/cloudberry-backup/history" +) + +// GetBackupType Get backup type. +// The value is calculated, based on: +// - full - contains user data, all global and local metadata for the database; +// - incremental – contains user data, all global and local metadata changed since a previous full backup; +// - metadata-only – contains only global and local metadata for the database; +// - data-only – contains only user data from the database. +// +// In all other cases, an error is returned. +func GetBackupType(backupConfig *history.BackupConfig) (string, error) { + // For gpbackup you cannot combine --data-only or --metadata-only with --incremental (see docs). + // So these options cannot be set at the same time. + // If not one of the --data-only, --metadata-only and --incremental flags is not set, + // the full value is returned. + // But if there are no tables in backup set contain data, + // the metadata-only value is returned. + // See https://github.com/woblerr/gpbackup/blob/b061a47b673238439442340e66ca57d896edacd5/backup/backup.go#L127-L129 + switch { + case !backupConfig.Incremental && !backupConfig.DataOnly && !backupConfig.MetadataOnly: + return BackupTypeFull, nil + case backupConfig.Incremental && !backupConfig.DataOnly && !backupConfig.MetadataOnly: + return BackupTypeIncremental, nil + case backupConfig.DataOnly && !backupConfig.Incremental && !backupConfig.MetadataOnly: + return BackupTypeDataOnly, nil + // If only metadata-only value. + // Or combination metadata-only and incremental or metadata-only and data-only. + // The case when there are no tables in backup set contain data. + case (backupConfig.MetadataOnly && !backupConfig.Incremental) || (backupConfig.MetadataOnly && !backupConfig.DataOnly): + return BackupTypeMetadataOnly, nil + default: + return "", errors.New("backup type does not match any of the available values") + } +} + +// GetObjectFilteringInfo Get object filtering information. +// The value is calculated, base on whether at least one of the flags was specified: +// - include-schema – at least one "--include-schema" option was specified; +// - exclude-schema – at least one "--exclude-schema" option was specified; +// - include-table – at least one "--include-table" option was specified; +// - exclude-table – at least one "--exclude-table" option was specified; +// - "" - no options was specified. +// +// For gpbackup only one type of filters can be used (see docs). +// So these options cannot be set at the same time. +// If not one of these flags is not set, +// the "" value is returned. +// In all other cases, an error is returned. +func GetObjectFilteringInfo(backupConfig *history.BackupConfig) (string, error) { + switch { + case backupConfig.IncludeSchemaFiltered && + !backupConfig.ExcludeSchemaFiltered && + !backupConfig.IncludeTableFiltered && + !backupConfig.ExcludeTableFiltered: + return objectFilteringIncludeSchema, nil + case backupConfig.ExcludeSchemaFiltered && + !backupConfig.IncludeSchemaFiltered && + !backupConfig.IncludeTableFiltered && + !backupConfig.ExcludeTableFiltered: + return objectFilteringExcludeSchema, nil + case backupConfig.IncludeTableFiltered && + !backupConfig.IncludeSchemaFiltered && + !backupConfig.ExcludeSchemaFiltered && + !backupConfig.ExcludeTableFiltered: + return objectFilteringIncludeTable, nil + case backupConfig.ExcludeTableFiltered && + !backupConfig.IncludeSchemaFiltered && + !backupConfig.ExcludeSchemaFiltered && + !backupConfig.IncludeTableFiltered: + return objectFilteringExcludeTable, nil + case !backupConfig.ExcludeTableFiltered && + !backupConfig.IncludeSchemaFiltered && + !backupConfig.ExcludeSchemaFiltered && + !backupConfig.IncludeTableFiltered: + return "", nil + default: + return "", errors.New("backup filtering type does not match any of the available values") + } +} + +// GetObjectFilteringDetails returns a comma-separated string with object filtering details +// depending on the active filtering type. If no filtering is active, it returns an empty string. +func GetObjectFilteringDetails(backupConfig *history.BackupConfig) string { + filter, _ := GetObjectFilteringInfo(backupConfig) + switch filter { + case objectFilteringIncludeTable: + return strings.Join(backupConfig.IncludeRelations, ", ") + case objectFilteringExcludeTable: + return strings.Join(backupConfig.ExcludeRelations, ", ") + case objectFilteringIncludeSchema: + return strings.Join(backupConfig.IncludeSchemas, ", ") + case objectFilteringExcludeSchema: + return strings.Join(backupConfig.ExcludeSchemas, ", ") + default: + return "" + } +} + +// GetBackupDate Get backup date. +// If an error occurs when parsing the date, the empty string and error are returned. +func GetBackupDate(backupConfig *history.BackupConfig) (string, error) { + var date string + t, err := time.Parse(Layout, backupConfig.Timestamp) + if err != nil { + return date, err + } + date = t.Format(DateFormat) + return date, nil +} + +// GetBackupDuration Get backup duration in seconds. +// If an error occurs when parsing the date, the zero duration and error are returned. +func GetBackupDuration(backupConfig *history.BackupConfig) (float64, error) { + var zeroDuration float64 + startTime, err := time.Parse(Layout, backupConfig.Timestamp) + if err != nil { + return zeroDuration, err + } + endTime, err := time.Parse(Layout, backupConfig.EndTime) + if err != nil { + return zeroDuration, err + } + return endTime.Sub(startTime).Seconds(), nil +} + +// GetBackupDateDeleted Get backup deletion date or backup deletion status. +// The possible values are: +// - In progress - if the value is set to "In progress"; +// - Plugin Backup Delete Failed - if the value is set to "Plugin Backup Delete Failed"; +// - Local Delete Failed - if the value is set to "Local Delete Failed"; +// - "" - if backup is active; +// - date in format "Mon Jan 02 2006 15:04:05" - if backup is deleted and deletion timestamp is set. +// +// In all other cases, an error is returned. +func GetBackupDateDeleted(backupConfig *history.BackupConfig) (string, error) { + switch backupConfig.DateDeleted { + case "", DateDeletedInProgress, DateDeletedPluginFailed, DateDeletedLocalFailed: + return backupConfig.DateDeleted, nil + default: + t, err := time.Parse(Layout, backupConfig.DateDeleted) + if err != nil { + return backupConfig.DateDeleted, err + } + return t.Format(DateFormat), nil + } +} + +// IsSuccess Check backup status. +// Returns: +// - true - if backup is successful, +// - false - false if backup is not successful or in progress. +// +// In all other cases, an error is returned. +func IsSuccess(backupConfig *history.BackupConfig) (bool, error) { + switch backupConfig.Status { + case history.BackupStatusSucceed: + return true, nil + case history.BackupStatusFailed, history.BackupStatusInProgress: + return false, nil + default: + return false, errors.New("backup status does not match any of the available values") + } +} + +// IsLocal Check if the backup in local or in plugin storage. +// Returns: +// - true - if the backup in local storage (plugin field is empty); +// - false - if the backup in plugin storage (plugin field is not empty). +func IsLocal(backupConfig *history.BackupConfig) bool { + return backupConfig.Plugin == "" +} + +// IsInProgress Check if the backup is in progress. +func IsInProgress(backupConfig *history.BackupConfig) bool { + return backupConfig.Status == history.BackupStatusInProgress +} + +// GetReportFilePathPlugin Return path to report file name for specific plugin. +// If custom report path is set, it is returned. +// Otherwise, the path from plugin is returned. +func GetReportFilePathPlugin(backupConfig *history.BackupConfig, customReportPath string, pluginOptions map[string]string) (string, error) { + if customReportPath != "" { + return backupPluginCustomReportPath(backupConfig.Timestamp, customReportPath), nil + } + // In future another plugins may be added. + switch backupConfig.Plugin { + case BackupS3Plugin: + return backupS3PluginReportPath(backupConfig.Timestamp, pluginOptions) + default: + // nothing to do + } + return "", errors.New("the path to the report is not specified") +} + +// CheckObjectFilteringExists checks if the object filtering exists in the backup. +// +// This function is responsible for determining whether table or schema filtering exists in the backup, and if so, whether the specified filter type is being used. +// Returns: +// - true - if table or schema filtering exists in the backup or no filters are specified; +// - false - if table or schema filtering does not exists in the backup. +func CheckObjectFilteringExists(backupConfig *history.BackupConfig, tableFilter, schemaFilter, objectFilter string, excludeFilter bool) bool { + switch { + case tableFilter != "" && !excludeFilter: + if objectFilter == objectFilteringIncludeTable { + return searchFilter(backupConfig.IncludeRelations, tableFilter) + } + return false + case tableFilter != "" && excludeFilter: + if objectFilter == objectFilteringExcludeTable { + return searchFilter(backupConfig.ExcludeRelations, tableFilter) + } + return false + case schemaFilter != "" && !excludeFilter: + if objectFilter == objectFilteringIncludeSchema { + return searchFilter(backupConfig.IncludeSchemas, schemaFilter) + } + return false + case schemaFilter != "" && excludeFilter: + if objectFilter == objectFilteringExcludeSchema { + return searchFilter(backupConfig.ExcludeSchemas, schemaFilter) + } + return false + default: + return true + } +} diff --git a/gpbackman/gpbckpconfig/helpers_test.go b/gpbackman/gpbckpconfig/helpers_test.go new file mode 100644 index 00000000..8065daa1 --- /dev/null +++ b/gpbackman/gpbckpconfig/helpers_test.go @@ -0,0 +1,558 @@ +package gpbckpconfig + +import ( + "github.com/apache/cloudberry-backup/history" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("helpers tests", func() { + Describe("GetBackupType", func() { + It("returns correct backup type", func() { + tests := []struct { + name string + config history.BackupConfig + want string + wantErr bool + }{ + { + name: "incremental backup", + config: history.BackupConfig{Incremental: true}, + want: BackupTypeIncremental, + wantErr: false, + }, + { + name: "data-only backup", + config: history.BackupConfig{DataOnly: true}, + want: BackupTypeDataOnly, + wantErr: false, + }, + { + name: "metadata-only backup", + config: history.BackupConfig{MetadataOnly: true}, + want: BackupTypeMetadataOnly, + wantErr: false, + }, + { + name: "metadata-only when data-only also set", + config: history.BackupConfig{DataOnly: true, MetadataOnly: true}, + want: BackupTypeMetadataOnly, + wantErr: false, + }, + { + name: "metadata-only when incremental also set", + config: history.BackupConfig{Incremental: true, MetadataOnly: true}, + want: BackupTypeMetadataOnly, + wantErr: false, + }, + { + name: "full backup", + config: history.BackupConfig{ + Incremental: false, + DataOnly: false, + MetadataOnly: false, + }, + want: BackupTypeFull, + wantErr: false, + }, + { + name: "invalid backup case 1", + config: history.BackupConfig{Incremental: true, DataOnly: true}, + want: "", + wantErr: true, + }, + { + name: "invalid backup case 2", + config: history.BackupConfig{Incremental: true, DataOnly: true, MetadataOnly: true}, + want: "", + wantErr: true, + }, + } + for _, tt := range tests { + cfg := tt.config + got, err := GetBackupType(&cfg) + if tt.wantErr { + Expect(err).To(HaveOccurred(), tt.name) + } else { + Expect(err).ToNot(HaveOccurred(), tt.name) + } + Expect(got).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("GetObjectFilteringInfo", func() { + It("returns correct filtering info", func() { + tests := []struct { + name string + config history.BackupConfig + want string + wantErr bool + }{ + { + name: "IncludeSchemaFiltered", + config: history.BackupConfig{IncludeSchemaFiltered: true}, + want: objectFilteringIncludeSchema, + }, + { + name: "ExcludeSchemaFiltered", + config: history.BackupConfig{ExcludeSchemaFiltered: true}, + want: objectFilteringExcludeSchema, + }, + { + name: "IncludeTableFiltered", + config: history.BackupConfig{IncludeTableFiltered: true}, + want: objectFilteringIncludeTable, + }, + { + name: "ExcludeTableFiltered", + config: history.BackupConfig{ExcludeTableFiltered: true}, + want: objectFilteringExcludeTable, + }, + { + name: "NoFiltering", + config: history.BackupConfig{}, + want: "", + }, + { + name: "Invalid IncludeTable and ExcludeTable", + config: history.BackupConfig{IncludeTableFiltered: true, ExcludeTableFiltered: true}, + wantErr: true, + }, + { + name: "Invalid IncludeSchema and ExcludeSchema", + config: history.BackupConfig{IncludeSchemaFiltered: true, ExcludeSchemaFiltered: true}, + wantErr: true, + }, + { + name: "Invalid IncludeSchema and IncludeTable", + config: history.BackupConfig{IncludeSchemaFiltered: true, IncludeTableFiltered: true}, + wantErr: true, + }, + { + name: "Invalid IncludeSchema and ExcludeTable", + config: history.BackupConfig{IncludeSchemaFiltered: true, ExcludeTableFiltered: true}, + wantErr: true, + }, + { + name: "Invalid ExcludeSchema and IncludeTable", + config: history.BackupConfig{ExcludeSchemaFiltered: true, IncludeTableFiltered: true}, + wantErr: true, + }, + { + name: "Invalid ExcludeSchema and ExcludeTable", + config: history.BackupConfig{ExcludeSchemaFiltered: true, ExcludeTableFiltered: true}, + wantErr: true, + }, + { + name: "Invalid IncludeSchema IncludeTable and ExcludeTable", + config: history.BackupConfig{IncludeSchemaFiltered: true, IncludeTableFiltered: true, ExcludeTableFiltered: true}, + wantErr: true, + }, + { + name: "Invalid IncludeSchema ExcludeSchema and IncludeTable", + config: history.BackupConfig{IncludeSchemaFiltered: true, ExcludeSchemaFiltered: true, IncludeTableFiltered: true}, + wantErr: true, + }, + { + name: "Invalid IncludeSchema ExcludeSchema and ExcludeTable", + config: history.BackupConfig{IncludeSchemaFiltered: true, ExcludeSchemaFiltered: true, ExcludeTableFiltered: true}, + wantErr: true, + }, + { + name: "Invalid all filters set", + config: history.BackupConfig{IncludeSchemaFiltered: true, ExcludeSchemaFiltered: true, IncludeTableFiltered: true, ExcludeTableFiltered: true}, + wantErr: true, + }, + } + for _, tt := range tests { + cfg := tt.config + got, err := GetObjectFilteringInfo(&cfg) + if tt.wantErr { + Expect(err).To(HaveOccurred(), tt.name) + } else { + Expect(err).ToNot(HaveOccurred(), tt.name) + Expect(got).To(Equal(tt.want), tt.name) + } + } + }) + }) + + Describe("GetObjectFilteringDetails", func() { + It("returns correct filtering details", func() { + tests := []struct { + name string + config history.BackupConfig + want string + }{ + { + name: "IncludeTable details", + config: history.BackupConfig{ + IncludeTableFiltered: true, + IncludeRelations: []string{"public.t1", "s.t2"}, + }, + want: "public.t1, s.t2", + }, + { + name: "ExcludeTable details", + config: history.BackupConfig{ + ExcludeTableFiltered: true, + ExcludeRelations: []string{"public.t3"}, + }, + want: "public.t3", + }, + { + name: "IncludeSchema details", + config: history.BackupConfig{ + IncludeSchemaFiltered: true, + IncludeSchemas: []string{"public", "sales"}, + }, + want: "public, sales", + }, + { + name: "ExcludeSchema details", + config: history.BackupConfig{ + ExcludeSchemaFiltered: true, + ExcludeSchemas: []string{"tmp"}, + }, + want: "tmp", + }, + { + name: "No filtering", + config: history.BackupConfig{}, + want: "", + }, + } + for _, tt := range tests { + cfg := tt.config + got := GetObjectFilteringDetails(&cfg) + Expect(got).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("GetBackupDate", func() { + It("parses valid timestamp", func() { + cfg := history.BackupConfig{Timestamp: "20220401102430"} + got, err := GetBackupDate(&cfg) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal("Fri Apr 01 2022 10:24:30")) + }) + + It("returns error for invalid timestamp", func() { + cfg := history.BackupConfig{Timestamp: "invalid"} + _, err := GetBackupDate(&cfg) + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("GetBackupDuration", func() { + It("calculates duration correctly", func() { + cfg := history.BackupConfig{ + Timestamp: "20220401102430", + EndTime: "20220401115502", + } + got, err := GetBackupDuration(&cfg) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal(float64(5432))) + }) + + It("returns error for invalid start timestamp", func() { + cfg := history.BackupConfig{ + Timestamp: "invalid", + EndTime: "20220401115502", + } + _, err := GetBackupDuration(&cfg) + Expect(err).To(HaveOccurred()) + }) + + It("returns error for invalid end timestamp", func() { + cfg := history.BackupConfig{ + Timestamp: "20220401102430", + EndTime: "invalid", + } + _, err := GetBackupDuration(&cfg) + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("GetBackupDateDeleted", func() { + It("returns correct date deleted", func() { + tests := []struct { + name string + config history.BackupConfig + want string + wantErr bool + }{ + { + name: "empty", + config: history.BackupConfig{DateDeleted: ""}, + want: "", + }, + { + name: "in progress", + config: history.BackupConfig{DateDeleted: DateDeletedInProgress}, + want: DateDeletedInProgress, + }, + { + name: "plugin backup delete failed", + config: history.BackupConfig{DateDeleted: DateDeletedPluginFailed}, + want: DateDeletedPluginFailed, + }, + { + name: "local delete failed", + config: history.BackupConfig{DateDeleted: DateDeletedLocalFailed}, + want: DateDeletedLocalFailed, + }, + { + name: "valid date", + config: history.BackupConfig{DateDeleted: "20220401102430"}, + want: "Fri Apr 01 2022 10:24:30", + }, + { + name: "invalid date", + config: history.BackupConfig{DateDeleted: "InvalidDate"}, + want: "InvalidDate", + wantErr: true, + }, + } + for _, tt := range tests { + cfg := tt.config + got, err := GetBackupDateDeleted(&cfg) + if tt.wantErr { + Expect(err).To(HaveOccurred(), tt.name) + } else { + Expect(err).ToNot(HaveOccurred(), tt.name) + } + Expect(got).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("IsSuccess", func() { + It("returns true for success status", func() { + cfg := history.BackupConfig{Status: history.BackupStatusSucceed} + got, err := IsSuccess(&cfg) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(BeTrue()) + }) + + It("returns false for failure status", func() { + cfg := history.BackupConfig{Status: history.BackupStatusFailed} + got, err := IsSuccess(&cfg) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(BeFalse()) + }) + + It("returns error for unknown status", func() { + cfg := history.BackupConfig{Status: "unknown"} + _, err := IsSuccess(&cfg) + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("IsLocal", func() { + It("returns true when plugin is empty", func() { + cfg := history.BackupConfig{Plugin: ""} + Expect(IsLocal(&cfg)).To(BeTrue()) + }) + + It("returns false when plugin is set", func() { + cfg := history.BackupConfig{Plugin: "plugin"} + Expect(IsLocal(&cfg)).To(BeFalse()) + }) + }) + + Describe("IsInProgress", func() { + It("returns correct result for various statuses", func() { + tests := []struct { + name string + status string + want bool + }{ + {"in progress", history.BackupStatusInProgress, true}, + {"success", history.BackupStatusSucceed, false}, + {"failure", history.BackupStatusFailed, false}, + {"empty", "", false}, + {"unknown", "unknown", false}, + } + for _, tt := range tests { + cfg := history.BackupConfig{Status: tt.status} + Expect(IsInProgress(&cfg)).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("GetReportFilePathPlugin", func() { + It("returns correct report path", func() { + tests := []struct { + name string + config history.BackupConfig + customReportPath string + pluginOptions map[string]string + want string + wantErr bool + }{ + { + name: "custom report path", + config: history.BackupConfig{ + Timestamp: "20220401102430", + Plugin: BackupS3Plugin, + }, + customReportPath: "/path/to/report", + pluginOptions: make(map[string]string), + want: "/path/to/report/gpbackup_20220401102430_report", + }, + { + name: "s3 plugin folder absent", + config: history.BackupConfig{ + Timestamp: "20220401102430", + Plugin: BackupS3Plugin, + }, + pluginOptions: map[string]string{"bucket": "bucket"}, + wantErr: true, + }, + { + name: "s3 plugin folder empty", + config: history.BackupConfig{ + Timestamp: "20220401102430", + Plugin: BackupS3Plugin, + }, + pluginOptions: map[string]string{"folder": ""}, + wantErr: true, + }, + { + name: "s3 plugin folder ok", + config: history.BackupConfig{ + Timestamp: "20220401102430", + Plugin: BackupS3Plugin, + }, + pluginOptions: map[string]string{"folder": "/path/to/report"}, + want: "/path/to/report/backups/20220401/20220401102430/gpbackup_20220401102430_report", + }, + { + name: "unknown plugin without custom report path", + config: history.BackupConfig{ + Timestamp: "20220401102430", + Plugin: "some_plugin", + }, + pluginOptions: map[string]string{"folder": "/path/to/report"}, + wantErr: true, + }, + } + for _, tt := range tests { + cfg := tt.config + got, err := GetReportFilePathPlugin(&cfg, tt.customReportPath, tt.pluginOptions) + if tt.wantErr { + Expect(err).To(HaveOccurred(), tt.name) + } else { + Expect(err).ToNot(HaveOccurred(), tt.name) + Expect(got).To(Equal(tt.want), tt.name) + } + } + }) + }) + + Describe("CheckObjectFilteringExists", func() { + It("returns correct result for various filtering scenarios", func() { + tests := []struct { + name string + tableFilter string + schemaFilter string + objectFilter string + excludeFilter bool + want bool + config history.BackupConfig + }{ + { + name: "no filters specified", + want: true, + }, + { + name: "table filter matches included table", + tableFilter: "test.table1", + objectFilter: "include-table", + want: true, + config: history.BackupConfig{ + IncludeRelations: []string{"test.table1", "test.table2"}, + }, + }, + { + name: "table filter does not match included table", + tableFilter: "test.table1", + objectFilter: "include-table", + want: false, + config: history.BackupConfig{ + IncludeRelations: []string{"test.table2", "test.table3"}, + }, + }, + { + name: "table filter with no object filter", + tableFilter: "test.table1", + objectFilter: "", + want: false, + }, + { + name: "table filter with different object filter", + tableFilter: "test.table1", + objectFilter: "include-schema", + want: false, + config: history.BackupConfig{ + IncludeSchemas: []string{"test"}, + }, + }, + { + name: "exclude table filter matches", + tableFilter: "test.table1", + objectFilter: "exclude-table", + excludeFilter: true, + want: true, + config: history.BackupConfig{ + ExcludeRelations: []string{"test.table1", "test.table2"}, + }, + }, + { + name: "exclude table filter with no object filter", + tableFilter: "test.table1", + excludeFilter: true, + want: false, + }, + { + name: "schema filter matches included schema", + schemaFilter: "test", + objectFilter: "include-schema", + want: true, + config: history.BackupConfig{ + IncludeSchemas: []string{"test"}, + }, + }, + { + name: "schema filter with no object filter", + schemaFilter: "test", + want: false, + }, + { + name: "exclude schema filter matches", + schemaFilter: "test", + objectFilter: "exclude-schema", + excludeFilter: true, + want: true, + config: history.BackupConfig{ + ExcludeSchemas: []string{"test"}, + }, + }, + { + name: "exclude schema filter with no object filter", + schemaFilter: "test", + excludeFilter: true, + want: false, + }, + } + for _, tt := range tests { + cfg := tt.config + got := CheckObjectFilteringExists(&cfg, tt.tableFilter, tt.schemaFilter, tt.objectFilter, tt.excludeFilter) + Expect(got).To(Equal(tt.want), tt.name) + } + }) + }) +}) diff --git a/gpbackman/gpbckpconfig/struct.go b/gpbackman/gpbckpconfig/struct.go new file mode 100644 index 00000000..1ed1c09b --- /dev/null +++ b/gpbackman/gpbckpconfig/struct.go @@ -0,0 +1,22 @@ +package gpbckpconfig + +const ( + Layout = "20060102150405" + DateFormat = "Mon Jan 02 2006 15:04:05" + // Backup types. + BackupTypeFull = "full" + BackupTypeIncremental = "incremental" + BackupTypeDataOnly = "data-only" + BackupTypeMetadataOnly = "metadata-only" + // Object filtering types. + objectFilteringIncludeSchema = "include-schema" + objectFilteringExcludeSchema = "exclude-schema" + objectFilteringIncludeTable = "include-table" + objectFilteringExcludeTable = "exclude-table" + // Date deleted types. + DateDeletedInProgress = "In progress" + DateDeletedPluginFailed = "Plugin Backup Delete Failed" + DateDeletedLocalFailed = "Local Delete Failed" + // BackupS3Plugin S3 plugin names. + BackupS3Plugin = "gpbackup_s3_plugin" +) diff --git a/gpbackman/gpbckpconfig/utils.go b/gpbackman/gpbckpconfig/utils.go new file mode 100644 index 00000000..ecfd223f --- /dev/null +++ b/gpbackman/gpbckpconfig/utils.go @@ -0,0 +1,164 @@ +package gpbckpconfig + +import ( + "errors" + "fmt" + "os" + "path" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/apache/cloudberry-backup/gpbackman/textmsg" + "github.com/apache/cloudberry-go-libs/operating" +) + +// CheckTimestamp Returns error if timestamp is not valid. +func CheckTimestamp(timestamp string) error { + timestampFormat := regexp.MustCompile(`^(\d{14})$`) + if !timestampFormat.MatchString(timestamp) { + return textmsg.ErrorValidationTimestamp() + } + return nil +} + +func GetTimestampOlderThen(value uint) string { + return time.Now().AddDate(0, 0, -int(value)).Format(Layout) +} + +// CheckFullPath Returns error if path is not an absolute path or +// file does not exist. +func CheckFullPath(checkPath string, checkFileExists bool) error { + if !filepath.IsAbs(checkPath) { + return textmsg.ErrorValidationFullPath() + } + // In most cases this check should be mandatory. + // But there are commands, that allows the history db file to be missing. + if checkFileExists { + if _, err := os.Stat(checkPath); errors.Is(err, os.ErrNotExist) { + return textmsg.ErrorFileNotExist() + } + } + return nil +} + +// CheckTableFQN Returns error if table FQN is not in the format . +func CheckTableFQN(table string) error { + format := regexp.MustCompile(`^.+\..+$`) + if !format.MatchString(table) { + return textmsg.ErrorValidationTableFQN() + } + return nil +} + +// IsBackupActive Returns true if backup is active (not deleted). +func IsBackupActive(dateDeleted string) bool { + return (dateDeleted == "" || + dateDeleted == DateDeletedPluginFailed || + dateDeleted == DateDeletedLocalFailed) +} + +// IsPositiveValue Returns true if the value is positive. +func IsPositiveValue(value int) bool { + return value > 0 +} + +// backupPluginCustomReportPath Returns custom report path: +// +// /gpbackup__report +func backupPluginCustomReportPath(timestamp, folderValue string) string { + return filepath.Join("/", folderValue, ReportFileName(timestamp)) +} + +// backupS3PluginReportPath Returns path to report file name for gpbackup_s3_plugin plugin. +// Basic path for s3 plugin format: +// +// /backups///gpbackup__report +// +// See GetS3Path() func in https://github.com/greenplum-db/gpbackup-s3-plugin. +// If folder option is not specified or it is empty, the error will be returned. +func backupS3PluginReportPath(timestamp string, pluginOptions map[string]string) (string, error) { + pathOption := "folder" + // Timestamp validation is done on flags validation. + // We assume, that is the correct value coming from. + reportPathBasic := "backups/" + timestamp[0:8] + "/" + timestamp + folderValue, exists := pluginOptions[pathOption] + if !exists || folderValue == "" { + return "", textmsg.ErrorValidationPluginOption(pathOption, BackupS3Plugin) + } + // It's necessary to return full path to report file with leading '/'. + // But in config file folder value could be with leading '/' or without. + // So we need to remove leading '/' and add it back. + folderValue = strings.TrimPrefix(folderValue, "/") + folderValue = strings.TrimSuffix(folderValue, "/") + return filepath.Join("/", folderValue, reportPathBasic, ReportFileName(timestamp)), nil +} + +// ReportFileName Returns report file name for specific timestamp. +// Report file name format: gpbackup__report. +func ReportFileName(timestamp string) string { + return "gpbackup_" + timestamp + "_report" +} + +// CheckMasterBackupDir checks the backup directory for the master backup. +// It first tries to find the backup directory in the single-backup-dir format. +// If the single-backup-dir format is not used, it returns an error. +// If the single-backup-dir format is used, it returns the backup directory and sets the prefix to an empty string. +// If the single-backup-dir format is not found, it tries to find the backup directory with segment prefix format. +// If the backup directory with segment prefix format is not found, it returns an error. +// If multiple backup directories with segment prefix format are found, it returns an error. +// Otherwise, it returns the backup directory with segment prefix format, the segment prefix, and useSingleBackupDir flag to false. +func CheckMasterBackupDir(backupDir string) (string, string, bool, error) { + // Try to find the backup directory in the single-backup-dir format. + _, err := operating.System.Stat(fmt.Sprintf("%s/backups", backupDir)) + // The single-backup-dir directory format is not used. + if err != nil && !os.IsNotExist(err) { + return "", "", false, textmsg.ErrorFindBackupDirIn(backupDir, err) + } + if err == nil { + // The single-backup-dir directory format is used, there's no prefix to parse. + return backupDir, "", true, nil + } + // Try to find the backup directory with segment prefix format. + backupDirForMaster, err := operating.System.Glob(fmt.Sprintf("%s/*-1/backups", backupDir)) + if err != nil { + return "", "", false, textmsg.ErrorFindBackupDirIn(backupDir, err) + } + if len(backupDirForMaster) == 0 { + return "", "", false, textmsg.ErrorNotFoundBackupDirIn(backupDir) + } + if len(backupDirForMaster) != 1 { + return "", "", false, textmsg.ErrorSeveralFoundBackupDirIn(backupDir) + } + segPrefix := GetSegPrefix(backupDirForMaster[0]) + returnDir := filepath.Join(backupDir, fmt.Sprintf("%s-1", segPrefix)) + return returnDir, segPrefix, false, nil +} + +// GetSegPrefix Returns segment prefix from the master backup directory. +func GetSegPrefix(backupDir string) string { + indexOfBackupsSubstr := strings.LastIndex(backupDir, "-1/backups") + _, segPrefix := path.Split(backupDir[:indexOfBackupsSubstr]) + return segPrefix +} + +// ReportFilePath Returns path to report file. +func ReportFilePath(backupDir, timestamp string) string { + return filepath.Join(BackupDirPath(backupDir, timestamp), ReportFileName(timestamp)) +} + +// BackupDirPath Returns path to full backup directory. +func BackupDirPath(backupDir, timestamp string) string { + return filepath.Join(backupDir, "backups", timestamp[0:8], timestamp) +} + +// searchFilter returns true if the value is present in the list +func searchFilter(list []string, value string) bool { + for _, item := range list { + if item == value { + return true + } + } + return false +} diff --git a/gpbackman/gpbckpconfig/utils_db.go b/gpbackman/gpbckpconfig/utils_db.go new file mode 100644 index 00000000..102e7290 --- /dev/null +++ b/gpbackman/gpbckpconfig/utils_db.go @@ -0,0 +1,191 @@ +package gpbckpconfig + +import ( + "database/sql" + "fmt" + "strings" + + "github.com/apache/cloudberry-backup/history" +) + +// OpenHistoryDB opens the history backup database. +func OpenHistoryDB(historyDBPath string) (*sql.DB, error) { + db, err := sql.Open("sqlite3", historyDBPath) + if err != nil { + return nil, err + } + return db, nil +} + +// GetBackupDataDB reads backup data from history database. +func GetBackupDataDB(backupName string, hDB *sql.DB) (*history.BackupConfig, error) { + return history.GetBackupConfig(backupName, hDB) +} + +// GetBackupNamesDB returns a list of backup names. +func GetBackupNamesDB(showD, showF bool, historyDB *sql.DB) ([]string, error) { + return execQueryFunc(getBackupNameQuery(showD, showF), historyDB) +} + +func GetBackupDependencies(backupName string, historyDB *sql.DB) ([]string, error) { + return execQueryFunc(getBackupDependenciesQuery(backupName), historyDB) +} + +func GetBackupNamesBeforeTimestamp(timestamp string, historyDB *sql.DB) ([]string, error) { + return execQueryFunc(getBackupNameBeforeTimestampQuery(timestamp), historyDB) +} + +func GetBackupNamesAfterTimestamp(timestamp string, historyDB *sql.DB) ([]string, error) { + return execQueryFunc(getBackupNameAfterTimestampQuery(timestamp), historyDB) +} + +func GetBackupNamesForCleanBeforeTimestamp(timestamp string, historyDB *sql.DB) ([]string, error) { + return execQueryFunc(getBackupNameForCleanBeforeTimestampQuery(timestamp), historyDB) +} + +func getBackupNameQuery(showD, showF bool) string { + orderBy := "ORDER BY timestamp DESC;" + getBackupsQuery := "SELECT timestamp FROM backups" + switch { + case showD && showF: + getBackupsQuery = fmt.Sprintf("%s %s", getBackupsQuery, orderBy) + case showD && !showF: + getBackupsQuery = fmt.Sprintf("%s WHERE status != '%s' %s", getBackupsQuery, history.BackupStatusFailed, orderBy) + case !showD && showF: + getBackupsQuery = fmt.Sprintf("%s WHERE date_deleted IN ('', '%s', '%s', '%s') %s", getBackupsQuery, DateDeletedInProgress, DateDeletedPluginFailed, DateDeletedLocalFailed, orderBy) + default: + getBackupsQuery = fmt.Sprintf("%s WHERE status != '%s' AND date_deleted IN ('', '%s', '%s', '%s') %s", getBackupsQuery, history.BackupStatusFailed, DateDeletedInProgress, DateDeletedPluginFailed, DateDeletedLocalFailed, orderBy) + } + return getBackupsQuery +} + +func getBackupDependenciesQuery(backupName string) string { + return fmt.Sprintf(` +SELECT timestamp +FROM restore_plans +WHERE timestamp != '%s' + AND restore_plan_timestamp = '%s' +ORDER BY timestamp DESC; +`, backupName, backupName) +} + +func getBackupNameBeforeTimestampQuery(timestamp string) string { + return fmt.Sprintf(` +SELECT timestamp +FROM backups +WHERE timestamp < '%s' + AND status != '%s' + AND date_deleted IN ('', '%s', '%s') +ORDER BY timestamp DESC; +`, timestamp, history.BackupStatusInProgress, DateDeletedPluginFailed, DateDeletedLocalFailed) +} + +func getBackupNameAfterTimestampQuery(timestamp string) string { + return fmt.Sprintf(` +SELECT timestamp +FROM backups +WHERE timestamp > '%s' + AND status != '%s' + AND date_deleted IN ('', '%s', '%s') +ORDER BY timestamp DESC; +`, timestamp, history.BackupStatusInProgress, DateDeletedPluginFailed, DateDeletedLocalFailed) +} + +func getBackupNameForCleanBeforeTimestampQuery(timestamp string) string { + return fmt.Sprintf(` +SELECT timestamp +FROM backups +WHERE timestamp < '%s' + AND date_deleted NOT IN ('', '%s', '%s', '%s') +ORDER BY timestamp DESC; +`, timestamp, DateDeletedPluginFailed, DateDeletedLocalFailed, DateDeletedInProgress) +} + +// UpdateDeleteStatus updates the date_deleted column in the history database. +func UpdateDeleteStatus(backupName, dateDeleted string, historyDB *sql.DB) error { + return execStatementFunc(updateDeleteStatusQuery(backupName, dateDeleted), historyDB) +} + +// CleanBackupsDB cleans the backup history database. +func CleanBackupsDB(list []string, batchSize int, historyDB *sql.DB) error { + for i := 0; i < len(list); i += batchSize { + end := i + batchSize + if end > len(list) { + end = len(list) + } + batchIDs := list[i:end] + idStr := "'" + strings.Join(batchIDs, "','") + "'" + err := execStatementFunc(deleteBackupsFormTableQuery("backups", idStr), historyDB) + if err != nil { + return err + } + err = execStatementFunc(deleteBackupsFormTableQuery("restore_plans", idStr), historyDB) + if err != nil { + return err + } + err = execStatementFunc(deleteBackupsFormTableQuery("restore_plan_tables", idStr), historyDB) + if err != nil { + return err + } + err = execStatementFunc(deleteBackupsFormTableQuery("exclude_relations", idStr), historyDB) + if err != nil { + return err + } + err = execStatementFunc(deleteBackupsFormTableQuery("exclude_schemas", idStr), historyDB) + if err != nil { + return err + } + err = execStatementFunc(deleteBackupsFormTableQuery("include_relations", idStr), historyDB) + if err != nil { + return err + } + err = execStatementFunc(deleteBackupsFormTableQuery("include_schemas", idStr), historyDB) + if err != nil { + return err + } + } + return nil +} + +func deleteBackupsFormTableQuery(db, value string) string { + return fmt.Sprintf(`DELETE FROM %s WHERE timestamp IN (%s);`, db, value) +} + +func updateDeleteStatusQuery(timestamp, status string) string { + return fmt.Sprintf(`UPDATE backups SET date_deleted = '%s' WHERE timestamp = '%s';`, status, timestamp) +} + +func execQueryFunc(query string, historyDB *sql.DB) ([]string, error) { + sqlRow, err := historyDB.Query(query) + if err != nil { + return nil, err + } + defer sqlRow.Close() + var resultList []string + for sqlRow.Next() { + var b string + err := sqlRow.Scan(&b) + if err != nil { + return nil, err + } + resultList = append(resultList, b) + } + if err := sqlRow.Err(); err != nil { + return nil, err + } + return resultList, nil +} + +func execStatementFunc(query string, historyDB *sql.DB) error { + tx, err := historyDB.Begin() + if err != nil { + return err + } + _, err = tx.Exec(query) + if err != nil { + _ = tx.Rollback() + return err + } + err = tx.Commit() + return err +} diff --git a/gpbackman/gpbckpconfig/utils_db_test.go b/gpbackman/gpbckpconfig/utils_db_test.go new file mode 100644 index 00000000..ebc1e6f1 --- /dev/null +++ b/gpbackman/gpbckpconfig/utils_db_test.go @@ -0,0 +1,118 @@ +package gpbckpconfig + +import ( + "fmt" + + "github.com/apache/cloudberry-backup/history" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("utils_db tests", func() { + Describe("getBackupNameQuery", func() { + It("returns correct query for various flag combinations", func() { + tests := []struct { + name string + showD bool + showF bool + want string + }{ + { + name: "show all", + showD: true, + showF: true, + want: `SELECT timestamp FROM backups ORDER BY timestamp DESC;`, + }, + { + name: "show deleted", + showD: true, + showF: false, + want: `SELECT timestamp FROM backups WHERE status != 'Failure' ORDER BY timestamp DESC;`, + }, + { + name: "show failed", + showD: false, + showF: true, + want: `SELECT timestamp FROM backups WHERE date_deleted IN ('', 'In progress', 'Plugin Backup Delete Failed', 'Local Delete Failed') ORDER BY timestamp DESC;`, + }, + { + name: "show default", + showD: false, + showF: false, + want: `SELECT timestamp FROM backups WHERE status != 'Failure' AND date_deleted IN ('', 'In progress', 'Plugin Backup Delete Failed', 'Local Delete Failed') ORDER BY timestamp DESC;`, + }, + } + for _, tt := range tests { + Expect(getBackupNameQuery(tt.showD, tt.showF)).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("getBackupDependenciesQuery", func() { + It("returns correct query", func() { + want := ` +SELECT timestamp +FROM restore_plans +WHERE timestamp != 'TestBackup' + AND restore_plan_timestamp = 'TestBackup' +ORDER BY timestamp DESC; +` + Expect(getBackupDependenciesQuery("TestBackup")).To(Equal(want)) + }) + }) + + Describe("getBackupNameBeforeTimestampQuery", func() { + It("returns correct query", func() { + want := fmt.Sprintf(` +SELECT timestamp +FROM backups +WHERE timestamp < '20240101120000' + AND status != '%s' + AND date_deleted IN ('', 'Plugin Backup Delete Failed', 'Local Delete Failed') +ORDER BY timestamp DESC; +`, history.BackupStatusInProgress) + Expect(getBackupNameBeforeTimestampQuery("20240101120000")).To(Equal(want)) + }) + }) + + Describe("getBackupNameAfterTimestampQuery", func() { + It("returns correct query", func() { + want := fmt.Sprintf(` +SELECT timestamp +FROM backups +WHERE timestamp > '20240101120000' + AND status != '%s' + AND date_deleted IN ('', 'Plugin Backup Delete Failed', 'Local Delete Failed') +ORDER BY timestamp DESC; +`, history.BackupStatusInProgress) + Expect(getBackupNameAfterTimestampQuery("20240101120000")).To(Equal(want)) + }) + }) + + Describe("getBackupNameForCleanBeforeTimestampQuery", func() { + It("returns correct query", func() { + want := ` +SELECT timestamp +FROM backups +WHERE timestamp < '20240101120000' + AND date_deleted NOT IN ('', 'Plugin Backup Delete Failed', 'Local Delete Failed', 'In progress') +ORDER BY timestamp DESC; +` + Expect(getBackupNameForCleanBeforeTimestampQuery("20240101120000")).To(Equal(want)) + }) + }) + + Describe("deleteBackupsFormTableQuery", func() { + It("returns correct query", func() { + got := deleteBackupsFormTableQuery("TestBackup", "'20220401102430', '20220401102430'") + Expect(got).To(Equal("DELETE FROM TestBackup WHERE timestamp IN ('20220401102430', '20220401102430');")) + }) + }) + + Describe("updateDeleteStatusQuery", func() { + It("returns correct query", func() { + got := updateDeleteStatusQuery("TestBackup", "20220401102430") + Expect(got).To(Equal("UPDATE backups SET date_deleted = '20220401102430' WHERE timestamp = 'TestBackup';")) + }) + }) +}) diff --git a/gpbackman/gpbckpconfig/utils_test.go b/gpbackman/gpbckpconfig/utils_test.go new file mode 100644 index 00000000..95b01b24 --- /dev/null +++ b/gpbackman/gpbckpconfig/utils_test.go @@ -0,0 +1,253 @@ +package gpbckpconfig + +import ( + "os" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("utils tests", func() { + Describe("CheckTimestamp", func() { + It("accepts valid timestamp", func() { + Expect(CheckTimestamp("20230822120000")).To(Succeed()) + }) + + It("rejects invalid timestamp", func() { + Expect(CheckTimestamp("invalid")).To(HaveOccurred()) + }) + + It("rejects wrong length timestamp", func() { + Expect(CheckTimestamp("2023082212000")).To(HaveOccurred()) + }) + }) + + Describe("CheckFullPath", func() { + It("accepts existing file with full path", func() { + tempFile, err := os.CreateTemp("", "testfile") + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tempFile.Name()) + Expect(CheckFullPath(tempFile.Name(), true)).To(Succeed()) + }) + + It("rejects non-existing file with full path", func() { + Expect(CheckFullPath("/some/path/test.txt", true)).To(HaveOccurred()) + }) + + It("rejects empty path", func() { + Expect(CheckFullPath("", false)).To(HaveOccurred()) + }) + + It("rejects relative path", func() { + Expect(CheckFullPath("test.txt", false)).To(HaveOccurred()) + }) + }) + + Describe("IsBackupActive", func() { + It("returns correct result for various date deleted values", func() { + tests := []struct { + name string + value string + want bool + }{ + {"empty delete date", "", true}, + {"plugin error", DateDeletedPluginFailed, true}, + {"local error", DateDeletedLocalFailed, true}, + {"deletion in progress", DateDeletedInProgress, false}, + {"deleted", "20220401102430", false}, + } + for _, tt := range tests { + Expect(IsBackupActive(tt.value)).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("IsPositiveValue", func() { + It("returns true for positive value", func() { + Expect(IsPositiveValue(10)).To(BeTrue()) + }) + + It("returns false for zero", func() { + Expect(IsPositiveValue(0)).To(BeFalse()) + }) + + It("returns false for negative value", func() { + Expect(IsPositiveValue(-5)).To(BeFalse()) + }) + }) + + Describe("backupS3PluginReportPath", func() { + It("returns correct path for valid options", func() { + got, err := backupS3PluginReportPath("20230112131415", map[string]string{"folder": "/path/to/folder"}) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal("/path/to/folder/backups/20230112/20230112131415/gpbackup_20230112131415_report")) + }) + + It("returns error for missing options", func() { + _, err := backupS3PluginReportPath("20230112131415", nil) + Expect(err).To(HaveOccurred()) + }) + + It("returns error for wrong options key", func() { + _, err := backupS3PluginReportPath("20230112131415", map[string]string{"wrong_key": "/path/to/folder"}) + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("ReportFileName", func() { + It("returns correct report file name", func() { + Expect(ReportFileName("202301011234")).To(Equal("gpbackup_202301011234_report")) + }) + }) + + Describe("backupPluginCustomReportPath", func() { + It("returns correct path", func() { + tests := []struct { + name string + timestamp string + folder string + want string + }{ + {"basic", "20230101123456", "/backup/folder", "/backup/folder/gpbackup_20230101123456_report"}, + {"trailing slashes", "20230101123456", "/backup//folder//", "/backup/folder/gpbackup_20230101123456_report"}, + {"folder with spaces", "20230101123456", "/backup/folder with spaces", "/backup/folder with spaces/gpbackup_20230101123456_report"}, + {"no leading slash", "20230101123456", "backup/folder with spaces", "/backup/folder with spaces/gpbackup_20230101123456_report"}, + } + for _, tt := range tests { + Expect(backupPluginCustomReportPath(tt.timestamp, tt.folder)).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("GetTimestampOlderThen", func() { + It("returns timestamp within expected range", func() { + input := uint(1) + got := GetTimestampOlderThen(input) + parsedTime, err := time.ParseInLocation(Layout, got, time.Now().Location()) + Expect(err).ToNot(HaveOccurred()) + now := time.Now() + expected := now.AddDate(0, 0, -int(input)) + Expect(parsedTime.Before(now)).To(BeTrue()) + Expect(parsedTime.Sub(expected).Seconds()).To(BeNumerically("<=", 1)) + }) + }) + + Describe("CheckTableFQN", func() { + It("accepts valid table name", func() { + Expect(CheckTableFQN("public.table_1")).To(Succeed()) + }) + + It("rejects invalid table name", func() { + Expect(CheckTableFQN("invalid_table")).To(HaveOccurred()) + }) + }) + + Describe("ReportFilePath", func() { + It("returns correct report file path", func() { + got := ReportFilePath("/path/to/backup", "20230101123456") + Expect(got).To(Equal("/path/to/backup/backups/20230101/20230101123456/gpbackup_20230101123456_report")) + }) + }) + + Describe("GetSegPrefix", func() { + It("returns correct segment prefix", func() { + Expect(GetSegPrefix("/path/to/backup/segment-1/backups")).To(Equal("segment")) + }) + }) + + Describe("CheckMasterBackupDir", func() { + It("returns correct values for various backup dirs", func() { + tempDir := os.TempDir() + tests := []struct { + name string + testDir string + backupDir string + wantDir string + wantPrefix string + wantSingleBackupDir bool + wantErr bool + }{ + { + name: "valid single backup dir", + testDir: filepath.Join(tempDir, "noSegPrefix", "backups"), + backupDir: filepath.Join(tempDir, "noSegPrefix"), + wantDir: filepath.Join(tempDir, "noSegPrefix"), + wantPrefix: "", + wantSingleBackupDir: true, + }, + { + name: "valid backup dir with segment prefix", + testDir: filepath.Join(tempDir, "segPrefix", "segment-1", "backups"), + backupDir: filepath.Join(tempDir, "segPrefix"), + wantDir: filepath.Join(tempDir, "segPrefix", "segment-1"), + wantPrefix: "segment", + wantSingleBackupDir: false, + }, + { + name: "invalid backup dir", + testDir: filepath.Join(tempDir, "invalid"), + backupDir: filepath.Join(tempDir, "invalid"), + wantErr: true, + }, + { + name: "non-existent path", + testDir: tempDir, + backupDir: "some/path", + wantErr: true, + }, + } + for _, tt := range tests { + err := os.MkdirAll(tt.testDir, 0o755) + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tt.testDir) + gotDir, gotPrefix, gotIsSingleBackupDir, err := CheckMasterBackupDir(tt.backupDir) + if tt.wantErr { + Expect(err).To(HaveOccurred(), tt.name) + } else { + Expect(err).ToNot(HaveOccurred(), tt.name) + Expect(gotDir).To(Equal(tt.wantDir), tt.name) + Expect(gotPrefix).To(Equal(tt.wantPrefix), tt.name) + Expect(gotIsSingleBackupDir).To(Equal(tt.wantSingleBackupDir), tt.name) + } + } + }) + }) + + Describe("BackupDirPath", func() { + It("returns correct backup dir path", func() { + tests := []struct { + name string + backupDir string + timestamp string + want string + }{ + {"basic path", "/data/backup", "20230101123456", "/data/backup/backups/20230101/20230101123456"}, + {"path with trailing slash", "/data/backup/", "20230101123456", "/data/backup/backups/20230101/20230101123456"}, + } + for _, tt := range tests { + Expect(BackupDirPath(tt.backupDir, tt.timestamp)).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("searchFilter", func() { + It("returns correct result for various inputs", func() { + tests := []struct { + name string + list []string + value string + want bool + }{ + {"value in list", []string{"item1", "item2", "item3"}, "item2", true}, + {"value not in list", []string{"item1", "item2", "item3"}, "item4", false}, + {"empty list", []string{}, "item1", false}, + {"empty value", []string{"item1", "item2", "item3"}, "", false}, + } + for _, tt := range tests { + Expect(searchFilter(tt.list, tt.value)).To(Equal(tt.want), tt.name) + } + }) + }) +}) diff --git a/gpbackman/textmsg/error.go b/gpbackman/textmsg/error.go new file mode 100644 index 00000000..c9d1e27e --- /dev/null +++ b/gpbackman/textmsg/error.go @@ -0,0 +1,253 @@ +package textmsg + +import ( + "errors" + "fmt" + "strings" +) + +// Collection of possible error texts. + +// Errors that occur when working with a history db. + +func ErrorTextUnableActionHistoryDB(value string, err error) string { + return fmt.Sprintf("Unable to %s history db. Error: %v", value, err) +} + +func ErrorTextUnableReadHistoryDB(err error) string { + return fmt.Sprintf("Unable to read data from history db. Error: %v", err) +} + +// Errors that occur when working with a backup data. + +func ErrorTextUnableGetBackupInfo(backupName string, err error) string { + return fmt.Sprintf("Unable to get info for backup %s. Error: %v", backupName, err) +} + +func ErrorTextUnableGetBackupValue(value, backupName string, err error) string { + return fmt.Sprintf("Unable to get backup %s for backup %s. Error: %v", value, backupName, err) +} + +func ErrorTextUnableSetBackupStatus(value, backupName string, err error) string { + return fmt.Sprintf("Unable to set %s status for backup %s. Error: %v", value, backupName, err) +} + +func ErrorTextUnableDeleteBackup(backupName string, err error) string { + return fmt.Sprintf("Unable to delete backup %s. Error: %v", backupName, err) +} + +func ErrorTextUnableWorkBackup(backupName string, err error) string { + return fmt.Sprintf("Unable to work with backup %s. Error: %v", backupName, err) +} + +func ErrorTextUnableDeleteBackupCascade(backupName string, err error) string { + return fmt.Sprintf("Unable to delete dependent backups for backup %s. Error: %v", backupName, err) +} + +func ErrorTextUnableDeleteBackupUseCascade(backupName string, err error) string { + return fmt.Sprintf("Backup %s has dependent backups. Use --cascade option. Error: %v", backupName, err) +} + +func ErrorTextBackupDeleteInProgress(backupName string, err error) string { + return fmt.Sprintf("Backup %s deletion in progress. Error: %v", backupName, err) +} + +func ErrorTextUnableGetBackupReport(backupName string, err error) string { + return fmt.Sprintf("Unable to get report for the backup %s. Error: %v", backupName, err) +} + +func ErrorTextUnableGetBackupPath(value, backupName string, err error) string { + return fmt.Sprintf("Unable to get path to %s for the backup %s. Error: %v", value, backupName, err) +} + +// Errors that occur when working with a backup plugin. + +func ErrorTextUnableReadPluginConfigFile(err error) string { + return fmt.Sprintf("Unable to read plugin config file. Error: %v", err) +} + +// Error that occur when working with a local backup. + +func ErrorTextCommandExecutionFailed(err error, values ...string) string { + return fmt.Sprintf("Command failed: %s. Error: %v", strings.Join(values, " "), err) +} + +// Errors that occur during flags validation. + +func ErrorTextUnableValidateFlag(value, flag string, err error) string { + return fmt.Sprintf( + "Unable to validate value %s for flag %s. Error: %v", value, flag, err) +} + +func ErrorTextUnableCompatibleFlags(err error, values ...string) string { + return fmt.Sprintf( + "Unable to use the following flags together: %s. Error: %v", + strings.Join(values, ", "), err) +} + +func ErrorTextUnableValidateValue(err error, values ...string) string { + return fmt.Sprintf("Unable to validate provided arguments. Try to use one of flags: %s. Error: %v", + strings.Join(values, ", "), err) +} + +// Errors that occur when working with a local cluster. + +func ErrorTextUnableConnectLocalCluster(err error) string { + return fmt.Sprintf("Unable to connect to the cluster locally. Error: %v", err) +} + +func ErrorTextUnableGetBackupDirLocalClusterConn(err error) string { + return fmt.Sprintf("Unable to get backup directory from a local connection to the cluster. Error: %v", err) +} + +// Errors that occur when working with backup reports. + +func ErrorTextUnableGetReport(err error) string { + return fmt.Sprintf("Unable to get report. Error: %v", err) +} + +func ErrorTextUnableCheckTimestamp(err error) string { + return fmt.Sprintf("Unable to check timestamp. Error: %v", err) +} + +func ErrorTextUnableGetSegPrefix(err error) string { + return fmt.Sprintf("Unable to get segment prefix. Error: %v", err) +} + +func ErrorTextUnableCheckPath(err error) string { + return fmt.Sprintf("Unable to check path. Error: %v", err) +} + +// Errors that occur when working with local backup directories. + +func ErrorTextUnableDeleteLocalBackup(err error) string { + return fmt.Sprintf("Unable to delete local backup. Error: %v", err) +} + +func ErrorTextUnableCleanDB(err error) string { + return fmt.Sprintf("Unable to clean db. Error: %v", err) +} + +func ErrorTextUnableDeletePluginBackup(backupName string, err error) string { + return fmt.Sprintf("Unable to delete plugin backup %s. Error: %v", backupName, err) +} + +func ErrorTextUnableDeletePluginReport(backupName string, err error) string { + return fmt.Sprintf("Unable to delete plugin report %s. Error: %v", backupName, err) +} + +func ErrorTextUnableUpdateDeleteStatus(backupName string, err error) string { + return fmt.Sprintf("Unable to update delete status for %s. Error: %v", backupName, err) +} + +func ErrorTextUnableCheckBackupDir(backupName string, err error) string { + return fmt.Sprintf("Unable to check backup dir %s. Error: %v", backupName, err) +} + +func ErrorTextUnableDeleteFile(value1, value2 string, err error) string { + return fmt.Sprintf("Unable to delete %s %s. Error: %v", value1, value2, err) +} + +func InfoTextSetBackupDeleteStatus(backupName, status string) string { + return fmt.Sprintf("Set backup %s delete status to %s", backupName, status) +} + +// Error that is returned when flags validation not passed. + +func ErrorInvalidValueError() error { + return errors.New("invalid flag value") +} + +func ErrorIncompatibleFlagsError() error { + return errors.New("incompatible flags") +} + +func ErrorNotIndependentFlagsError() error { + return errors.New("not an independent flag") +} + +// Error that is returned when backup deletion fails. + +func ErrorBackupDeleteInProgressError() error { + return errors.New("backup deletion in progress") +} + +func ErrorBackupDeleteCascadeOptionError() error { + return errors.New("use cascade option") +} + +func ErrorBackupLocalStorageError() error { + return errors.New("is a local backup") +} + +func ErrorBackupNotLocalStorageError() error { + return errors.New("is not a local backup") +} + +// Error that is returned when some validation fails. + +func ErrorValidationFullPath() error { + return errors.New("not an absolute path") +} + +func ErrorFileNotExist() error { + return errors.New("file not exist") +} + +func ErrorValidationTableFQN() error { + return errors.New("not a fully qualified table name") +} + +func ErrorValidationTimestamp() error { + return errors.New("not a timestamp") +} + +func ErrorValidationValue() error { + return errors.New("value not set") +} + +func ErrorEmptyDatabase() error { + return errors.New("database name cannot be empty") +} + +// Error that is returned when some plugin options validation fails. + +func ErrorValidationPluginOption(value, pluginName string) error { + return fmt.Errorf("invalid plugin %s option value for plugin %s", value, pluginName) +} + +// Errors that are returned when some backup directory validation fails. + +func ErrorFindBackupDirIn(value string, err error) error { + return fmt.Errorf("can not find backup directory in %s, error: %v", value, err.Error()) +} + +func ErrorNotFoundBackupDirIn(value string) error { + return fmt.Errorf("no backup directory found in %s", value) +} + +func ErrorSeveralFoundBackupDirIn(value string) error { + return fmt.Errorf("several backup directory found in %s", value) +} + +// Error that is returned when backup not found. + +func ErrorBackupNotFoundError(backupName string) error { + return fmt.Errorf("backup %s not found", backupName) +} + +func ErrorInvalidInputValueError(value string) error { + return fmt.Errorf("invalid input value: %s", value) +} + +// Error that is returned when backup has specific delete status. + +func ErrorSetBackupDeleteStatus(backupName, status string) error { + return fmt.Errorf("backup %s has delete status %s", backupName, status) +} + +// Error that is returned when backup dir is not specified. + +func ErrorBackupDirNotSpecifiedError() error { + return errors.New("backup dir is not specified") +} diff --git a/gpbackman/textmsg/error_test.go b/gpbackman/textmsg/error_test.go new file mode 100644 index 00000000..b94ef9da --- /dev/null +++ b/gpbackman/textmsg/error_test.go @@ -0,0 +1,145 @@ +package textmsg + +import ( + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("error tests", func() { + Describe("error text functions with error only", func() { + It("returns correct error text", func() { + testError := errors.New("test error") + tests := []struct { + name string + function func(error) string + want string + }{ + {"ErrorTextUnableReadHistoryDB", ErrorTextUnableReadHistoryDB, "Unable to read data from history db. Error: test error"}, + {"ErrorTextUnableGetReport", ErrorTextUnableGetReport, "Unable to get report. Error: test error"}, + {"ErrorTextUnableCheckTimestamp", ErrorTextUnableCheckTimestamp, "Unable to check timestamp. Error: test error"}, + {"ErrorTextUnableGetSegPrefix", ErrorTextUnableGetSegPrefix, "Unable to get segment prefix. Error: test error"}, + {"ErrorTextUnableCheckPath", ErrorTextUnableCheckPath, "Unable to check path. Error: test error"}, + {"ErrorTextUnableDeleteLocalBackup", ErrorTextUnableDeleteLocalBackup, "Unable to delete local backup. Error: test error"}, + {"ErrorTextUnableCleanDB", ErrorTextUnableCleanDB, "Unable to clean db. Error: test error"}, + } + for _, tt := range tests { + Expect(tt.function(testError)).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("error text functions with error and arg", func() { + It("returns correct error text", func() { + testError := errors.New("test error") + tests := []struct { + name string + value string + function func(string, error) string + want string + }{ + {"ErrorTextUnableGetBackupInfo", "TestValue", ErrorTextUnableGetBackupInfo, "Unable to get info for backup TestValue. Error: test error"}, + {"ErrorTextUnableDeletePluginBackup", "TestValue", ErrorTextUnableDeletePluginBackup, "Unable to delete plugin backup TestValue. Error: test error"}, + {"ErrorTextUnableDeletePluginReport", "TestValue", ErrorTextUnableDeletePluginReport, "Unable to delete plugin report TestValue. Error: test error"}, + {"ErrorTextUnableUpdateDeleteStatus", "TestValue", ErrorTextUnableUpdateDeleteStatus, "Unable to update delete status for TestValue. Error: test error"}, + {"ErrorTextUnableCheckBackupDir", "TestValue", ErrorTextUnableCheckBackupDir, "Unable to check backup dir TestValue. Error: test error"}, + {"ErrorTextUnableActionHistoryDB", "TestAction", ErrorTextUnableActionHistoryDB, "Unable to TestAction history db. Error: test error"}, + } + for _, tt := range tests { + Expect(tt.function(tt.value, testError)).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("error text functions with error and two args", func() { + It("returns correct error text", func() { + testError := errors.New("test error") + tests := []struct { + name string + value1 string + value2 string + function func(string, string, error) string + want string + }{ + {"ErrorTextUnableDeleteFile", "TestValue1", "TestValue2", ErrorTextUnableDeleteFile, "Unable to delete TestValue1 TestValue2. Error: test error"}, + } + for _, tt := range tests { + Expect(tt.function(tt.value1, tt.value2, testError)).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("error text functions with error and multiple args", func() { + It("returns correct error text", func() { + testError := errors.New("test error") + tests := []struct { + name string + values []string + function func(error, ...string) string + want string + }{ + {"ErrorTextUnableCompatibleFlags", []string{"flag1", "flag2"}, ErrorTextUnableCompatibleFlags, "Unable to use the following flags together: flag1, flag2. Error: test error"}, + } + for _, tt := range tests { + Expect(tt.function(testError, tt.values...)).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("error functions with one arg", func() { + It("returns correct error", func() { + tests := []struct { + name string + value string + function func(string) error + want string + }{ + {"ErrorBackupNotFoundError", "TestBackup", ErrorBackupNotFoundError, "backup TestBackup not found"}, + {"ErrorInvalidInputValueError", "TestValue", ErrorInvalidInputValueError, "invalid input value: TestValue"}, + } + for _, tt := range tests { + err := tt.function(tt.value) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("error functions with two args", func() { + It("returns correct error", func() { + tests := []struct { + name string + value1 string + value2 string + function func(string, string) error + want string + }{ + {"ErrorSetBackupDeleteStatus", "TestBackup", "TestStatus", ErrorSetBackupDeleteStatus, "backup TestBackup has delete status TestStatus"}, + } + for _, tt := range tests { + err := tt.function(tt.value1, tt.value2) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("error functions returning error", func() { + It("returns correct error", func() { + tests := []struct { + name string + function func() error + want string + }{ + {"ErrorBackupDirNotSpecifiedError", ErrorBackupDirNotSpecifiedError, "backup dir is not specified"}, + {"ErrorBackupDeleteInProgressError", ErrorBackupDeleteInProgressError, "backup deletion in progress"}, + } + for _, tt := range tests { + err := tt.function() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal(tt.want), tt.name) + } + }) + }) +}) diff --git a/gpbackman/textmsg/info.go b/gpbackman/textmsg/info.go new file mode 100644 index 00000000..8ab403a6 --- /dev/null +++ b/gpbackman/textmsg/info.go @@ -0,0 +1,54 @@ +package textmsg + +import ( + "fmt" + "strings" +) + +func InfoTextBackupDeleteStart(backupName string) string { + return fmt.Sprintf("Start deleting backup %s", backupName) +} + +func InfoTextBackupAlreadyDeleted(backupName string) string { + return fmt.Sprintf("Backup %s has already been deleted", backupName) +} + +func InfoTextBackupStatus(backupName, backupStatus string) string { + return fmt.Sprintf("Backup %s has status: %s", backupName, backupStatus) +} + +func InfoTextBackupDeleteSuccess(backupName string) string { + return fmt.Sprintf("Backup %s successfully deleted", backupName) +} + +func InfoTextBackupDependenciesList(backupName string, list []string) string { + return fmt.Sprintf("Backup %s has dependent backups: %s", backupName, strings.Join(list, ", ")) +} + +func InfoTextBackupDeleteList(list []string) string { + return fmt.Sprintf("The following backups will be deleted: %s", strings.Join(list, ", ")) +} + +func InfoTextBackupDeleteListFromHistory(list []string) string { + return fmt.Sprintf("The following backups will be deleted from history: %s", strings.Join(list, ", ")) +} + +func InfoTextCommandExecution(list ...string) string { + return fmt.Sprintf("Executing command: %s", strings.Join(list, " ")) +} + +func InfoTextCommandExecutionSucceeded(list ...string) string { + return fmt.Sprintf("Command succeeded: %s", strings.Join(list, " ")) +} + +func InfoTextBackupDirPath(backupDir string) string { + return fmt.Sprintf("Path to backup directory: %s", backupDir) +} + +func InfoTextSegmentPrefix(segPrefix string) string { + return fmt.Sprintf("Segment Prefix: %s", segPrefix) +} + +func InfoTextNothingToDo() string { + return "Nothing to do" +} diff --git a/gpbackman/textmsg/info_test.go b/gpbackman/textmsg/info_test.go new file mode 100644 index 00000000..03e38214 --- /dev/null +++ b/gpbackman/textmsg/info_test.go @@ -0,0 +1,102 @@ +package textmsg + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("info tests", func() { + Describe("info text functions with one arg", func() { + It("returns correct info text", func() { + tests := []struct { + name string + value string + function func(string) string + want string + }{ + {"InfoTextBackupDeleteStart", "TestBackup", InfoTextBackupDeleteStart, "Start deleting backup TestBackup"}, + {"InfoTextBackupDeleteSuccess", "TestBackup", InfoTextBackupDeleteSuccess, "Backup TestBackup successfully deleted"}, + {"InfoTextBackupAlreadyDeleted", "TestBackup", InfoTextBackupAlreadyDeleted, "Backup TestBackup has already been deleted"}, + {"InfoTextBackupDirPath", "/test/path", InfoTextBackupDirPath, "Path to backup directory: /test/path"}, + {"InfoTextSegmentPrefix", "TestValue", InfoTextSegmentPrefix, "Segment Prefix: TestValue"}, + } + for _, tt := range tests { + Expect(tt.function(tt.value)).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("info text functions with two args", func() { + It("returns correct info text", func() { + tests := []struct { + name string + value1 string + value2 string + function func(string, string) string + want string + }{ + {"InfoTextBackupStatus", "TestBackup", "In Progress", InfoTextBackupStatus, "Backup TestBackup has status: In Progress"}, + } + for _, tt := range tests { + Expect(tt.function(tt.value1, tt.value2)).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("info text functions with multiple args", func() { + It("returns correct info text", func() { + tests := []struct { + name string + value string + valueList []string + function func(string, []string) string + want string + }{ + {"InfoTextBackupDependenciesList", "TestBackup1", []string{"TestBackup2", "TestBackup3"}, InfoTextBackupDependenciesList, "Backup TestBackup1 has dependent backups: TestBackup2, TestBackup3"}, + } + for _, tt := range tests { + Expect(tt.function(tt.value, tt.valueList)).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("info text functions with multiple separate args", func() { + It("returns correct info text", func() { + tests := []struct { + name string + values []string + function func(...string) string + want string + }{ + {"InfoTextCommandExecution", []string{"execution_command", "some_argument"}, InfoTextCommandExecution, "Executing command: execution_command some_argument"}, + {"InfoTextCommandExecutionSucceeded", []string{"execution_command", "some_argument"}, InfoTextCommandExecutionSucceeded, "Command succeeded: execution_command some_argument"}, + } + for _, tt := range tests { + Expect(tt.function(tt.values...)).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("info text functions with list args", func() { + It("returns correct info text", func() { + tests := []struct { + name string + values []string + function func([]string) string + want string + }{ + {"InfoTextBackupDeleteList", []string{"TestBackup1", "TestBackup2"}, InfoTextBackupDeleteList, "The following backups will be deleted: TestBackup1, TestBackup2"}, + {"InfoTextBackupDeleteListFromHistory", []string{"TestBackup1", "TestBackup2"}, InfoTextBackupDeleteListFromHistory, "The following backups will be deleted from history: TestBackup1, TestBackup2"}, + } + for _, tt := range tests { + Expect(tt.function(tt.values)).To(Equal(tt.want), tt.name) + } + }) + }) + + Describe("info text functions with no args", func() { + It("returns correct info text", func() { + Expect(InfoTextNothingToDo()).To(Equal("Nothing to do")) + }) + }) +}) diff --git a/gpbackman/textmsg/textmsg_suite_test.go b/gpbackman/textmsg/textmsg_suite_test.go new file mode 100644 index 00000000..c4affdcf --- /dev/null +++ b/gpbackman/textmsg/textmsg_suite_test.go @@ -0,0 +1,13 @@ +package textmsg + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestTextmsg(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Textmsg Suite") +} diff --git a/gpbackman/textmsg/warn.go b/gpbackman/textmsg/warn.go new file mode 100644 index 00000000..d5bd0c5e --- /dev/null +++ b/gpbackman/textmsg/warn.go @@ -0,0 +1,7 @@ +package textmsg + +import "fmt" + +func WarnTextBackupUnableGetReport(backupName string) string { + return fmt.Sprintf("Unable to get report for backup %s. Check if backup is active", backupName) +} diff --git a/gpbackman/textmsg/warn_test.go b/gpbackman/textmsg/warn_test.go new file mode 100644 index 00000000..b5f3ad68 --- /dev/null +++ b/gpbackman/textmsg/warn_test.go @@ -0,0 +1,24 @@ +package textmsg + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("warn tests", func() { + Describe("warn text functions with one arg", func() { + It("returns correct warn text", func() { + tests := []struct { + name string + value string + function func(string) string + want string + }{ + {"WarnTextBackupUnableGetReport", "TestBackup", WarnTextBackupUnableGetReport, "Unable to get report for backup TestBackup. Check if backup is active"}, + } + for _, tt := range tests { + Expect(tt.function(tt.value)).To(Equal(tt.want), tt.name) + } + }) + }) +}) From a6ec715e14005d396acc168f527e734fafd9b825 Mon Sep 17 00:00:00 2001 From: woblerr Date: Tue, 17 Mar 2026 21:46:27 +0300 Subject: [PATCH 02/39] Replace duplicate utility functions. - Replace searchFilter() with utils.Exists() in gpbckpconfig/helpers.go - Remove searchFilter() from gpbckpconfig/utils.go and its test - Replace getCurrentTimestamp() with history.CurrentTimestamp() in cmd/backup_delete.go - Remove getCurrentTimestamp() from cmd/wrappers.go and its test --- gpbackman/cmd/backup_delete.go | 5 +++-- gpbackman/cmd/wrappers.go | 5 ----- gpbackman/cmd/wrappers_test.go | 9 --------- gpbackman/gpbckpconfig/helpers.go | 9 +++++---- gpbackman/gpbckpconfig/utils.go | 10 +--------- gpbackman/gpbckpconfig/utils_test.go | 19 +------------------ 6 files changed, 10 insertions(+), 47 deletions(-) diff --git a/gpbackman/cmd/backup_delete.go b/gpbackman/cmd/backup_delete.go index bd673a96..bae6bdf3 100644 --- a/gpbackman/cmd/backup_delete.go +++ b/gpbackman/cmd/backup_delete.go @@ -19,6 +19,7 @@ import ( "github.com/apache/cloudberry-backup/gpbackman/gpbckpconfig" "github.com/apache/cloudberry-backup/gpbackman/textmsg" + "github.com/apache/cloudberry-backup/history" "github.com/apache/cloudberry-backup/utils" ) @@ -300,7 +301,7 @@ func backupDeleteDBCascade(backupList []string, deleteForce, ignoreErrors, skipL func backupDeleteDBPluginFunc(backupName, pluginConfigPath string, pluginConfig *utils.PluginConfig, hDB *sql.DB, ignoreErrors bool) error { var err error - dateDeleted := getCurrentTimestamp() + dateDeleted := history.CurrentTimestamp() gplog.Info("%s", textmsg.InfoTextBackupDeleteStart(backupName)) err = gpbckpconfig.UpdateDeleteStatus(backupName, gpbckpconfig.DateDeletedInProgress, hDB) if err != nil { @@ -353,7 +354,7 @@ func backupDeleteDBPluginFunc(backupName, pluginConfigPath string, pluginConfig func backupDeleteDBLocalFunc(backupName, backupDir string, maxParallelProcesses int, hDB *sql.DB, ignoreErrors bool) error { var err, errUpdate error - dateDeleted := getCurrentTimestamp() + dateDeleted := history.CurrentTimestamp() gplog.Info("%s", textmsg.InfoTextBackupDeleteStart(backupName)) errUpdate = gpbckpconfig.UpdateDeleteStatus(backupName, gpbckpconfig.DateDeletedInProgress, hDB) if errUpdate != nil { diff --git a/gpbackman/cmd/wrappers.go b/gpbackman/cmd/wrappers.go index 81c8bec0..48549b51 100644 --- a/gpbackman/cmd/wrappers.go +++ b/gpbackman/cmd/wrappers.go @@ -6,7 +6,6 @@ import ( "os" "path/filepath" "strings" - "time" "github.com/apache/cloudberry-go-libs/gplog" "github.com/spf13/pflag" @@ -71,10 +70,6 @@ func getHistoryDBPath(historyDBPath string) string { return historyDBName } -func getCurrentTimestamp() string { - return time.Now().Format(gpbckpconfig.Layout) -} - func checkCompatibleFlags(flags *pflag.FlagSet, flagNames ...string) error { n := 0 for _, name := range flagNames { diff --git a/gpbackman/cmd/wrappers_test.go b/gpbackman/cmd/wrappers_test.go index ddd4634a..550a91d9 100644 --- a/gpbackman/cmd/wrappers_test.go +++ b/gpbackman/cmd/wrappers_test.go @@ -4,7 +4,6 @@ import ( "fmt" "os" "path/filepath" - "time" "github.com/apache/cloudberry-backup/gpbackman/gpbckpconfig" "github.com/apache/cloudberry-backup/history" @@ -41,14 +40,6 @@ var _ = Describe("wrappers tests", func() { }) }) - Describe("getCurrentTimestamp", func() { - It("returns valid timestamp", func() { - result := getCurrentTimestamp() - _, err := time.Parse(gpbckpconfig.Layout, result) - Expect(err).ToNot(HaveOccurred()) - }) - }) - Describe("checkCompatibleFlags", func() { It("does not return error when no flags changed", func() { flags := pflag.NewFlagSet("test", pflag.ContinueOnError) diff --git a/gpbackman/gpbckpconfig/helpers.go b/gpbackman/gpbckpconfig/helpers.go index 81c1da6b..3336c026 100644 --- a/gpbackman/gpbckpconfig/helpers.go +++ b/gpbackman/gpbckpconfig/helpers.go @@ -6,6 +6,7 @@ import ( "time" "github.com/apache/cloudberry-backup/history" + "github.com/apache/cloudberry-backup/utils" ) // GetBackupType Get backup type. @@ -210,22 +211,22 @@ func CheckObjectFilteringExists(backupConfig *history.BackupConfig, tableFilter, switch { case tableFilter != "" && !excludeFilter: if objectFilter == objectFilteringIncludeTable { - return searchFilter(backupConfig.IncludeRelations, tableFilter) + return utils.Exists(backupConfig.IncludeRelations, tableFilter) } return false case tableFilter != "" && excludeFilter: if objectFilter == objectFilteringExcludeTable { - return searchFilter(backupConfig.ExcludeRelations, tableFilter) + return utils.Exists(backupConfig.ExcludeRelations, tableFilter) } return false case schemaFilter != "" && !excludeFilter: if objectFilter == objectFilteringIncludeSchema { - return searchFilter(backupConfig.IncludeSchemas, schemaFilter) + return utils.Exists(backupConfig.IncludeSchemas, schemaFilter) } return false case schemaFilter != "" && excludeFilter: if objectFilter == objectFilteringExcludeSchema { - return searchFilter(backupConfig.ExcludeSchemas, schemaFilter) + return utils.Exists(backupConfig.ExcludeSchemas, schemaFilter) } return false default: diff --git a/gpbackman/gpbckpconfig/utils.go b/gpbackman/gpbckpconfig/utils.go index ecfd223f..55954966 100644 --- a/gpbackman/gpbckpconfig/utils.go +++ b/gpbackman/gpbckpconfig/utils.go @@ -153,12 +153,4 @@ func BackupDirPath(backupDir, timestamp string) string { return filepath.Join(backupDir, "backups", timestamp[0:8], timestamp) } -// searchFilter returns true if the value is present in the list -func searchFilter(list []string, value string) bool { - for _, item := range list { - if item == value { - return true - } - } - return false -} + diff --git a/gpbackman/gpbckpconfig/utils_test.go b/gpbackman/gpbckpconfig/utils_test.go index 95b01b24..0703194c 100644 --- a/gpbackman/gpbckpconfig/utils_test.go +++ b/gpbackman/gpbckpconfig/utils_test.go @@ -232,22 +232,5 @@ var _ = Describe("utils tests", func() { }) }) - Describe("searchFilter", func() { - It("returns correct result for various inputs", func() { - tests := []struct { - name string - list []string - value string - want bool - }{ - {"value in list", []string{"item1", "item2", "item3"}, "item2", true}, - {"value not in list", []string{"item1", "item2", "item3"}, "item4", false}, - {"empty list", []string{}, "item1", false}, - {"empty value", []string{"item1", "item2", "item3"}, "", false}, - } - for _, tt := range tests { - Expect(searchFilter(tt.list, tt.value)).To(Equal(tt.want), tt.name) - } - }) - }) + }) From b6ac99c20dc58458a975612b94ef7ad54b6d8a85 Mon Sep 17 00:00:00 2001 From: woblerr Date: Tue, 17 Mar 2026 21:51:55 +0300 Subject: [PATCH 03/39] Use same style as other main files. --- gpbackman.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gpbackman.go b/gpbackman.go index cc1cbc8e..e67fc0a3 100644 --- a/gpbackman.go +++ b/gpbackman.go @@ -3,8 +3,8 @@ package main -import "github.com/apache/cloudberry-backup/gpbackman/cmd" +import . "github.com/apache/cloudberry-backup/gpbackman/cmd" func main() { - cmd.Execute() + Execute() } From 6a148625c7160a66380a7ceec8928e4e842cf51b Mon Sep 17 00:00:00 2001 From: woblerr Date: Sun, 22 Mar 2026 14:46:03 +0300 Subject: [PATCH 04/39] Disable auto-formatting for table headers in backup-info command. --- gpbackman/cmd/backup_info.go | 1 - 1 file changed, 1 deletion(-) diff --git a/gpbackman/cmd/backup_info.go b/gpbackman/cmd/backup_info.go index f7a2595f..18671a3b 100644 --- a/gpbackman/cmd/backup_info.go +++ b/gpbackman/cmd/backup_info.go @@ -270,7 +270,6 @@ func backupInfoDB(opts BackupInfoOptions, hDB *sql.DB, t *tablewriter.Table) err func initTable(t *tablewriter.Table, includeDetails bool) { t.SetBorder(false) - t.SetAutoFormatHeaders(false) header := []string{ "timestamp", "date", From fa6fd3b5d709fed54cc40c8f8baafae902f6e840 Mon Sep 17 00:00:00 2001 From: woblerr Date: Sun, 22 Mar 2026 15:12:36 +0300 Subject: [PATCH 05/39] Add end-to-end tests for gpbackman functionality. Also add vscode to gitignore. --- .gitignore | 6 + end_to_end/end_to_end_suite_test.go | 36 ++ end_to_end/gpbackman_test.go | 719 ++++++++++++++++++++++++++++ 3 files changed, 761 insertions(+) create mode 100644 end_to_end/gpbackman_test.go diff --git a/.gitignore b/.gitignore index 3bef3410..a6a328c1 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,12 @@ _testmain.go gpbackup gprestore gpbackup_helper +gpbackman + +!gpbackman/ # Logs *.log + +# vscode +.vscode/ diff --git a/end_to_end/end_to_end_suite_test.go b/end_to_end/end_to_end_suite_test.go index 5738e924..2eed0150 100644 --- a/end_to_end/end_to_end_suite_test.go +++ b/end_to_end/end_to_end_suite_test.go @@ -1,6 +1,7 @@ package end_to_end_test import ( + "database/sql" "encoding/csv" "flag" "fmt" @@ -30,6 +31,7 @@ import ( "github.com/apache/cloudberry-go-libs/structmatcher" "github.com/apache/cloudberry-go-libs/testhelper" "github.com/blang/semver" + _ "github.com/mattn/go-sqlite3" "github.com/pkg/errors" "github.com/spf13/pflag" @@ -69,6 +71,7 @@ var ( schema2TupleCounts map[string]int backupDir string segmentCount int + gpbackmanPath string ) const ( @@ -463,6 +466,38 @@ func moveSegmentBackupFiles(tarBaseName string, extractDirectory string, isMulti } } +// gpbackman helpers + +func gpbackman(args ...string) []byte { + command := exec.Command(gpbackmanPath, args...) + return mustRunCommand(command) +} + +func gpbackmanWithError(args ...string) ([]byte, error) { + command := exec.Command(gpbackmanPath, args...) + return command.CombinedOutput() +} + +func getHistoryDBPathForCluster() string { + mdd := backupCluster.GetDirForContent(-1) + return path.Join(mdd, "gpbackup_history.db") +} + +// queryHistoryDB runs an SQL query against gpbackup_history.db using database/sql and returns the trimmed output. +func queryHistoryDB(historyDB string, query string) string { + db, err := sql.Open("sqlite3", historyDB) + Expect(err).ToNot(HaveOccurred()) + defer db.Close() + + var result string + err = db.QueryRow(query).Scan(&result) + if err == sql.ErrNoRows { + return "" + } + Expect(err).ToNot(HaveOccurred()) + return strings.TrimSpace(result) +} + func TestEndToEnd(t *testing.T) { format.MaxLength = 0 RegisterFailHandler(Fail) @@ -544,6 +579,7 @@ options: gprestorePath = fmt.Sprintf("%s/gprestore", binDir) backupHelperPath = fmt.Sprintf("%s/gpbackup_helper", binDir) restoreHelperPath = backupHelperPath + gpbackmanPath = fmt.Sprintf("%s/gpbackman", binDir) } segConfig := cluster.MustGetSegmentConfiguration(backupConn) backupCluster = cluster.NewCluster(segConfig) diff --git a/end_to_end/gpbackman_test.go b/end_to_end/gpbackman_test.go new file mode 100644 index 00000000..6db59af4 --- /dev/null +++ b/end_to_end/gpbackman_test.go @@ -0,0 +1,719 @@ +package end_to_end_test + +import ( + "fmt" + "strings" + + "github.com/apache/cloudberry-backup/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// countBackupInfoLines counts the number of backup entry rows in backup-info +// output. Data rows contain '|' separators but do not contain the header +// label "TIMESTAMP". +func countBackupInfoLines(output []byte) int { + count := 0 + for _, line := range strings.Split(string(output), "\n") { + trimmed := strings.TrimSpace(line) + if trimmed == "" { + continue + } + if strings.Contains(trimmed, "|") && + !strings.Contains(trimmed, "TIMESTAMP") && + !isSeparatorLine(trimmed) { + count++ + } + } + return count +} + +// isSeparatorLine returns true for lines like "---+---+---". +func isSeparatorLine(line string) bool { + for _, c := range line { + if c != '-' && c != '+' && c != ' ' { + return false + } + } + return true +} + +var _ = Describe("gpbackman end to end tests", func() { + + // ------------------------------------------------------------------ // + // backup-info + // ------------------------------------------------------------------ // + Describe("backup-info", func() { + var ( + historyDB string + timestampMap map[string]string + ) + + BeforeEach(func() { + end_to_end_setup() + historyDB = getHistoryDBPathForCluster() + timestampMap = make(map[string]string) + + // 1. Full local backup (with --leaf-partition-data for incremental compatibility) + output := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir, + "--leaf-partition-data") + timestampMap["full_local"] = getBackupTimestamp(string(output)) + + // 2. Full local backup with --include-table + output = gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir, + "--include-table", "public.foo") + timestampMap["full_include_table"] = getBackupTimestamp(string(output)) + + // 3. Full local backup with --exclude-schema + output = gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir, + "--exclude-schema", "schema2") + timestampMap["full_exclude_schema"] = getBackupTimestamp(string(output)) + + // 4. Incremental local backup (depends on full_local) + output = gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir, + "--incremental", + "--leaf-partition-data") + timestampMap["incremental"] = getBackupTimestamp(string(output)) + + // 5. Metadata-only backup + output = gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir, + "--metadata-only") + timestampMap["metadata_only"] = getBackupTimestamp(string(output)) + }) + + AfterEach(func() { + end_to_end_teardown() + }) + + It("lists all backups", func() { + output := gpbackman( + "backup-info", + "--history-db", historyDB, + ) + lines := countBackupInfoLines(output) + Expect(lines).To(BeNumerically(">=", 5), + fmt.Sprintf("Expected at least 5 backup entries, got %d.\nOutput:\n%s", + lines, string(output))) + }) + + It("filters by type full", func() { + output := gpbackman( + "backup-info", + "--history-db", historyDB, + "--type", "full", + ) + Expect(string(output)).To(ContainSubstring("full")) + Expect(string(output)).ToNot(ContainSubstring("incremental")) + }) + + It("filters by type incremental", func() { + output := gpbackman( + "backup-info", + "--history-db", historyDB, + "--type", "incremental", + ) + Expect(string(output)).To(ContainSubstring("incremental")) + lines := countBackupInfoLines(output) + Expect(lines).To(BeNumerically(">=", 1), + "Expected at least 1 incremental backup") + }) + + It("filters by type metadata-only", func() { + output := gpbackman( + "backup-info", + "--history-db", historyDB, + "--type", "metadata-only", + ) + Expect(string(output)).To(ContainSubstring("metadata-only")) + lines := countBackupInfoLines(output) + Expect(lines).To(BeNumerically(">=", 1), + "Expected at least 1 metadata-only backup") + }) + + It("filters by include-table", func() { + output := gpbackman( + "backup-info", + "--history-db", historyDB, + "--table", "public.foo", + ) + Expect(string(output)).To(ContainSubstring(timestampMap["full_include_table"])) + }) + + It("filters by exclude-schema", func() { + output := gpbackman( + "backup-info", + "--history-db", historyDB, + "--schema", "schema2", + "--exclude", + ) + Expect(string(output)).To(ContainSubstring(timestampMap["full_exclude_schema"])) + }) + + It("shows backup chain with --timestamp", func() { + output := gpbackman( + "backup-info", + "--history-db", historyDB, + "--timestamp", timestampMap["full_local"], + ) + outputStr := string(output) + Expect(outputStr).To(ContainSubstring(timestampMap["full_local"]), + "Expected the specified backup timestamp in the output") + }) + + It("shows detail with --timestamp --detail", func() { + output := gpbackman( + "backup-info", + "--history-db", historyDB, + "--timestamp", timestampMap["incremental"], + "--detail", + ) + Expect(string(output)).To(Or( + ContainSubstring("OBJECT FILTERING"), + ContainSubstring("object filtering"), + )) + }) + + It("shows detail for all backups with --detail", func() { + output := gpbackman( + "backup-info", + "--history-db", historyDB, + "--detail", + ) + Expect(string(output)).To(Or( + ContainSubstring("OBJECT FILTERING"), + ContainSubstring("object filtering"), + )) + Expect(string(output)).To(ContainSubstring("public.foo")) + }) + + It("rejects incompatible flags --timestamp with --type", func() { + _, err := gpbackmanWithError( + "backup-info", + "--history-db", historyDB, + "--timestamp", timestampMap["full_local"], + "--type", "full", + ) + Expect(err).To(HaveOccurred()) + }) + + It("rejects invalid timestamp format", func() { + _, err := gpbackmanWithError( + "backup-info", + "--history-db", historyDB, + "--timestamp", "invalid", + ) + Expect(err).To(HaveOccurred()) + }) + }) + + // ------------------------------------------------------------------ // + // report-info + // ------------------------------------------------------------------ // + Describe("report-info", func() { + var ( + historyDB string + timestampMap map[string]string + ) + + BeforeEach(func() { + end_to_end_setup() + historyDB = getHistoryDBPathForCluster() + timestampMap = make(map[string]string) + + // 1. Full local backup + output := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir) + timestampMap["full_local"] = getBackupTimestamp(string(output)) + + // 2. Plugin backup using example_plugin + copyPluginToAllHosts(backupConn, examplePluginExec) + output = gpbackup(gpbackupPath, backupHelperPath, + "--plugin-config", examplePluginTestConfig) + timestampMap["plugin"] = getBackupTimestamp(string(output)) + }) + + AfterEach(func() { + end_to_end_teardown() + }) + + It("displays local backup report with --backup-dir", func() { + output := gpbackman( + "report-info", + "--history-db", historyDB, + "--timestamp", timestampMap["full_local"], + "--backup-dir", backupDir, + ) + Expect(string(output)).To(ContainSubstring("Backup Report")) + Expect(string(output)).To(ContainSubstring(timestampMap["full_local"])) + }) + + It("displays local backup report without --backup-dir", func() { + output := gpbackman( + "report-info", + "--history-db", historyDB, + "--timestamp", timestampMap["full_local"], + ) + Expect(string(output)).To(ContainSubstring("Backup Report")) + }) + + It("displays plugin backup report", func() { + ts := timestampMap["plugin"] + reportDir := fmt.Sprintf("/tmp/plugin_dest/%s/%s", ts[:8], ts) + output := gpbackman( + "report-info", + "--history-db", historyDB, + "--timestamp", ts, + "--plugin-config", examplePluginTestConfig, + "--plugin-report-file-path", reportDir, + ) + Expect(string(output)).To(ContainSubstring("Backup Report")) + Expect(string(output)).To(ContainSubstring(ts)) + }) + + It("rejects --plugin-report-file-path without --plugin-config", func() { + _, err := gpbackmanWithError( + "report-info", + "--history-db", historyDB, + "--timestamp", timestampMap["full_local"], + "--plugin-report-file-path", "/tmp/fake_report", + ) + Expect(err).To(HaveOccurred()) + }) + }) + + // ------------------------------------------------------------------ // + // backup-delete + // ------------------------------------------------------------------ // + Describe("backup-delete", func() { + var historyDB string + + BeforeEach(func() { + end_to_end_setup() + historyDB = getHistoryDBPathForCluster() + }) + + AfterEach(func() { + end_to_end_teardown() + }) + + It("deletes a local backup by timestamp", func() { + output := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir) + timestamp := getBackupTimestamp(string(output)) + + gpbackman( + "backup-delete", + "--history-db", historyDB, + "--timestamp", timestamp, + "--backup-dir", backupDir, + ) + + dateDeleted := queryHistoryDB(historyDB, + fmt.Sprintf("SELECT date_deleted FROM backups WHERE timestamp = '%s'", timestamp)) + Expect(dateDeleted).ToNot(BeEmpty()) + Expect(dateDeleted).ToNot(Equal("In progress")) + + fpInfo := filepath.NewFilePathInfo(backupCluster, backupDir, timestamp, "", false) + backupDirCoordinator := fpInfo.GetDirForContent(-1) + Expect(backupDirCoordinator).ToNot(BeADirectory()) + }) + + It("deletes a local backup without --backup-dir", func() { + output := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir) + timestamp := getBackupTimestamp(string(output)) + + gpbackman( + "backup-delete", + "--history-db", historyDB, + "--timestamp", timestamp, + ) + + dateDeleted := queryHistoryDB(historyDB, + fmt.Sprintf("SELECT date_deleted FROM backups WHERE timestamp = '%s'", timestamp)) + Expect(dateDeleted).ToNot(BeEmpty()) + }) + + It("deletes with --cascade for incremental chain", func() { + fullOutput := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir, + "--leaf-partition-data") + fullTimestamp := getBackupTimestamp(string(fullOutput)) + + incrOutput := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir, + "--incremental", + "--leaf-partition-data") + incrTimestamp := getBackupTimestamp(string(incrOutput)) + + gpbackman( + "backup-delete", + "--history-db", historyDB, + "--timestamp", fullTimestamp, + "--backup-dir", backupDir, + "--cascade", + ) + + fullDeleted := queryHistoryDB(historyDB, + fmt.Sprintf("SELECT date_deleted FROM backups WHERE timestamp = '%s'", fullTimestamp)) + Expect(fullDeleted).ToNot(BeEmpty()) + + incrDeleted := queryHistoryDB(historyDB, + fmt.Sprintf("SELECT date_deleted FROM backups WHERE timestamp = '%s'", incrTimestamp)) + Expect(incrDeleted).ToNot(BeEmpty()) + }) + + It("deletes a plugin backup", func() { + copyPluginToAllHosts(backupConn, examplePluginExec) + + output := gpbackup(gpbackupPath, backupHelperPath, + "--plugin-config", examplePluginTestConfig) + timestamp := getBackupTimestamp(string(output)) + + gpbackman( + "backup-delete", + "--history-db", historyDB, + "--timestamp", timestamp, + "--plugin-config", examplePluginTestConfig, + ) + + dateDeleted := queryHistoryDB(historyDB, + fmt.Sprintf("SELECT date_deleted FROM backups WHERE timestamp = '%s'", timestamp)) + Expect(dateDeleted).ToNot(BeEmpty()) + Expect(dateDeleted).ToNot(Equal("In progress")) + }) + + It("fails for non-existent timestamp", func() { + _, err := gpbackmanWithError( + "backup-delete", + "--history-db", historyDB, + "--timestamp", "29991231235959", + "--backup-dir", backupDir, + ) + Expect(err).To(HaveOccurred()) + }) + + It("skips already-deleted backup without --force", func() { + output := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir) + timestamp := getBackupTimestamp(string(output)) + + gpbackman( + "backup-delete", + "--history-db", historyDB, + "--timestamp", timestamp, + "--backup-dir", backupDir, + ) + + // Second delete without --force should succeed without error. + // gpbackman silently skips already-deleted backups (logged at debug level). + gpbackman( + "backup-delete", + "--history-db", historyDB, + "--timestamp", timestamp, + "--backup-dir", backupDir, + ) + + dateDeleted := queryHistoryDB(historyDB, + fmt.Sprintf("SELECT date_deleted FROM backups WHERE timestamp = '%s'", timestamp)) + Expect(dateDeleted).ToNot(BeEmpty(), + "Backup should still be marked as deleted after second delete attempt") + }) + + It("re-deletes with --force", func() { + output := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir) + timestamp := getBackupTimestamp(string(output)) + + gpbackman( + "backup-delete", + "--history-db", historyDB, + "--timestamp", timestamp, + "--backup-dir", backupDir, + ) + + gpbackman( + "backup-delete", + "--history-db", historyDB, + "--timestamp", timestamp, + "--backup-dir", backupDir, + "--force", + "--ignore-errors", + ) + + dateDeleted := queryHistoryDB(historyDB, + fmt.Sprintf("SELECT date_deleted FROM backups WHERE timestamp = '%s'", timestamp)) + Expect(dateDeleted).ToNot(BeEmpty()) + }) + }) + + // ------------------------------------------------------------------ // + // backup-delete: local file cleanup on segments + // ------------------------------------------------------------------ // + Describe("backup-delete local file cleanup", func() { + var historyDB string + + BeforeEach(func() { + end_to_end_setup() + historyDB = getHistoryDBPathForCluster() + }) + + AfterEach(func() { + end_to_end_teardown() + }) + + It("removes backup files after deletion", func() { + output := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir, + "--single-backup-dir") + timestamp := getBackupTimestamp(string(output)) + + fpInfo := filepath.NewFilePathInfo(backupCluster, backupDir, timestamp, "", true) + backupTimestampDir := fpInfo.GetDirForContent(-1) + Expect(backupTimestampDir).To(BeADirectory(), + "Backup directory should exist before deletion") + + gpbackman( + "backup-delete", + "--history-db", historyDB, + "--timestamp", timestamp, + "--backup-dir", backupDir, + ) + + Expect(backupTimestampDir).ToNot(BeADirectory(), + "Backup directory should be removed after deletion") + }) + }) + + // ------------------------------------------------------------------ // + // backup-clean + // ------------------------------------------------------------------ // + Describe("backup-clean", func() { + var historyDB string + + BeforeEach(func() { + end_to_end_setup() + historyDB = getHistoryDBPathForCluster() + }) + + AfterEach(func() { + end_to_end_teardown() + }) + + It("cleans local backups with --before-timestamp", func() { + output1 := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir) + timestamp1 := getBackupTimestamp(string(output1)) + + output2 := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir) + timestamp2 := getBackupTimestamp(string(output2)) + + gpbackman( + "backup-clean", + "--history-db", historyDB, + "--before-timestamp", timestamp2, + "--backup-dir", backupDir, + ) + + dateDeleted1 := queryHistoryDB(historyDB, + fmt.Sprintf("SELECT date_deleted FROM backups WHERE timestamp = '%s'", timestamp1)) + Expect(dateDeleted1).ToNot(BeEmpty(), + fmt.Sprintf("Expected backup %s to be deleted", timestamp1)) + }) + + It("cleans local backups with --after-timestamp", func() { + output1 := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir) + timestamp1 := getBackupTimestamp(string(output1)) + + output2 := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir) + timestamp2 := getBackupTimestamp(string(output2)) + + gpbackman( + "backup-clean", + "--history-db", historyDB, + "--after-timestamp", timestamp1, + "--backup-dir", backupDir, + ) + + dateDeleted2 := queryHistoryDB(historyDB, + fmt.Sprintf("SELECT date_deleted FROM backups WHERE timestamp = '%s'", timestamp2)) + Expect(dateDeleted2).ToNot(BeEmpty(), + fmt.Sprintf("Expected backup %s to be deleted", timestamp2)) + }) + + It("cleans plugin backups with --before-timestamp", func() { + copyPluginToAllHosts(backupConn, examplePluginExec) + + output := gpbackup(gpbackupPath, backupHelperPath, + "--plugin-config", examplePluginTestConfig) + timestamp := getBackupTimestamp(string(output)) + + gpbackman( + "backup-clean", + "--history-db", historyDB, + "--before-timestamp", timestamp, + "--plugin-config", examplePluginTestConfig, + ) + + countStr := queryHistoryDB(historyDB, + fmt.Sprintf("SELECT count(*) FROM backups WHERE timestamp = '%s' AND date_deleted = ''", + timestamp)) + Expect(countStr).ToNot(BeEmpty()) + }) + + It("cleans local backups with --cascade for incremental chain", func() { + fullOutput := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir, + "--leaf-partition-data") + fullTimestamp := getBackupTimestamp(string(fullOutput)) + + _ = gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir, + "--incremental", + "--leaf-partition-data") + + gpbackman( + "backup-clean", + "--history-db", historyDB, + "--before-timestamp", fullTimestamp, + "--backup-dir", backupDir, + "--cascade", + ) + // Success if no error was thrown + }) + }) + + // ------------------------------------------------------------------ // + // history-clean + // ------------------------------------------------------------------ // + Describe("history-clean", func() { + var historyDB string + + BeforeEach(func() { + end_to_end_setup() + historyDB = getHistoryDBPathForCluster() + }) + + AfterEach(func() { + end_to_end_teardown() + }) + + It("cleans deleted backup entries from history DB", func() { + output := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir) + timestamp := getBackupTimestamp(string(output)) + + gpbackman( + "backup-delete", + "--history-db", historyDB, + "--timestamp", timestamp, + "--backup-dir", backupDir, + ) + + dateDeleted := queryHistoryDB(historyDB, + fmt.Sprintf("SELECT date_deleted FROM backups WHERE timestamp = '%s'", timestamp)) + Expect(dateDeleted).ToNot(BeEmpty()) + + // --before-timestamp uses strictly less than (<), so use a + // far-future cutoff to include the target timestamp. + gpbackman( + "history-clean", + "--history-db", historyDB, + "--before-timestamp", "99991231235959", + ) + + countStr := queryHistoryDB(historyDB, + fmt.Sprintf("SELECT count(*) FROM backups WHERE timestamp = '%s'", timestamp)) + Expect(countStr).To(Equal("0"), + fmt.Sprintf("Expected backup %s to be removed from history DB", timestamp)) + }) + + It("cleans with --older-than-days 0", func() { + output := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir) + timestamp := getBackupTimestamp(string(output)) + + gpbackman( + "backup-delete", + "--history-db", historyDB, + "--timestamp", timestamp, + "--backup-dir", backupDir, + ) + + gpbackman( + "history-clean", + "--history-db", historyDB, + "--older-than-days", "0", + ) + + countStr := queryHistoryDB(historyDB, + fmt.Sprintf("SELECT count(*) FROM backups WHERE timestamp = '%s'", timestamp)) + Expect(countStr).To(Equal("0")) + }) + + It("leaves non-deleted entries intact", func() { + output1 := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir) + timestamp1 := getBackupTimestamp(string(output1)) + + output2 := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir) + timestamp2 := getBackupTimestamp(string(output2)) + + gpbackman( + "backup-delete", + "--history-db", historyDB, + "--timestamp", timestamp1, + "--backup-dir", backupDir, + ) + + gpbackman( + "history-clean", + "--history-db", historyDB, + "--older-than-days", "0", + ) + + count1 := queryHistoryDB(historyDB, + fmt.Sprintf("SELECT count(*) FROM backups WHERE timestamp = '%s'", timestamp1)) + Expect(count1).To(Equal("0"), + "Deleted backup should be removed from history") + + count2 := queryHistoryDB(historyDB, + fmt.Sprintf("SELECT count(*) FROM backups WHERE timestamp = '%s'", timestamp2)) + Expect(count2).To(Equal("1"), + "Non-deleted backup should remain in history") + }) + }) + + // ------------------------------------------------------------------ // + // version & help + // ------------------------------------------------------------------ // + Describe("gpbackman --version", func() { + It("prints version information", func() { + output := gpbackman("--version") + Expect(string(output)).To(ContainSubstring("gpbackman")) + }) + }) + + Describe("gpbackman --help", func() { + It("prints help for all subcommands", func() { + for _, subcmd := range []string{ + "backup-info", "backup-delete", "backup-clean", + "history-clean", "report-info", + } { + output := gpbackman(subcmd, "--help") + Expect(string(output)).To(ContainSubstring(subcmd), + fmt.Sprintf("Help for %s should mention the command name", subcmd)) + } + }) + }) +}) From 596f7e5e84aadc66a5b63f735cd2038db02c3183 Mon Sep 17 00:00:00 2001 From: woblerr Date: Sun, 22 Mar 2026 15:28:18 +0300 Subject: [PATCH 06/39] Fix typo in flag names and function for backup clean and history clean commands. --- gpbackman/cmd/backup_clean.go | 14 +++++++------- gpbackman/cmd/constants.go | 2 +- gpbackman/cmd/history_clean.go | 16 ++++++++-------- gpbackman/gpbckpconfig/utils.go | 2 +- gpbackman/gpbckpconfig/utils_test.go | 4 ++-- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/gpbackman/cmd/backup_clean.go b/gpbackman/cmd/backup_clean.go index 1e610320..eb46ddba 100644 --- a/gpbackman/cmd/backup_clean.go +++ b/gpbackman/cmd/backup_clean.go @@ -19,7 +19,7 @@ var ( backupCleanAfterTimestamp string backupCleanPluginConfigFile string backupCleanBackupDir string - backupCleanOlderThenDays uint + backupCleanOlderThanDays uint backupCleanParallelProcesses int backupCleanCascade bool ) @@ -82,8 +82,8 @@ func init() { "delete all dependent backups", ) backupCleanCmd.PersistentFlags().UintVar( - &backupCleanOlderThenDays, - olderThenDaysFlagName, + &backupCleanOlderThanDays, + olderThanDaysFlagName, 0, "delete backup sets older than the given number of days", ) @@ -111,7 +111,7 @@ func init() { 1, "the number of parallel processes to delete local backups", ) - backupCleanCmd.MarkFlagsMutuallyExclusive(beforeTimestampFlagName, olderThenDaysFlagName, afterTimestampFlagName) + backupCleanCmd.MarkFlagsMutuallyExclusive(beforeTimestampFlagName, olderThanDaysFlagName, afterTimestampFlagName) } // These flag checks are applied only for backup-clean command. @@ -126,8 +126,8 @@ func doCleanBackupFlagValidation(flags *pflag.FlagSet) { } beforeTimestamp = backupCleanBeforeTimestamp } - if flags.Changed(olderThenDaysFlagName) { - beforeTimestamp = gpbckpconfig.GetTimestampOlderThen(backupCleanOlderThenDays) + if flags.Changed(olderThanDaysFlagName) { + beforeTimestamp = gpbckpconfig.GetTimestampOlderThan(backupCleanOlderThanDays) } // If after-timestamp flag is specified and have correct values. if flags.Changed(afterTimestampFlagName) { @@ -172,7 +172,7 @@ func doCleanBackupFlagValidation(flags *pflag.FlagSet) { } } if beforeTimestamp == "" && afterTimestamp == "" { - gplog.Error("%s", textmsg.ErrorTextUnableValidateValue(textmsg.ErrorValidationValue(), olderThenDaysFlagName, beforeTimestampFlagName, afterTimestampFlagName)) + gplog.Error("%s", textmsg.ErrorTextUnableValidateValue(textmsg.ErrorValidationValue(), olderThanDaysFlagName, beforeTimestampFlagName, afterTimestampFlagName)) execOSExit(exitErrorCode) } } diff --git a/gpbackman/cmd/constants.go b/gpbackman/cmd/constants.go index 28a3b209..0dce0aa3 100644 --- a/gpbackman/cmd/constants.go +++ b/gpbackman/cmd/constants.go @@ -26,7 +26,7 @@ const ( failedFlagName = "failed" cascadeFlagName = "cascade" forceFlagName = "force" - olderThenDaysFlagName = "older-than-days" + olderThanDaysFlagName = "older-than-days" beforeTimestampFlagName = "before-timestamp" afterTimestampFlagName = "after-timestamp" typeFlagName = "type" diff --git a/gpbackman/cmd/history_clean.go b/gpbackman/cmd/history_clean.go index dc62d5d6..473bf334 100644 --- a/gpbackman/cmd/history_clean.go +++ b/gpbackman/cmd/history_clean.go @@ -14,7 +14,7 @@ import ( // Flags for the gpbackman history-clean command (historyCleanCmd) var ( historyCleanBeforeTimestamp string - historyCleanOlderThenDays uint + historyCleanOlderThanDays uint ) var historyCleanCmd = &cobra.Command{ @@ -43,8 +43,8 @@ If the --history-db option is not specified, the history database will be search func init() { rootCmd.AddCommand(historyCleanCmd) historyCleanCmd.PersistentFlags().UintVar( - &historyCleanOlderThenDays, - olderThenDaysFlagName, + &historyCleanOlderThanDays, + olderThanDaysFlagName, 0, "delete information about backups older than the given number of days", ) @@ -54,10 +54,10 @@ func init() { "", "delete information about backups older than the given timestamp", ) - historyCleanCmd.MarkFlagsMutuallyExclusive(beforeTimestampFlagName, olderThenDaysFlagName) + historyCleanCmd.MarkFlagsMutuallyExclusive(beforeTimestampFlagName, olderThanDaysFlagName) } -// These flag checks are applied only for backup-clean command. +// These flag checks are applied only for history-clean command. func doCleanHistoryFlagValidation(flags *pflag.FlagSet) { var err error // If before-timestamp are specified and have correct values. @@ -69,11 +69,11 @@ func doCleanHistoryFlagValidation(flags *pflag.FlagSet) { } beforeTimestamp = historyCleanBeforeTimestamp } - if flags.Changed(olderThenDaysFlagName) { - beforeTimestamp = gpbckpconfig.GetTimestampOlderThen(historyCleanOlderThenDays) + if flags.Changed(olderThanDaysFlagName) { + beforeTimestamp = gpbckpconfig.GetTimestampOlderThan(historyCleanOlderThanDays) } if beforeTimestamp == "" { - gplog.Error("%s", textmsg.ErrorTextUnableValidateValue(textmsg.ErrorValidationValue(), olderThenDaysFlagName, beforeTimestampFlagName)) + gplog.Error("%s", textmsg.ErrorTextUnableValidateValue(textmsg.ErrorValidationValue(), olderThanDaysFlagName, beforeTimestampFlagName)) execOSExit(exitErrorCode) } } diff --git a/gpbackman/gpbckpconfig/utils.go b/gpbackman/gpbckpconfig/utils.go index 55954966..ef5e07bf 100644 --- a/gpbackman/gpbckpconfig/utils.go +++ b/gpbackman/gpbckpconfig/utils.go @@ -23,7 +23,7 @@ func CheckTimestamp(timestamp string) error { return nil } -func GetTimestampOlderThen(value uint) string { +func GetTimestampOlderThan(value uint) string { return time.Now().AddDate(0, 0, -int(value)).Format(Layout) } diff --git a/gpbackman/gpbckpconfig/utils_test.go b/gpbackman/gpbckpconfig/utils_test.go index 0703194c..825f2a74 100644 --- a/gpbackman/gpbckpconfig/utils_test.go +++ b/gpbackman/gpbckpconfig/utils_test.go @@ -121,10 +121,10 @@ var _ = Describe("utils tests", func() { }) }) - Describe("GetTimestampOlderThen", func() { + Describe("GetTimestampOlderThan", func() { It("returns timestamp within expected range", func() { input := uint(1) - got := GetTimestampOlderThen(input) + got := GetTimestampOlderThan(input) parsedTime, err := time.ParseInLocation(Layout, got, time.Now().Location()) Expect(err).ToNot(HaveOccurred()) now := time.Now() From b127c75a183c2016afdfd22af56f2dd04c654f59 Mon Sep 17 00:00:00 2001 From: woblerr Date: Sun, 22 Mar 2026 15:36:42 +0300 Subject: [PATCH 07/39] Add gpbackman path setup and skip condition for old backup version tests. --- end_to_end/end_to_end_suite_test.go | 1 + end_to_end/gpbackman_test.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/end_to_end/end_to_end_suite_test.go b/end_to_end/end_to_end_suite_test.go index 2eed0150..6a71a0d6 100644 --- a/end_to_end/end_to_end_suite_test.go +++ b/end_to_end/end_to_end_suite_test.go @@ -559,6 +559,7 @@ options: oldBackupVersionStr := os.Getenv("OLD_BACKUP_VERSION") _, restoreHelperPath, gprestorePath = buildAndInstallBinaries() + gpbackmanPath = fmt.Sprintf("%s/go/bin/gpbackman", operating.System.Getenv("HOME")) // Precompiled binaries will exist when running the ci job, `backward-compatibility` if _, err := os.Stat(fmt.Sprintf("/tmp/%s", oldBackupVersionStr)); err == nil { diff --git a/end_to_end/gpbackman_test.go b/end_to_end/gpbackman_test.go index 6db59af4..488bf644 100644 --- a/end_to_end/gpbackman_test.go +++ b/end_to_end/gpbackman_test.go @@ -40,6 +40,11 @@ func isSeparatorLine(line string) bool { } var _ = Describe("gpbackman end to end tests", func() { + BeforeEach(func() { + if useOldBackupVersion { + Skip("gpbackman tests are not applicable in old backup version mode") + } + }) // ------------------------------------------------------------------ // // backup-info From 237dc98798ca76693de5f0f311e83c512e6dfca7 Mon Sep 17 00:00:00 2001 From: woblerr Date: Sun, 22 Mar 2026 15:58:17 +0300 Subject: [PATCH 08/39] Refactor backup deletion commands to use direct SSH execution. --- go.mod | 2 +- go.sum | 2 - gpbackman/cmd/backup_delete.go | 77 ++++++---------------------------- 3 files changed, 13 insertions(+), 68 deletions(-) diff --git a/go.mod b/go.mod index 463ee9e4..3adaf23e 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 github.com/urfave/cli v1.22.13 - golang.org/x/crypto v0.21.0 golang.org/x/sys v0.18.0 golang.org/x/tools v0.12.0 gopkg.in/cheggaaa/pb.v1 v1.0.28 @@ -50,6 +49,7 @@ require ( github.com/mattn/go-runewidth v0.0.13 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index 7495cb68..ca12ca24 100644 --- a/go.sum +++ b/go.sum @@ -252,8 +252,6 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/gpbackman/cmd/backup_delete.go b/gpbackman/cmd/backup_delete.go index bae6bdf3..34647d93 100644 --- a/gpbackman/cmd/backup_delete.go +++ b/gpbackman/cmd/backup_delete.go @@ -8,9 +8,6 @@ import ( "os/exec" "strconv" "sync" - "time" - - "golang.org/x/crypto/ssh" "github.com/apache/cloudberry-go-libs/gplog" "github.com/apache/cloudberry-go-libs/operating" @@ -432,10 +429,8 @@ func executeDeleteBackupOnSegments(backupDir, backupDataBackupDir, backupName, s limit := make(chan bool, maxParallelProcesses) wg := &sync.WaitGroup{} errCh := make(chan error, len(configs)) - sshClientConf, err := getSSHConfig() - if err != nil { - return err - } + currentUser, _ := operating.System.CurrentUser() + userName := currentUser.Username // Check that the directory exists on all segment hosts. for _, config := range configs { wg.Add(1) @@ -447,7 +442,7 @@ func executeDeleteBackupOnSegments(backupDir, backupDataBackupDir, backupName, s go func(backupPath, host string) { defer func() { <-limit }() defer wg.Done() - checkBackupDirExistsOnSegments(gpbckpconfig.BackupDirPath(backupPath, backupName), host, sshClientConf, errCh) + checkBackupDirExistsOnSegments(gpbckpconfig.BackupDirPath(backupPath, backupName), host, userName, errCh) }(backupPath, config.Hostname) } // We should block the main function and wait for the WaitGroup to complete. @@ -483,7 +478,7 @@ func executeDeleteBackupOnSegments(backupDir, backupDataBackupDir, backupName, s go func(backupPath, host string) { defer func() { <-limit }() defer wg.Done() - deleteBackupDirOnSegments(gpbckpconfig.BackupDirPath(backupPath, backupName), host, sshClientConf, errCh) + deleteBackupDirOnSegments(gpbckpconfig.BackupDirPath(backupPath, backupName), host, userName, errCh) }(backupPath, config.Hostname) } wg.Wait() @@ -498,23 +493,15 @@ func executeDeleteBackupOnSegments(backupDir, backupDataBackupDir, backupName, s } return nil } -func checkBackupDirExistsOnSegments(path, host string, sshConf *ssh.ClientConfig, errCh chan error) { - connection, err := ssh.Dial("tcp", host+":22", sshConf) - if err != nil { - errCh <- err - return - } - defer connection.Close() +func runSSHCommand(remoteCmd, host, userName string) ([]byte, error) { + cmd := exec.Command("ssh", "-o", "StrictHostKeyChecking=no", fmt.Sprintf("%s@%s", userName, host), remoteCmd) + return cmd.CombinedOutput() +} - session, err := connection.NewSession() - if err != nil { - errCh <- err - return - } - defer session.Close() +func checkBackupDirExistsOnSegments(path, host, userName string, errCh chan error) { command := fmt.Sprintf("test -d %s", path) gplog.Debug("%s", textmsg.InfoTextCommandExecution(command, "on host", host)) - if err := session.Run(command); err != nil { + if _, err := runSSHCommand(command, host, userName); err != nil { gplog.Error("%s", textmsg.ErrorTextCommandExecutionFailed(err, command, "on host", host)) errCh <- textmsg.ErrorNotFoundBackupDirIn(fmt.Sprintf("%s on host %s", path, host)) return @@ -522,53 +509,13 @@ func checkBackupDirExistsOnSegments(path, host string, sshConf *ssh.ClientConfig gplog.Debug("%s", textmsg.InfoTextCommandExecutionSucceeded(command, "on host", host)) } -func deleteBackupDirOnSegments(path, host string, sshConf *ssh.ClientConfig, errCh chan error) { - connection, err := ssh.Dial("tcp", host+":22", sshConf) - if err != nil { - errCh <- err - return - } - defer connection.Close() - - session, err := connection.NewSession() - if err != nil { - errCh <- err - return - } - defer session.Close() +func deleteBackupDirOnSegments(path, host, userName string, errCh chan error) { command := fmt.Sprintf("rm -rf %s", path) gplog.Debug("%s", textmsg.InfoTextCommandExecution(command, "on host", host)) - if err := session.Run(command); err != nil { + if _, err := runSSHCommand(command, host, userName); err != nil { gplog.Error("%s", textmsg.ErrorTextCommandExecutionFailed(err, command, "on host", host)) errCh <- err return } gplog.Debug("%s", textmsg.InfoTextCommandExecutionSucceeded(command, "on host", host)) } - -func getSSHConfig() (*ssh.ClientConfig, error) { - currentUser, _ := operating.System.CurrentUser() - key, err := os.ReadFile(currentUser.HomeDir + "/.ssh/id_rsa") - if err != nil { - return nil, err - } - signer, err := ssh.ParsePrivateKey(key) - if err != nil { - return nil, err - } - // sshConfig is a configuration object for establishing an SSH connection. - // It contains the user's username, authentication method using public keys, - // and a host key callback that ignores insecure host keys. - sshConfig := &ssh.ClientConfig{ - User: currentUser.Username, - Auth: []ssh.AuthMethod{ - ssh.PublicKeys(signer), - }, - // Disable known_hosts check. - // This check also disables in gpbackup utility. - // #nosec G106 - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - Timeout: 30 * time.Second, - } - return sshConfig, nil -} From 74e065b9dd4c0ea9151ef27e122fe9958c3aef54 Mon Sep 17 00:00:00 2001 From: woblerr Date: Thu, 26 Mar 2026 17:54:48 +0300 Subject: [PATCH 09/39] Add documentation for gpBackMan. --- gpbackman/COMMANDS.md | 558 ++++++++++++++++++++++++++++++++++++++++++ gpbackman/README.md | 58 +++++ 2 files changed, 616 insertions(+) create mode 100644 gpbackman/COMMANDS.md create mode 100644 gpbackman/README.md diff --git a/gpbackman/COMMANDS.md b/gpbackman/COMMANDS.md new file mode 100644 index 00000000..031fb72f --- /dev/null +++ b/gpbackman/COMMANDS.md @@ -0,0 +1,558 @@ +- [Delete all existing backups older than the specified time condition (`backup-clean`)](#delete-all-existing-backups-older-than-the-specified-time-condition-backup-clean) + - [Examples](#examples) + - [Delete all backups from local storage older than the specified time condition](#delete-all-backups-from-local-storage-older-than-the-specified-time-condition) + - [Delete all backups using storage plugin older than n days](#delete-all-backups-using-storage-plugin-older-than-n-days) +- [Delete a specific existing backup (`backup-delete`)](#delete-a-specific-existing-backup-backup-delete) + - [Examples](#examples-1) + - [Delete existing backup from local storage](#delete-existing-backup-from-local-storage) + - [Delete existing backup using storage plugin](#delete-existing-backup-using-storage-plugin) +- [Display information about backups (`backup-info`)](#display-information-about-backups-backup-info) + - [Examples](#examples-2) +- [Clean deleted backups from the history database (`history-clean`)](#clean-deleted-backups-from-the-history-database-history-clean) + - [Examples](#examples-3) + - [Delete information about deleted backups from history database older than n days](#delete-information-about-deleted-backups-from-history-database-older-than-n-days) + - [Delete information about deleted backups from history database older than timestamp](#delete-information-about-deleted-backups-from-history-database-older-than-timestamp) +- [Display the report for a specific backup (`report-info`)](#display-the-report-for-a-specific-backup-report-info) + - [Examples](#examples-4) + - [Display the backup report from local storage](#display-the-backup-report-from-local-storage) + - [Display the backup report using storage plugin](#display-the-backup-report-using-storage-plugin) + +# Delete all existing backups older than the specified time condition (`backup-clean`) + +Available options for `backup-clean` command and their description: +```bash +./gpbackman backup-clean -h +elete all existing backups older than the specified time condition. + +To delete backup sets older than the given timestamp, use the --before-timestamp option. +To delete backup sets older than the given number of days, use the --older-than-day option. +To delete backup sets newer than the given timestamp, use the --after-timestamp option. +Only --older-than-days, --before-timestamp or --after-timestamp option must be specified. + +By default, the existence of dependent backups is checked and deletion process is not performed, +unless the --cascade option is passed in. + +By default, the deletion will be performed for local backup. + +The full path to the backup directory can be set using the --backup-dir option. + +For local backups the following logic are applied: + * If the --backup-dir option is specified, the deletion will be performed in provided path. + * If the --backup-dir option is not specified, but the backup was made with --backup-dir flag for gpbackup, the deletion will be performed in the backup manifest path. + * If the --backup-dir option is not specified and backup directory is not specified in backup manifest, the deletion will be performed in backup folder in the master and segments data directories. + * If backup is not local, the error will be returned. + +For control over the number of parallel processes and ssh connections to delete local backups, the --parallel-processes option can be used. + +The storage plugin config file location can be set using the --plugin-config option. +The full path to the file is required. In this case, the deletion will be performed using the storage plugin. + +For non local backups the following logic are applied: + * If the --plugin-config option is specified, the deletion will be performed using the storage plugin. + * If backup is local, the error will be returned. + +The gpbackup_history.db file location can be set using the --history-db option. +Can be specified only once. The full path to the file is required. +If the --history-db option is not specified, the history database will be searched in the current directory. + +Usage: + gpbackman backup-clean [flags] + +Flags: + --after-timestamp string delete backup sets newer than the given timestamp + --backup-dir string the full path to backup directory for local backups + --before-timestamp string delete backup sets older than the given timestamp + --cascade delete all dependent backups + -h, --help help for backup-clean + --older-than-days uint delete backup sets older than the given number of days + --parallel-processes int the number of parallel processes to delete local backups (default 1) + --plugin-config string the full path to plugin config file + +Global Flags: + --history-db string full path to the gpbackup_history.db file + --log-file string full path to log file directory, if not specified, the log file will be created in the $HOME/gpAdminLogs directory + --log-level-console string level for console logging (error, info, debug, verbose) (default "info") + --log-level-file string level for file logging (error, info, debug, verbose) (default "info") +``` + +## Examples +### Delete all backups from local storage older than the specified time condition + +Delete specific backup : +```bash +./gpbackman backup-clean \ + --before-timestamp 20240701100000 \ + --cascade +``` + +Delete specific backup with specifying the number of parallel processes: +```bash +./gpbackman backup-delete \ + --older-than-days 7 \ + --parallel-processes 5 +``` + +### Delete all backups using storage plugin older than n days +Delete all backups older than 7 days and all dependent backups: +```bash +./gpbackman backup-clean \ + --older-than-days 7 \ + --plugin-config /tmp/gpbackup_plugin_config.yaml \ + --cascade +``` + +# Delete a specific existing backup (`backup-delete`) + +Available options for `backup-delete` command and their description: + +```bash +./gpbackman backup-delete -h +Delete a specific existing backup. + +The --timestamp option must be specified. It could be specified multiple times. + +By default, the existence of dependent backups is checked and deletion process is not performed, +unless the --cascade option is passed in. + +If backup already deleted, the deletion process is skipped, unless --force option is specified. +If errors occur during the deletion process, the errors can be ignored using the --ignore-errors option. +The --ignore-errors option can be used only with --force option. + +By default, the deletion will be performed for local backup. + +The full path to the backup directory can be set using the --backup-dir option. + +For local backups the following logic are applied: + * If the --backup-dir option is specified, the deletion will be performed in provided path. + * If the --backup-dir option is not specified, but the backup was made with --backup-dir flag for gpbackup, the deletion will be performed in the backup manifest path. + * If the --backup-dir option is not specified and backup directory is not specified in backup manifest, the deletion will be performed in backup folder in the master and segments data directories. + * If backup is not local, the error will be returned. + +For control over the number of parallel processes and ssh connections to delete local backups, the --parallel-processes option can be used. + +The storage plugin config file location can be set using the --plugin-config option. +The full path to the file is required. In this case, the deletion will be performed using the storage plugin. + +For non local backups the following logic are applied: + * If the --plugin-config option is specified, the deletion will be performed using the storage plugin. + * If backup is local, the error will be returned. + +The gpbackup_history.db file location can be set using the --history-db option. +Can be specified only once. The full path to the file is required. +If the --history-db option is not specified, the history database will be searched in the current directory. + +Usage: + gpbackman backup-delete [flags] + +Flags: + --backup-dir string the full path to backup directory for local backups + --cascade delete all dependent backups for the specified backup timestamp + --force try to delete, even if the backup already mark as deleted + -h, --help help for backup-delete + --ignore-errors ignore errors when deleting backups + --parallel-processes int the number of parallel processes to delete local backups (default 1) + --plugin-config string the full path to plugin config file + --timestamp stringArray the backup timestamp for deleting, could be specified multiple times + +Global Flags: + --history-db string full path to the gpbackup_history.db file + --log-file string full path to log file directory, if not specified, the log file will be created in the $HOME/gpAdminLogs directory + --log-level-console string level for console logging (error, info, debug, verbose) (default "info") + --log-level-file string level for file logging (error, info, debug, verbose) (default "info") +``` + +## Examples +### Delete existing backup from local storage +Delete specific backup with specifying directory path: +```bash +./gpbackman backup-delete \ + --timestamp 20230809232817 \ + --backup-dir /some/path +``` + +Delete specific backup with specifying the number of parallel processes: +```bash +./gpbackman backup-delete \ + --timestamp 20230809212220 \ + --parallel-processes 5 +``` + +### Delete existing backup using storage plugin +Delete specific backup: +```bash +./gpbackman backup-delete \ + --timestamp 20230725101959 \ + --plugin-config /tmp/gpbackup_plugin_config.yaml +``` + +Delete specific backup and all dependent backups: +```bash +./gpbackman backup-delete \ + --timestamp 20230725101115 \ + --plugin-config /tmp/gpbackup_plugin_config.yaml \ + --cascade +``` + +# Display information about backups (`backup-info`) + +Available options for `backup-info` command and their description: + +```bash +./gpbackman backup-info -h +Display information about backups. + +By default, only active backups or backups with deletion status "In progress" from gpbackup_history.db are displayed. + +To display deleted backups, use the --deleted option. +To display failed backups, use the --failed option. +To display all backups, use --deleted and --failed options together. + +To display backups of a specific type, use the --type option. + +To display backups that include the specified table, use the --table option. +The formatting rules for .
match those of the --include-table option in gpbackup. + +To display backups that include the specified schema, use the --schema option. +The formatting rules for match those of the --include-schema option in gpbackup. + +To display backups that exclude the specified table, use the --table and --exclude options. +The formatting rules for .
match those of the --exclude-table option in gpbackup. + +To display backups that exclude the specified schema, use the --schema and --exclude options. +The formatting rules for match those of the --exclude-schema option in gpbackup. + +To display details about object filtering, use the --detail option. +The details are presented as follows, depending on the active filtering type: + * include-table / exclude-table: a comma-separated list of fully-qualified table names in the format .
; + * include-schema / exclude-schema: a comma-separated list of schema names; + * if no object filtering was used, the value is empty. + +To display a backup chain for a specific backup, use the --timestamp option. +In this mode, the backup with the specified timestamp and all of its dependent backups will be displayed. +The deleted and failed backups are always included in this mode. +To display object filtering details in this mode, use the --detail option. +When --timestamp is set, the following options cannot be used: --type, --table, --schema, --exclude, --failed, --deleted. + +To display the "object filtering details" column for all backups without using --timestamp, use the --detail option. + +The gpbackup_history.db file location can be set using the --history-db option. +Can be specified only once. The full path to the file is required. +If the --history-db option is not specified, the history database will be searched in the current directory. + +Usage: + gpbackman backup-info [flags] + +Flags: + --deleted show deleted backups + --detail show object filtering details + --exclude show backups that exclude the specific table (format .
) or schema + --failed show failed backups + -h, --help help for backup-info + --schema string show backups that include the specified schema + --table string show backups that include the specified table (format .
) + --timestamp string show backup info and its dependent backups for the specified timestamp + --type string backup type filter (full, incremental, data-only, metadata-only) + +Global Flags: + --history-db string full path to the gpbackup_history.db file + --log-file string full path to log file directory, if not specified, the log file will be created in the $HOME/gpAdminLogs directory + --log-level-console string level for console logging (error, info, debug, verbose) (default "info") + --log-level-file string level for file logging (error, info, debug, verbose) (default "info") +``` + +The following information is provided about each backup: +* `TIMESTAMP` - backup name, timestamp (`YYYYMMDDHHMMSS`) when the backup was taken; +* `DATE`- date in format `Mon Jan 02 2006 15:04:05` when the backup was taken; +* `STATUS`- backup status: `Success` or `Failure`; +* `DATABASE` - database name for which the backup was performed (specified by `--dbname` option on the `gpbackup` command). +* `TYPE` - backup type: + - `full` - contains user data, all global and local metadata for the database; + - `incremental` – contains user data, all global and local metadata changed since a previous full backup; + - `metadata-only` – contains only global and local metadata for the database; + - `data-only` – contains only user data from the database. + +* `OBJECT FILTERING` - whether the object filtering options were used when executing the `gpbackup` command: + - `include-schema` – at least one `--include-schema` option was specified; + - `exclude-schema` – at least one `--exclude-schema` option was specified; + - `include-table` – at least one `--include-table` option was specified; + - `exclude-table` – at least one `--exclude-table` option was specified; + - `""` - no options was specified. + +* `PLUGIN` - plugin name that was used to configure the backup destination; +* `DURATION` - backup duration in the format `hh:mm:ss`; +* `DATE DELETED` - backup deletion status: + - `In progress` - the deletion is in progress; + - `Plugin Backup Delete Failed` - last delete attempt failed to delete backup from plugin storage; + - `Local Delete Failed` - last delete attempt failed to delete backup from local storage.; + - `""` - if backup is active; + - date in format `Mon Jan 02 2006 15:04:05` - if backup is deleted and deletion timestamp is set. + +If the `--detail` option is specified, the following additional information is provided: +* `OBJECT FILTERING DETAILS` - details about object filtering: + - if `include-table` or `exclude-table` filtering was used, a comma-separated list of fully-qualified table names in the format `.
`; + - if `include-schema` or `exclude-schema` filtering was used, a comma-separated list of schema names; + - if no object filtering was used, the value is empty. + +If gpbackup is launched without specifying `--metadata-only` flag, but there were no tables that contain data for backup, then gpbackup will only perform a `metadata-only` backup. The logs will contain messages like `No tables in backup set contain data. Performing metadata-only backup instead.` As a result, gpBackMan will display such backups as `metadata-only`. + +## Examples + +Display info for active backups from `gpbackup_history.db`: +```bash +./gpbackman backup-info + + TIMESTAMP | DATE | STATUS | DATABASE | TYPE | OBJECT FILTERING | PLUGIN | DURATION | DATE DELETED +----------------+--------------------------+---------+----------+---------------+------------------+--------------------+----------+----------------------------- + 20230809232817 | Wed Aug 09 2023 23:28:17 | Success | demo | full | | | 04:00:03 | + 20230725110051 | Tue Jul 25 2023 11:00:51 | Success | demo | incremental | | gpbackup_s3_plugin | 00:00:20 | + 20230725102950 | Tue Jul 25 2023 10:29:50 | Success | demo | incremental | | gpbackup_s3_plugin | 00:00:19 | + 20230725102831 | Tue Jul 25 2023 10:28:31 | Success | demo | incremental | | gpbackup_s3_plugin | 00:00:18 | + 20230725101959 | Tue Jul 25 2023 10:19:59 | Success | demo | incremental | | gpbackup_s3_plugin | 00:00:22 | + 20230725101152 | Tue Jul 25 2023 10:11:52 | Success | demo | incremental | | gpbackup_s3_plugin | 00:00:18 | + 20230725101115 | Tue Jul 25 2023 10:11:15 | Success | demo | full | | gpbackup_s3_plugin | 00:00:20 | + 20230724090000 | Mon Jul 24 2023 09:00:00 | Success | demo | metadata-only | | gpbackup_s3_plugin | 00:05:17 | + 20230723082000 | Sun Jul 23 2023 08:20:00 | Success | demo | data-only | | gpbackup_s3_plugin | 00:35:17 | + 20230722100000 | Sat Jul 22 2023 10:00:00 | Success | demo | full | | gpbackup_s3_plugin | 00:25:17 | + 20230721090000 | Fri Jul 21 2023 09:00:00 | Success | demo | metadata-only | | gpbackup_s3_plugin | 00:04:17 | + 20230625110310 | Sun Jun 25 2023 11:03:10 | Success | demo | incremental | include-table | gpbackup_s3_plugin | 00:40:18 | Plugin Backup Delete Failed + 20230624101152 | Sat Jun 24 2023 10:11:52 | Success | demo | incremental | include-table | gpbackup_s3_plugin | 00:30:00 | + 20230623101115 | Fri Jun 23 2023 10:11:15 | Success | demo | full | include-table | gpbackup_s3_plugin | 01:01:00 | + 20230524101152 | Wed May 24 2023 10:11:52 | Success | demo | incremental | include-schema | gpbackup_s3_plugin | 00:30:00 | + 20230523101115 | Tue May 23 2023 10:11:15 | Success | demo | full | include-schema | gpbackup_s3_plugin | 01:01:00 | + ``` + +Display info for active full backups from `gpbackup_history.db`: +```bash +./gpbackman backup-info \ + --type full + + TIMESTAMP | DATE | STATUS | DATABASE | TYPE | OBJECT FILTERING | PLUGIN | DURATION | DATE DELETED +----------------+--------------------------+---------+----------+------+------------------+--------------------+----------+-------------- + 20230809232817 | Wed Aug 09 2023 23:28:17 | Success | demo | full | | | 04:00:03 | + 20230725101115 | Tue Jul 25 2023 10:11:15 | Success | demo | full | | gpbackup_s3_plugin | 00:00:20 | + 20230722100000 | Sat Jul 22 2023 10:00:00 | Success | demo | full | | gpbackup_s3_plugin | 00:25:17 | + 20230623101115 | Fri Jun 23 2023 10:11:15 | Success | demo | full | include-table | gpbackup_s3_plugin | 01:01:00 | + 20230523101115 | Tue May 23 2023 10:11:15 | Success | demo | full | include-schema | gpbackup_s3_plugin | 01:01:00 | +``` + +Find all backups, including deleted ones, containing the `test1` schema. +```bash +./gpbackman backup-info \ + --deleted \ + --schema test1 + + TIMESTAMP | DATE | STATUS | DATABASE | TYPE | OBJECT FILTERING | PLUGIN | DURATION | DATE DELETED +----------------+--------------------------+---------+----------+-------------+------------------+--------------------+----------+-------------------------- + 20230525101152 | Thu May 25 2023 10:11:52 | Success | demo | incremental | include-schema | gpbackup_s3_plugin | 00:30:00 | Sun Jun 25 2023 10:11:52 + 20230524101152 | Wed May 24 2023 10:11:52 | Success | demo | incremental | include-schema | gpbackup_s3_plugin | 00:30:00 | + 20230523101115 | Tue May 23 2023 10:11:15 | Success | demo | full | include-schema | gpbackup_s3_plugin | 01:01:00 | + ``` + +Display info for all backups, including deleted and failed ones, from `gpbackup_history.db`: +```bash +./gpbackman backup-info \ + --deleted \ + --failed \ + --history-db /data/master/gpseg-1/gpbackup_history.db + + TIMESTAMP | DATE | STATUS | DATABASE | TYPE | OBJECT FILTERING | PLUGIN | DURATION | DATE DELETED +----------------+--------------------------+---------+----------+---------------+------------------+--------------------+----------+----------------------------- + 20230809232817 | Wed Aug 09 2023 23:28:17 | Success | demo | full | | | 04:00:03 | + 20230806230400 | Sun Aug 06 2023 23:04:00 | Failure | demo | full | | gpbackup_s3_plugin | 00:00:38 | + 20230725110310 | Tue Jul 25 2023 11:03:10 | Success | demo | incremental | | gpbackup_s3_plugin | 00:00:18 | Wed Jul 26 2023 11:03:28 + 20230725110051 | Tue Jul 25 2023 11:00:51 | Success | demo | incremental | | gpbackup_s3_plugin | 00:00:20 | + 20230725102950 | Tue Jul 25 2023 10:29:50 | Success | demo | incremental | | gpbackup_s3_plugin | 00:00:19 | + 20230725102831 | Tue Jul 25 2023 10:28:31 | Success | demo | incremental | | gpbackup_s3_plugin | 00:00:18 | + 20230725101959 | Tue Jul 25 2023 10:19:59 | Success | demo | incremental | | gpbackup_s3_plugin | 00:00:22 | + 20230725101152 | Tue Jul 25 2023 10:11:52 | Success | demo | incremental | | gpbackup_s3_plugin | 00:00:18 | + 20230725101115 | Tue Jul 25 2023 10:11:15 | Success | demo | full | | gpbackup_s3_plugin | 00:00:20 | + 20230724090000 | Mon Jul 24 2023 09:00:00 | Success | demo | metadata-only | | gpbackup_s3_plugin | 00:05:17 | + 20230723082000 | Sun Jul 23 2023 08:20:00 | Success | demo | data-only | | gpbackup_s3_plugin | 00:35:17 | + 20230722100000 | Sat Jul 22 2023 10:00:00 | Success | demo | full | | gpbackup_s3_plugin | 00:25:17 | + 20230721090000 | Fri Jul 21 2023 09:00:00 | Success | demo | metadata-only | | gpbackup_s3_plugin | 00:04:17 | + 20230706230400 | Thu Jul 06 2023 23:04:00 | Failure | demo | full | | gpbackup_s3_plugin | 00:00:38 | + 20230625110310 | Sun Jun 25 2023 11:03:10 | Success | demo | incremental | include-table | gpbackup_s3_plugin | 00:40:18 | Plugin Backup Delete Failed + 20230624101152 | Sat Jun 24 2023 10:11:52 | Success | demo | incremental | include-table | gpbackup_s3_plugin | 00:30:00 | + 20230623101115 | Fri Jun 23 2023 10:11:15 | Success | demo | full | include-table | gpbackup_s3_plugin | 01:01:00 | + 20230606230400 | Tue Jun 06 2023 23:04:00 | Failure | demo | full | | gpbackup_s3_plugin | 00:00:38 | + 20230525101152 | Thu May 25 2023 10:11:52 | Success | demo | incremental | include-schema | gpbackup_s3_plugin | 00:30:00 | Sun Jun 25 2023 10:11:52 + 20230524101152 | Wed May 24 2023 10:11:52 | Success | demo | incremental | include-schema | gpbackup_s3_plugin | 00:30:00 | + 20230523101115 | Tue May 23 2023 10:11:15 | Success | demo | full | include-schema | gpbackup_s3_plugin | 01:01:00 | + ``` + +Display full backup with object filtering details: +```bash +./gpbackman backup-info \ + --type full \ + --detail + + TIMESTAMP | DATE | STATUS | DATABASE | TYPE | OBJECT FILTERING | PLUGIN | DURATION | DATE DELETED | OBJECT FILTERING DETAILS +----------------+--------------------------+---------+----------+------+------------------+--------------------+----------+--------------+-------------------------- + 20250915221743 | Mon Sep 15 2025 22:17:43 | Success | demo | full | | | 00:00:01 | | + 20250915221643 | Mon Sep 15 2025 22:16:43 | Success | demo | full | exclude-schema | gpbackup_s3_plugin | 00:00:01 | | sch1 + 20250915221631 | Mon Sep 15 2025 22:16:31 | Success | demo | full | include-table | gpbackup_s3_plugin | 00:00:01 | | sch2.tbl_c, sch2.tbl_d + 20250915221616 | Mon Sep 15 2025 22:16:16 | Success | demo | full | | gpbackup_s3_plugin | 00:00:05 | | + 20250915221553 | Mon Sep 15 2025 22:15:53 | Success | demo | full | exclude-table | | 00:00:02 | | sch1.tbl_b + 20250915221542 | Mon Sep 15 2025 22:15:42 | Success | demo | full | include-table | | 00:00:01 | | sch1.tbl_a + 20250915221531 | Mon Sep 15 2025 22:15:31 | Success | demo | full | | | 00:00:01 | | + +``` + +Display info for the backup chain for a specific backup. In this example, the backup with timestamp `20250913210921` is a full backup, and all its dependent incremental backups are displayed as well: +```bash +./gpbackman backup-info \ + --timestamp 20250913210921 \ + --detail + + TIMESTAMP | DATE | STATUS | DATABASE | TYPE | OBJECT FILTERING | PLUGIN | DURATION | DATE DELETED | OBJECT FILTERING DETAILS +----------------+--------------------------+---------+----------+-------------+------------------+--------------------+----------+--------------------------+-------------------------- + 20250915201446 | Mon Sep 15 2025 20:14:46 | Success | demo | incremental | include-table | gpbackup_s3_plugin | 00:00:02 | | sch2.tbl_c + 20250915201439 | Mon Sep 15 2025 20:14:39 | Success | demo | incremental | include-table | gpbackup_s3_plugin | 00:00:01 | | sch2.tbl_c + 20250915201307 | Mon Sep 15 2025 20:13:07 | Success | demo | incremental | include-table | gpbackup_s3_plugin | 00:00:02 | Mon Sep 15 2025 20:17:56 | sch2.tbl_c + 20250915200929 | Mon Sep 15 2025 20:09:29 | Success | demo | incremental | include-table | gpbackup_s3_plugin | 00:00:01 | | sch2.tbl_c + 20250913210957 | Sat Sep 13 2025 21:09:57 | Success | demo | incremental | include-table | gpbackup_s3_plugin | 00:00:01 | | sch2.tbl_c + 20250913210921 | Sat Sep 13 2025 21:09:21 | Success | demo | full | include-table | gpbackup_s3_plugin | 00:00:02 | | sch2.tbl_c +``` + +When using the option `--detail`, the column `OBJECT FILTERING DETAILS` may contain a large output. For pretty display, you can use `less -XS`: +```bash +./gpbackman backup-info --detail | less -XS +``` + +# Clean deleted backups from the history database (`history-clean`) + +Available options for `history-clean` command and their description: + +```bash +./gpbackman history-clean -h +Clean deleted backups from the history database. +Only the database is being cleaned up. + +Information is deleted only about deleted backups from gpbackup_history.db. Each backup must be deleted first. + +To delete information about backups older than the given timestamp, use the --before-timestamp option. +To delete information about backups older than the given number of days, use the --older-than-day option. +Only --older-than-days or --before-timestamp option must be specified, not both. + +The gpbackup_history.db file location can be set using the --history-db option. +Can be specified only once. The full path to the file is required. +If the --history-db option is not specified, the history database will be searched in the current directory. + +Usage: + gpbackman history-clean [flags] + +Flags: + --before-timestamp string delete information about backups older than the given timestamp + -h, --help help for history-clean + --older-than-days uint delete information about backups older than the given number of days + +Global Flags: + --history-db string full path to the gpbackup_history.db file + --log-file string full path to log file directory, if not specified, the log file will be created in the $HOME/gpAdminLogs directory + --log-level-console string level for console logging (error, info, debug, verbose) (default "info") + --log-level-file string level for file logging (error, info, debug, verbose) (default "info") +``` + +## Examples +### Delete information about deleted backups from history database older than n days +Delete information about deleted backups from history database older than 7 days: +```bash +./gpbackman history-clean \ + --older-than-days 7 \ +``` + +### Delete information about deleted backups from history database older than timestamp +Delete information about deleted backups from history database older than timestamp `20240101100000`: +```bash +./gpbackman history-clean \ + --before-timestamp 20240101100000 \ +``` + +# Display the report for a specific backup (`report-info`) + +Available options for `report-info` command and their description: + +```bash +./gpbackman.go report-info -h +Display the report for a specific backup. + +The --timestamp option must be specified. + +The report could be displayed only for active backups. + +The full path to the backup directory can be set using the --backup-dir option. +The full path to the data directory is required. + +For local backups the following logic are applied: + * If the --backup-dir option is specified, the report will be searched in provided path. + * If the --backup-dir option is not specified, but the backup was made with --backup-dir flag for gpbackup, the report will be searched in provided path from backup manifest. + * If the --backup-dir option is not specified and backup directory is not specified in backup manifest, the utility try to connect to local cluster and get master data directory. + If this information is available, the report will be in master data directory. + * If backup is not local, the error will be returned. + +The storage plugin config file location can be set using the --plugin-config option. +The full path to the file is required. + +For non local backups the following logic are applied: + * If the --plugin-config option is specified, the report will be searched in provided location. + * If backup is local, the error will be returned. + +Only --backup-dir or --plugin-config option can be specified, not both. + +If a custom plugin is used, it is required to specify the path to the directory with the repo file using the --plugin-report-file-path option. +It is not necessary to use the --plugin-report-file-path flag for the following plugins (the path is generated automatically): + * gpbackup_s3_plugin. + +The gpbackup_history.db file location can be set using the --history-db option. +Can be specified only once. The full path to the file is required. +If the --history-db option is not specified, the history database will be searched in the current directory. + +Usage: + gpbackman report-info [flags] + +Flags: + --backup-dir string the full path to backup directory + -h, --help help for report-info + --plugin-config string the full path to plugin config file + --plugin-report-file-path string the full path to plugin report file + --timestamp string the backup timestamp for report displaying + +Global Flags: + --history-db string full path to the gpbackup_history.db file + --log-file string full path to log file directory, if not specified, the log file will be created in the $HOME/gpAdminLogs directory + --log-level-console string level for console logging (error, info, debug, verbose) (default "info") + --log-level-file string level for file logging (error, info, debug, verbose) (default "info") +``` + +## Examples +### Display the backup report from local storage + +With specifying backup directory path: +```bash +./gpbackman report-info \ + --timestamp 20230809232817 \ + --backup-dir /some/path +``` + +With specifying backup directory path: +```bash +./gpbackman report-info \ + --timestamp 20230809232817 \ +``` + +### Display the backup report using storage plugin + +For `gpbackup_s3_plugin`: +```bash +./gpbackman report-info \ + --timestamp 20230725101959 \ + --plugin-config /tmp/gpbackup_plugin_config.yaml +``` + +For other plugins: +```bash +./gpbackman report-infodoc \ + --timestamp 20230725101959 \ + --plugin-config /tmp/gpbackup_plugin_config.yaml \ + --plugin-report-file-path /some/path/to/report +``` diff --git a/gpbackman/README.md b/gpbackman/README.md new file mode 100644 index 00000000..9c6fb72c --- /dev/null +++ b/gpbackman/README.md @@ -0,0 +1,58 @@ +# gpBackMan + +**gpBackMan** is designed to manage backups created by gpbackup. + +The utility works with `gpbackup_history.db` SQLite history database format. + +**gpBackMan** provides the following features: +* display information about backups; +* display the backup report for existing backups; +* delete existing backups from local storage or using storage plugins; +* delete all existing backups from local storage or using storage plugins older than the specified time condition; +* clean deleted backups from the history database; + +## Commands +### Introduction + +Available commands and global options: + +```bash +./gpbackman --help +gpBackMan - utility for managing backups created by gpbackup + +Usage: + gpbackman [command] + +Available Commands: + backup-clean Delete all existing backups older than the specified time condition + backup-delete Delete a specific existing backup + backup-info Display information about backups + completion Generate the autocompletion script for the specified shell + help Help about any command + history-clean Clean deleted backups from the history database + report-info Display the report for a specific backup + +Flags: + -h, --help help for gpbackman + --history-db string full path to the gpbackup_history.db file + --log-file string full path to log file directory, if not specified, the log file will be created in the $HOME/gpAdminLogs directory + --log-level-console string level for console logging (error, info, debug, verbose) (default "info") + --log-level-file string level for file logging (error, info, debug, verbose) (default "info") + -v, --version version for gpbackman + +Use "gpbackman [command] --help" for more information about a command. +``` + +### Detail info about commands + +Description of each command: +* [Delete all existing backups older than the specified time condition (`backup-clean`)](./COMMANDS.md#delete-all-existing-backups-older-than-the-specified-time-condition-backup-clean) +* [Delete a specific existing backup (`backup-delete`)](./COMMANDS.md#delete-a-specific-existing-backup-backup-delete) +* [Display information about backups (`backup-info`)](./COMMANDS.md#display-information-about-backups-backup-info) +* [Clean deleted backups from the history database (`history-clean`)](./COMMANDS.md#clean-deleted-backups-from-the-history-database-history-clean) +* [Display the report for a specific backup (`report-info`)](./COMMANDS.md#display-the-report-for-a-specific-backup-report-info) + +## About + +gpBackMan is part of the Apache Cloudberry Backup (Incubating) toolset. It is based on the original [gpbackman](https://github.com/woblerr/gpbackman) project. + From b58de876417c8ef5478594ba6dcac6805fbc3abd Mon Sep 17 00:00:00 2001 From: woblerr Date: Mon, 30 Mar 2026 16:10:06 +0300 Subject: [PATCH 10/39] Add Apache License header to files in gpBackMan. --- end_to_end/gpbackman_test.go | 19 +++++++++++++++++ gpbackman.go | 19 +++++++++++++++++ gpbackman/COMMANDS.md | 19 +++++++++++++++++ gpbackman/README.md | 19 +++++++++++++++++ gpbackman/cmd/backup_clean.go | 19 +++++++++++++++++ gpbackman/cmd/backup_delete.go | 19 +++++++++++++++++ gpbackman/cmd/backup_info.go | 21 ++++++++++++++++++- gpbackman/cmd/cmd_suite_test.go | 19 +++++++++++++++++ gpbackman/cmd/constants.go | 19 +++++++++++++++++ gpbackman/cmd/history_clean.go | 19 +++++++++++++++++ gpbackman/cmd/interfaces.go | 19 +++++++++++++++++ gpbackman/cmd/report_info.go | 19 +++++++++++++++++ gpbackman/cmd/root.go | 19 +++++++++++++++++ gpbackman/cmd/wrappers.go | 19 +++++++++++++++++ gpbackman/cmd/wrappers_test.go | 19 +++++++++++++++++ gpbackman/gpbckpconfig/cluster.go | 19 +++++++++++++++++ .../gpbckpconfig/gpbckpconfig_suite_test.go | 19 +++++++++++++++++ gpbackman/gpbckpconfig/helpers.go | 19 +++++++++++++++++ gpbackman/gpbckpconfig/helpers_test.go | 19 +++++++++++++++++ gpbackman/gpbckpconfig/struct.go | 19 +++++++++++++++++ gpbackman/gpbckpconfig/utils.go | 21 +++++++++++++++++-- gpbackman/gpbckpconfig/utils_db.go | 19 +++++++++++++++++ gpbackman/gpbckpconfig/utils_db_test.go | 19 +++++++++++++++++ gpbackman/gpbckpconfig/utils_test.go | 20 +++++++++++++++++- gpbackman/textmsg/error.go | 19 +++++++++++++++++ gpbackman/textmsg/error_test.go | 19 +++++++++++++++++ gpbackman/textmsg/info.go | 19 +++++++++++++++++ gpbackman/textmsg/info_test.go | 19 +++++++++++++++++ gpbackman/textmsg/textmsg_suite_test.go | 19 +++++++++++++++++ gpbackman/textmsg/warn.go | 19 +++++++++++++++++ gpbackman/textmsg/warn_test.go | 19 +++++++++++++++++ 31 files changed, 590 insertions(+), 4 deletions(-) diff --git a/end_to_end/gpbackman_test.go b/end_to_end/gpbackman_test.go index 488bf644..cabf46c1 100644 --- a/end_to_end/gpbackman_test.go +++ b/end_to_end/gpbackman_test.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package end_to_end_test import ( diff --git a/gpbackman.go b/gpbackman.go index e67fc0a3..4d9a65bf 100644 --- a/gpbackman.go +++ b/gpbackman.go @@ -1,6 +1,25 @@ //go:build gpbackman // +build gpbackman +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package main import . "github.com/apache/cloudberry-backup/gpbackman/cmd" diff --git a/gpbackman/COMMANDS.md b/gpbackman/COMMANDS.md index 031fb72f..c67947a3 100644 --- a/gpbackman/COMMANDS.md +++ b/gpbackman/COMMANDS.md @@ -1,3 +1,22 @@ + + - [Delete all existing backups older than the specified time condition (`backup-clean`)](#delete-all-existing-backups-older-than-the-specified-time-condition-backup-clean) - [Examples](#examples) - [Delete all backups from local storage older than the specified time condition](#delete-all-backups-from-local-storage-older-than-the-specified-time-condition) diff --git a/gpbackman/README.md b/gpbackman/README.md index 9c6fb72c..388ff898 100644 --- a/gpbackman/README.md +++ b/gpbackman/README.md @@ -1,3 +1,22 @@ + + # gpBackMan **gpBackMan** is designed to manage backups created by gpbackup. diff --git a/gpbackman/cmd/backup_clean.go b/gpbackman/cmd/backup_clean.go index eb46ddba..4f5e2179 100644 --- a/gpbackman/cmd/backup_clean.go +++ b/gpbackman/cmd/backup_clean.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package cmd import ( diff --git a/gpbackman/cmd/backup_delete.go b/gpbackman/cmd/backup_delete.go index 34647d93..14fa2795 100644 --- a/gpbackman/cmd/backup_delete.go +++ b/gpbackman/cmd/backup_delete.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package cmd import ( diff --git a/gpbackman/cmd/backup_info.go b/gpbackman/cmd/backup_info.go index 18671a3b..199311cc 100644 --- a/gpbackman/cmd/backup_info.go +++ b/gpbackman/cmd/backup_info.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package cmd import ( @@ -22,7 +41,7 @@ var ( backupInfoTableNameFilter string backupInfoSchemaNameFilter string backupInfoExcludeFilter bool - backupInfoTimestamp string + backupInfoTimestamp string backupInfoShowDetails bool ) diff --git a/gpbackman/cmd/cmd_suite_test.go b/gpbackman/cmd/cmd_suite_test.go index d84a7680..42570cd6 100644 --- a/gpbackman/cmd/cmd_suite_test.go +++ b/gpbackman/cmd/cmd_suite_test.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package cmd import ( diff --git a/gpbackman/cmd/constants.go b/gpbackman/cmd/constants.go index 0dce0aa3..8259b594 100644 --- a/gpbackman/cmd/constants.go +++ b/gpbackman/cmd/constants.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package cmd const ( diff --git a/gpbackman/cmd/history_clean.go b/gpbackman/cmd/history_clean.go index 473bf334..d5de643a 100644 --- a/gpbackman/cmd/history_clean.go +++ b/gpbackman/cmd/history_clean.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package cmd import ( diff --git a/gpbackman/cmd/interfaces.go b/gpbackman/cmd/interfaces.go index 91608514..81b7e045 100644 --- a/gpbackman/cmd/interfaces.go +++ b/gpbackman/cmd/interfaces.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package cmd import ( diff --git a/gpbackman/cmd/report_info.go b/gpbackman/cmd/report_info.go index b8a2cd21..3ac7428f 100644 --- a/gpbackman/cmd/report_info.go +++ b/gpbackman/cmd/report_info.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package cmd import ( diff --git a/gpbackman/cmd/root.go b/gpbackman/cmd/root.go index 15f59f48..1c113b37 100644 --- a/gpbackman/cmd/root.go +++ b/gpbackman/cmd/root.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package cmd import ( diff --git a/gpbackman/cmd/wrappers.go b/gpbackman/cmd/wrappers.go index 48549b51..79e34f38 100644 --- a/gpbackman/cmd/wrappers.go +++ b/gpbackman/cmd/wrappers.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package cmd import ( diff --git a/gpbackman/cmd/wrappers_test.go b/gpbackman/cmd/wrappers_test.go index 550a91d9..e378fbf1 100644 --- a/gpbackman/cmd/wrappers_test.go +++ b/gpbackman/cmd/wrappers_test.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package cmd import ( diff --git a/gpbackman/gpbckpconfig/cluster.go b/gpbackman/gpbckpconfig/cluster.go index ed59370f..93f6f7c8 100644 --- a/gpbackman/gpbckpconfig/cluster.go +++ b/gpbackman/gpbckpconfig/cluster.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package gpbckpconfig import ( diff --git a/gpbackman/gpbckpconfig/gpbckpconfig_suite_test.go b/gpbackman/gpbckpconfig/gpbckpconfig_suite_test.go index d8602978..9e4fea50 100644 --- a/gpbackman/gpbckpconfig/gpbckpconfig_suite_test.go +++ b/gpbackman/gpbckpconfig/gpbckpconfig_suite_test.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package gpbckpconfig import ( diff --git a/gpbackman/gpbckpconfig/helpers.go b/gpbackman/gpbckpconfig/helpers.go index 3336c026..56e4f681 100644 --- a/gpbackman/gpbckpconfig/helpers.go +++ b/gpbackman/gpbckpconfig/helpers.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package gpbckpconfig import ( diff --git a/gpbackman/gpbckpconfig/helpers_test.go b/gpbackman/gpbckpconfig/helpers_test.go index 8065daa1..db2e9c31 100644 --- a/gpbackman/gpbckpconfig/helpers_test.go +++ b/gpbackman/gpbckpconfig/helpers_test.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package gpbckpconfig import ( diff --git a/gpbackman/gpbckpconfig/struct.go b/gpbackman/gpbckpconfig/struct.go index 1ed1c09b..1472afc5 100644 --- a/gpbackman/gpbckpconfig/struct.go +++ b/gpbackman/gpbckpconfig/struct.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package gpbckpconfig const ( diff --git a/gpbackman/gpbckpconfig/utils.go b/gpbackman/gpbckpconfig/utils.go index ef5e07bf..e803baa0 100644 --- a/gpbackman/gpbckpconfig/utils.go +++ b/gpbackman/gpbckpconfig/utils.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package gpbckpconfig import ( @@ -152,5 +171,3 @@ func ReportFilePath(backupDir, timestamp string) string { func BackupDirPath(backupDir, timestamp string) string { return filepath.Join(backupDir, "backups", timestamp[0:8], timestamp) } - - diff --git a/gpbackman/gpbckpconfig/utils_db.go b/gpbackman/gpbckpconfig/utils_db.go index 102e7290..304c0907 100644 --- a/gpbackman/gpbckpconfig/utils_db.go +++ b/gpbackman/gpbckpconfig/utils_db.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package gpbckpconfig import ( diff --git a/gpbackman/gpbckpconfig/utils_db_test.go b/gpbackman/gpbckpconfig/utils_db_test.go index ebc1e6f1..17c6530b 100644 --- a/gpbackman/gpbckpconfig/utils_db_test.go +++ b/gpbackman/gpbckpconfig/utils_db_test.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package gpbckpconfig import ( diff --git a/gpbackman/gpbckpconfig/utils_test.go b/gpbackman/gpbckpconfig/utils_test.go index 825f2a74..53934567 100644 --- a/gpbackman/gpbckpconfig/utils_test.go +++ b/gpbackman/gpbckpconfig/utils_test.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package gpbckpconfig import ( @@ -232,5 +251,4 @@ var _ = Describe("utils tests", func() { }) }) - }) diff --git a/gpbackman/textmsg/error.go b/gpbackman/textmsg/error.go index c9d1e27e..46599331 100644 --- a/gpbackman/textmsg/error.go +++ b/gpbackman/textmsg/error.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package textmsg import ( diff --git a/gpbackman/textmsg/error_test.go b/gpbackman/textmsg/error_test.go index b94ef9da..b022f504 100644 --- a/gpbackman/textmsg/error_test.go +++ b/gpbackman/textmsg/error_test.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package textmsg import ( diff --git a/gpbackman/textmsg/info.go b/gpbackman/textmsg/info.go index 8ab403a6..dafe1408 100644 --- a/gpbackman/textmsg/info.go +++ b/gpbackman/textmsg/info.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package textmsg import ( diff --git a/gpbackman/textmsg/info_test.go b/gpbackman/textmsg/info_test.go index 03e38214..34d7c5c1 100644 --- a/gpbackman/textmsg/info_test.go +++ b/gpbackman/textmsg/info_test.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package textmsg import ( diff --git a/gpbackman/textmsg/textmsg_suite_test.go b/gpbackman/textmsg/textmsg_suite_test.go index c4affdcf..c3bd43fe 100644 --- a/gpbackman/textmsg/textmsg_suite_test.go +++ b/gpbackman/textmsg/textmsg_suite_test.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package textmsg import ( diff --git a/gpbackman/textmsg/warn.go b/gpbackman/textmsg/warn.go index d5bd0c5e..97dc366e 100644 --- a/gpbackman/textmsg/warn.go +++ b/gpbackman/textmsg/warn.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package textmsg import "fmt" diff --git a/gpbackman/textmsg/warn_test.go b/gpbackman/textmsg/warn_test.go index b5f3ad68..775d9c51 100644 --- a/gpbackman/textmsg/warn_test.go +++ b/gpbackman/textmsg/warn_test.go @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package textmsg import ( From ba48a3b0d93bdf6b003544e11d4326af2b2ec2fb Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 1 Apr 2026 12:09:21 +0800 Subject: [PATCH 11/39] Fix release tarball base version naming For RC tags like X.Y.Z-incubating-rcN, generate the source tarball filename and top-level directory using VERSION_FILE (without -rcN). $VERSION_FILE is input from the file VERSION. This keeps the voted bits ready for promotion without rebuilding and avoids -rcN showing up in the extracted source directory. --- cloudberry-backup-release.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cloudberry-backup-release.sh b/cloudberry-backup-release.sh index 090c41cb..a15697db 100755 --- a/cloudberry-backup-release.sh +++ b/cloudberry-backup-release.sh @@ -518,16 +518,16 @@ section "Staging release: $TAG" export COPYFILE_DISABLE=1 export COPY_EXTENDED_ATTRIBUTES_DISABLE=1 - git archive --format=tar --prefix="apache-cloudberry-backup-${TAG}/" "$TAG" | tar -x -C "$TMP_DIR" + git archive --format=tar --prefix="apache-cloudberry-backup-${VERSION_FILE}-incubating/" "$TAG" | tar -x -C "$TMP_DIR" # Archive submodules if any if [ -s .gitmodules ]; then git submodule foreach --recursive --quiet " echo \"Archiving submodule: \$sm_path\" fullpath=\"\$toplevel/\$sm_path\" - destpath=\"$TMP_DIR/apache-cloudberry-backup-${TAG}/\$sm_path\" + destpath=\"$TMP_DIR/apache-cloudberry-backup-${VERSION_FILE}-incubating/\$sm_path\" mkdir -p \"\$destpath\" - git -C \"\$fullpath\" archive --format=tar --prefix=\"\$sm_path/\" HEAD | tar -x -C \"$TMP_DIR/apache-cloudberry-backup-${TAG}\" + git -C \"\$fullpath\" archive --format=tar --prefix=\"\$sm_path/\" HEAD | tar -x -C \"$TMP_DIR/apache-cloudberry-backup-${VERSION_FILE}-incubating\" " fi @@ -536,25 +536,25 @@ section "Staging release: $TAG" echo "Cleaning macOS extended attributes from extracted files..." # Remove all extended attributes recursively if command -v xattr >/dev/null 2>&1; then - find "$TMP_DIR/apache-cloudberry-backup-${TAG}" -type f -exec xattr -c {} \; 2>/dev/null || true + find "$TMP_DIR/apache-cloudberry-backup-${VERSION_FILE}-incubating" -type f -exec xattr -c {} \; 2>/dev/null || true echo "[OK] Extended attributes cleaned using xattr" fi # Remove any ._* files that might have been created - find "$TMP_DIR/apache-cloudberry-backup-${TAG}" -name '._*' -delete 2>/dev/null || true - find "$TMP_DIR/apache-cloudberry-backup-${TAG}" -name '.DS_Store' -delete 2>/dev/null || true - find "$TMP_DIR/apache-cloudberry-backup-${TAG}" -name '__MACOSX' -type d -exec rm -rf {} \; 2>/dev/null || true + find "$TMP_DIR/apache-cloudberry-backup-${VERSION_FILE}-incubating" -name '._*' -delete 2>/dev/null || true + find "$TMP_DIR/apache-cloudberry-backup-${VERSION_FILE}-incubating" -name '.DS_Store' -delete 2>/dev/null || true + find "$TMP_DIR/apache-cloudberry-backup-${VERSION_FILE}-incubating" -name '__MACOSX' -type d -exec rm -rf {} \; 2>/dev/null || true echo "[OK] macOS-specific files removed" fi # Create tarball using the detected tar tool if [[ "$DETECTED_PLATFORM" == "macOS" ]]; then echo "Using GNU tar for cross-platform compatibility..." - $DETECTED_TAR_TOOL --exclude='._*' --exclude='.DS_Store' --exclude='__MACOSX' -czf "$TAR_NAME" -C "$TMP_DIR" "apache-cloudberry-backup-${TAG}" + $DETECTED_TAR_TOOL --exclude='._*' --exclude='.DS_Store' --exclude='__MACOSX' -czf "$TAR_NAME" -C "$TMP_DIR" "apache-cloudberry-backup-${VERSION_FILE}-incubating" echo "INFO: macOS detected - applied extended attribute cleanup and GNU tar" else # On other platforms, use standard tar - $DETECTED_TAR_TOOL -czf "$TAR_NAME" -C "$TMP_DIR" "apache-cloudberry-backup-${TAG}" + $DETECTED_TAR_TOOL -czf "$TAR_NAME" -C "$TMP_DIR" "apache-cloudberry-backup-${VERSION_FILE}-incubating" fi rm -rf "$TMP_DIR" From 1b8641721308b33e2d60360fb54118d88175e9fc Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 8 Apr 2026 15:16:33 +0800 Subject: [PATCH 12/39] Include Apache compliance files in binary packages (#81) Add LICENSE, NOTICE, and DISCLAIMER files to the package target in Makefile to ensure Apache project compliance requirements are met when distributing binary releases. These files are now copied to the package directory alongside the binaries and install script, and will be included in the final tar.gz distribution. --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index c1f8cf05..9ec10c47 100644 --- a/Makefile +++ b/Makefile @@ -184,6 +184,10 @@ package: @GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=$(CGO) go build -tags '$(HELPER)' -o $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/bin/$(HELPER) --ldflags '-X $(HELPER_VERSION_STR)' @GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=$(CGO) go build -tags '$(S3PLUGIN)' -o $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/bin/$(S3PLUGIN) --ldflags '-X $(S3PLUGIN_VERSION_STR)' @GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=$(CGO) go build -tags '$(GPBACKMAN)' -o $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/bin/$(GPBACKMAN) --ldflags '-X $(GPBACKMAN_VERSION_STR)' + @echo "Copying Apache compliance files..." + @cp LICENSE $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/ + @cp NOTICE $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/ + @cp DISCLAIMER $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/ @echo "Creating install script..." @echo '#!/bin/bash' > $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh @echo 'set -e' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh From eaf5241691bea7b60e31af2a17b2e0b95a2be12d Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 8 Apr 2026 10:19:02 +0800 Subject: [PATCH 13/39] asf.yaml: add woblerr as a collaborator --- .asf.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.asf.yaml b/.asf.yaml index 9a377230..5c5f6472 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -45,6 +45,8 @@ github: merge: false # enable rebase button: rebase: true + collaborators: + - woblerr protected_branches: main: required_status_checks: From 57f513c45ffefebb81f179465eef64a98a98e1f7 Mon Sep 17 00:00:00 2001 From: woblerr Date: Tue, 7 Apr 2026 11:43:18 +0300 Subject: [PATCH 14/39] Bump Go version from 1.21 to 1.24. Update Go to 1.24. Fix non-constant format string errors for Go 1.24. Fix patterns applied: - func(err.Error()) -> func("%s", err.Error()) - func(fmt.Sprintf("...", args)) -> func("...", args) --- backup/backup.go | 14 +++++++------- backup/data.go | 2 +- backup/incremental.go | 6 +++--- backup/predata_functions.go | 8 ++++---- backup/queries_acl.go | 2 +- backup/wrappers.go | 4 ++-- go.mod | 2 +- plugins/s3plugin/backup.go | 2 +- plugins/s3plugin/restore.go | 6 +++--- plugins/s3plugin/s3plugin.go | 2 +- report/report.go | 6 +++--- restore/data.go | 10 +++++----- restore/restore.go | 4 ++-- restore/validate.go | 4 ++-- restore/wrappers.go | 4 ++-- utils/gpexpand_sensor.go | 9 ++++----- utils/io.go | 4 ++-- utils/plugin.go | 12 ++++++------ utils/util.go | 2 +- 19 files changed, 51 insertions(+), 52 deletions(-) diff --git a/backup/backup.go b/backup/backup.go index 0c8249ac..2711f83c 100644 --- a/backup/backup.go +++ b/backup/backup.go @@ -387,7 +387,7 @@ func DoTeardown() { if err := recover(); err != nil { // gplog's Fatal will cause a panic with error code 2 if gplog.GetErrorCode() != 2 { - gplog.Error(fmt.Sprintf("%v: %s", err, debug.Stack())) + gplog.Error("%v: %s", err, debug.Stack()) gplog.SetErrorCode(2) } else { errStr = fmt.Sprintf("%v", err) @@ -448,12 +448,12 @@ func DoTeardown() { if pluginConfig != nil { err = pluginConfig.BackupFile(configFilename) if err != nil { - gplog.Error(fmt.Sprintf("%v", err)) + gplog.Error("%v", err) return } err = pluginConfig.BackupFile(reportFilename) if err != nil { - gplog.Error(fmt.Sprintf("%v", err)) + gplog.Error("%v", err) return } } @@ -499,7 +499,7 @@ func DoCleanup(backupFailed bool) { if wasTerminated { err := utils.CheckAgentErrorsOnSegments(globalCluster, globalFPInfo) if err != nil { - gplog.Error(err.Error()) + gplog.Error("%s", err.Error()) } } } @@ -519,12 +519,12 @@ func DoCleanup(backupFailed bool) { historyDBName := globalFPInfo.GetBackupHistoryDatabasePath() historyDB, err := history.InitializeHistoryDatabase(historyDBName) if err != nil { - gplog.Error(fmt.Sprintf("Unable to update history database. Error: %v", err)) + gplog.Error("Unable to update history database. Error: %v", err) } else { _, err := historyDB.Exec(fmt.Sprintf("UPDATE backups SET status='%s', end_time='%s' WHERE timestamp='%s'", statusString, backupReport.BackupConfig.EndTime, globalFPInfo.Timestamp)) historyDB.Close() if err != nil { - gplog.Error(fmt.Sprintf("Unable to update history database. Error: %v", err)) + gplog.Error("Unable to update history database. Error: %v", err) } } } @@ -559,7 +559,7 @@ func cancelBlockedQueries(timestamp string) { return } - gplog.Info(fmt.Sprintf("Canceling %d blocked queries", len(pids))) + gplog.Info("Canceling %d blocked queries", len(pids)) // Cancel all gpbackup queries waiting for a lock for _, pid := range pids { conn.MustExec(fmt.Sprintf("SELECT pg_cancel_backend(%d)", pid)) diff --git a/backup/data.go b/backup/data.go index 85ee1de9..9d937bae 100644 --- a/backup/data.go +++ b/backup/data.go @@ -117,7 +117,7 @@ func BackupSingleTableData(table Table, rowsCopiedMap map[uint32]int64, counters logMessage := fmt.Sprintf("%sWriting data for table %s to file", workerInfo, table.FQN()) // Avoid race condition by incrementing counters in call to sprintf tableCount := fmt.Sprintf(" (table %d of %d)", atomic.AddInt64(&counters.NumRegTables, 1), counters.TotalRegTables) - utils.LogProgress(logMessage + tableCount) + utils.LogProgress("%s", logMessage+tableCount) destinationToWrite := "" if MustGetFlagBool(options.SINGLE_DATA_FILE) { diff --git a/backup/incremental.go b/backup/incremental.go index 0326c940..dcf70619 100644 --- a/backup/incremental.go +++ b/backup/incremental.go @@ -86,7 +86,7 @@ func GetLatestMatchingBackupConfig(historyDBPath string, currentBackupConfig *hi ORDER BY timestamp DESC`, whereClause) timestampRows, err := historyDB.Query(getBackupTimetampsQuery) if err != nil { - gplog.Error(err.Error()) + gplog.Error("%s", err.Error()) return nil } defer timestampRows.Close() @@ -96,7 +96,7 @@ func GetLatestMatchingBackupConfig(historyDBPath string, currentBackupConfig *hi var timestamp string err = timestampRows.Scan(×tamp) if err != nil { - gplog.Error(err.Error()) + gplog.Error("%s", err.Error()) return nil } timestamps = append(timestamps, timestamp) @@ -105,7 +105,7 @@ func GetLatestMatchingBackupConfig(historyDBPath string, currentBackupConfig *hi for _, ts := range timestamps { backupConfig, err := history.GetBackupConfig(ts, historyDB) if err != nil { - gplog.Error(err.Error()) + gplog.Error("%s", err.Error()) return nil } if !backupConfig.Failed() && matchesIncrementalFlags(backupConfig, currentBackupConfig) { diff --git a/backup/predata_functions.go b/backup/predata_functions.go index f7953932..9f6324ef 100644 --- a/backup/predata_functions.go +++ b/backup/predata_functions.go @@ -393,7 +393,7 @@ func PrintCreateTransformStatement(metadataFile *utils.FileWithByteCount, objToc TypeFQN := fmt.Sprintf("%s.%s", transform.TypeNamespace, transform.TypeName) if !fromSQLIsDefined && !toSQLIsDefined { - gplog.Warn(fmt.Sprintf("Skipping invalid transform object for type %s and language %s; At least one of FROM and TO functions should be specified.", TypeFQN, transform.LanguageName)) + gplog.Warn("Skipping invalid transform object for type %s and language %s; At least one of FROM and TO functions should be specified.", TypeFQN, transform.LanguageName) return } start := metadataFile.ByteCount @@ -401,7 +401,7 @@ func PrintCreateTransformStatement(metadataFile *utils.FileWithByteCount, objToc if fromSQLIsDefined { statement += fmt.Sprintf("FROM SQL WITH FUNCTION %s", fromSQLFunc.FQN()) } else { - gplog.Warn(fmt.Sprintf("No FROM function found for transform object with type %s and language %s\n", TypeFQN, transform.LanguageName)) + gplog.Warn("No FROM function found for transform object with type %s and language %s\n", TypeFQN, transform.LanguageName) } if toSQLIsDefined { @@ -410,10 +410,10 @@ func PrintCreateTransformStatement(metadataFile *utils.FileWithByteCount, objToc } statement += fmt.Sprintf("TO SQL WITH FUNCTION %s", toSQLFunc.FQN()) } else { - gplog.Warn(fmt.Sprintf("No TO function found for transform object with type %s and language %s\n", TypeFQN, transform.LanguageName)) + gplog.Warn("No TO function found for transform object with type %s and language %s\n", TypeFQN, transform.LanguageName) } statement += ");" - metadataFile.MustPrintf(statement) + metadataFile.MustPrintf("%s", statement) section, entry := transform.GetMetadataEntry() tier := globalTierMap[transform.GetUniqueID()] objToc.AddMetadataEntry(section, entry, start, metadataFile.ByteCount, tier) diff --git a/backup/queries_acl.go b/backup/queries_acl.go index b47a4be7..11d02b6b 100644 --- a/backup/queries_acl.go +++ b/backup/queries_acl.go @@ -138,7 +138,7 @@ type MetadataQueryStruct struct { } func GetMetadataForObjectType(connectionPool *dbconn.DBConn, params MetadataQueryParams) MetadataMap { - gplog.Verbose("Getting object type metadata from " + params.CatalogTable) + gplog.Verbose("Getting object type metadata from %s", params.CatalogTable) tableName := params.CatalogTable nameCol := "''" diff --git a/backup/wrappers.go b/backup/wrappers.go index c32df413..284348cd 100644 --- a/backup/wrappers.go +++ b/backup/wrappers.go @@ -63,7 +63,7 @@ func initializeConnectionPool(timestamp string) { numConns = 2 } - gplog.Verbose(fmt.Sprintf("Initializing %d database connections", numConns)) + gplog.Verbose("Initializing %d database connections", numConns) connectionPool.MustConnect(numConns) utils.ValidateGPDBVersionCompatibility(connectionPool) @@ -183,7 +183,7 @@ func createBackupLockFile(timestamp string) { gplog.FatalOnError(err) err = backupLockFile.TryLock() if err != nil { - gplog.Error(err.Error()) + gplog.Error("%s", err.Error()) gplog.Fatal(errors.Errorf("A backup with timestamp %s is already in progress. Wait 1 second and try the backup again.", timestamp), "") } } diff --git a/go.mod b/go.mod index 3adaf23e..cec552ff 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/apache/cloudberry-backup -go 1.21 +go 1.24.0 require ( github.com/DATA-DOG/go-sqlmock v1.5.0 diff --git a/plugins/s3plugin/backup.go b/plugins/s3plugin/backup.go index a2fcbea5..0a7d0c50 100644 --- a/plugins/s3plugin/backup.go +++ b/plugins/s3plugin/backup.go @@ -155,7 +155,7 @@ func BackupDirectoryParallel(c *cli.Context) error { totalBytes += bytes msg := fmt.Sprintf("Uploaded %d bytes for %s in %v", bytes, filepath.Base(fileKey), elapsed.Round(time.Millisecond)) - gplog.Verbose(msg) + gplog.Verbose("%s", msg) fmt.Println(msg) } else { finalErr = err diff --git a/plugins/s3plugin/restore.go b/plugins/s3plugin/restore.go index b7f31f23..deb3cc98 100644 --- a/plugins/s3plugin/restore.go +++ b/plugins/s3plugin/restore.go @@ -44,7 +44,7 @@ func RestoreFile(c *cli.Context) error { if err != nil { fileErr := os.Remove(fileName) if fileErr != nil { - gplog.Error(fileErr.Error()) + gplog.Error("%s", fileErr.Error()) } return err } @@ -95,7 +95,7 @@ func RestoreDirectory(c *cli.Context) error { if err != nil { fileErr := os.Remove(filename) if fileErr != nil { - gplog.Error(fileErr.Error()) + gplog.Error("%s", fileErr.Error()) } return err } @@ -175,7 +175,7 @@ func RestoreDirectoryParallel(c *cli.Context) error { numFiles++ msg := fmt.Sprintf("Downloaded %d bytes for %s in %v", bytes, filepath.Base(fileKey), elapsed.Round(time.Millisecond)) - gplog.Verbose(msg) + gplog.Verbose("%s", msg) fmt.Println(msg) } else { finalErr = err diff --git a/plugins/s3plugin/s3plugin.go b/plugins/s3plugin/s3plugin.go index 92f0e080..9a2a4e83 100644 --- a/plugins/s3plugin/s3plugin.go +++ b/plugins/s3plugin/s3plugin.go @@ -336,7 +336,7 @@ func DeleteBackup(c *cli.Context) error { if !IsValidTimestamp(timestamp) { msg := fmt.Sprintf("delete requires a with format "+ "YYYYMMDDHHMMSS, but received: %s", timestamp) - return fmt.Errorf(msg) + return fmt.Errorf("%s", msg) } date := timestamp[0:8] diff --git a/report/report.go b/report/report.go index 0b3a2153..d50ad9e5 100644 --- a/report/report.go +++ b/report/report.go @@ -236,9 +236,9 @@ func logOutputReport(reportFile io.WriteCloser, reportInfo []LineInfo) { for _, lineInfo := range reportInfo { if lineInfo.Key == "" { - utils.MustPrintf(reportFile, fmt.Sprintf("\n")) + utils.MustPrintf(reportFile, "\n") } else { - utils.MustPrintf(reportFile, fmt.Sprintf("%-*s%s\n", maxSize+3, lineInfo.Key, lineInfo.Value)) + utils.MustPrintf(reportFile, "%-*s%s\n", maxSize+3, lineInfo.Key, lineInfo.Value) } } } @@ -279,7 +279,7 @@ func PrintObjectCounts(reportFile io.WriteCloser, objectCounts map[string]int) { objectStr += fmt.Sprintf("%-*s%d\n", maxSize+3, strings.ToLower(object), objectCounts[object]) } } - utils.MustPrintf(reportFile, objectStr) + utils.MustPrintf(reportFile, "%s", objectStr) } /* diff --git a/restore/data.go b/restore/data.go index ee073379..11d2deea 100644 --- a/restore/data.go +++ b/restore/data.go @@ -111,7 +111,7 @@ func restoreSingleTableData(fpInfo *filepath.FilePathInfo, entry toc.Coordinator partialRowsRestored, copyErr := CopyTableIn(connectionPool, tableName, entry.AttributeString, destinationToRead, backupConfig.SingleDataFile, whichConn) if copyErr != nil { - gplog.Error(copyErr.Error()) + gplog.Error("%s", copyErr.Error()) if MustGetFlagBool(options.ON_ERROR_CONTINUE) { if ((connectionPool.Version.IsGPDB() && connectionPool.Version.AtLeast("6")) || connectionPool.Version.IsCBDB()) && backupConfig.SingleDataFile { // inform segment helpers to skip this entry @@ -149,7 +149,7 @@ func restoreSingleTableData(fpInfo *filepath.FilePathInfo, entry toc.Coordinator // The subsequent division in gprestore leads to a miscalculated row count, causing this check to fail. err := CheckRowsRestored(numRowsRestored, numRowsBackedUp, tableName) if err != nil { - gplog.Error(err.Error()) + gplog.Error("%s", err.Error()) return err } @@ -158,12 +158,12 @@ func restoreSingleTableData(fpInfo *filepath.FilePathInfo, entry toc.Coordinator if entry.IsReplicated && (origSize < destSize) { err := ExpandReplicatedTable(origSize, tableName, whichConn) if err != nil { - gplog.Error(err.Error()) + gplog.Error("%s", err.Error()) } } else { err := RedistributeTableData(tableName, whichConn) if err != nil { - gplog.Error(err.Error()) + gplog.Error("%s", err.Error()) } } } @@ -294,7 +294,7 @@ func restoreDataFromTimestamp(fpInfo filepath.FilePathInfo, dataEntries []toc.Co gplog.Verbose("Truncating table %s prior to restoring data", tableName) _, err := connectionPool.Exec(`TRUNCATE `+tableName, whichConn) if err != nil { - gplog.Error(err.Error()) + gplog.Error("%s", err.Error()) } } if err == nil { diff --git a/restore/restore.go b/restore/restore.go index c91d0ec1..91c6c5ab 100644 --- a/restore/restore.go +++ b/restore/restore.go @@ -612,7 +612,7 @@ func DoTeardown() { if err := recover(); err != nil { // Check if gplog.Fatal did not cause the panic if gplog.GetErrorCode() != 2 { - gplog.Error(fmt.Sprintf("%v: %s", err, debug.Stack())) + gplog.Error("%v: %s", err, debug.Stack()) gplog.SetErrorCode(2) } else { errStr = fmt.Sprintf("%+v", err) @@ -733,7 +733,7 @@ func DoCleanup(restoreFailed bool) { if wasTerminated { err := utils.CheckAgentErrorsOnSegments(globalCluster, globalFPInfo) if err != nil { - gplog.Error(err.Error()) + gplog.Error("%s", err.Error()) } } } diff --git a/restore/validate.go b/restore/validate.go index be0ee281..f9fb55fb 100644 --- a/restore/validate.go +++ b/restore/validate.go @@ -138,7 +138,7 @@ WHERE quote_ident(n.nspname) || '.' || quote_ident(c.relname) IN (%s)`, quotedTa errMsg = fmt.Sprintf("Relation %s already exists", relationsInDB[0]) } if errMsg != "" { - gplog.Fatal(nil, errMsg) + gplog.Fatal(nil, "%s", errMsg) } } @@ -147,7 +147,7 @@ func ValidateRedirectSchema(connectionPool *dbconn.DBConn, redirectSchema string schemaInDB := dbconn.MustSelectStringSlice(connectionPool, query) if len(schemaInDB) == 0 { - gplog.Fatal(nil, fmt.Sprintf("Schema %s to redirect into does not exist", redirectSchema)) + gplog.Fatal(nil, "Schema %s to redirect into does not exist", redirectSchema) } } diff --git a/restore/wrappers.go b/restore/wrappers.go index 73d7e22c..ceafeb25 100644 --- a/restore/wrappers.go +++ b/restore/wrappers.go @@ -366,10 +366,10 @@ func RestoreSchemas(schemaStatements []toc.StatementWithType, progressBar utils. } else { errMsg := fmt.Sprintf("Error encountered while creating schema %s", schema.Name) if MustGetFlagBool(options.ON_ERROR_CONTINUE) { - gplog.Verbose(fmt.Sprintf("%s: %s", errMsg, err.Error())) + gplog.Verbose("%s: %s", errMsg, err.Error()) numErrors++ } else { - gplog.Fatal(err, errMsg) + gplog.Fatal(err, "%s", errMsg) } } } diff --git a/utils/gpexpand_sensor.go b/utils/gpexpand_sensor.go index 8ca83676..0d0d4655 100644 --- a/utils/gpexpand_sensor.go +++ b/utils/gpexpand_sensor.go @@ -1,7 +1,6 @@ package utils import ( - "fmt" "os" "path/filepath" @@ -54,12 +53,12 @@ func NewGpexpandSensor(myfs vfs.Filesystem, conn *dbconn.DBConn) GpexpandSensor func (sensor GpexpandSensor) IsGpexpandRunning() (bool, error) { err := validateConnection(sensor.postgresConn) if err != nil { - gplog.Error(fmt.Sprintf("Error encountered validating db connection: %v", err)) + gplog.Error("Error encountered validating db connection: %v", err) return false, err } coordinatorDataDir, err := dbconn.SelectString(sensor.postgresConn, CoordinatorDataDirQuery) if err != nil { - gplog.Error(fmt.Sprintf("Error encountered retrieving data directory: %v", err)) + gplog.Error("Error encountered retrieving data directory: %v", err) return false, err } @@ -76,7 +75,7 @@ func (sensor GpexpandSensor) IsGpexpandRunning() (bool, error) { var tableName string tableName, err = dbconn.SelectString(sensor.postgresConn, GpexpandStatusTableExistsQuery) if err != nil { - gplog.Error(fmt.Sprintf("Error encountered retrieving gpexpand status: %v", err)) + gplog.Error("Error encountered retrieving gpexpand status: %v", err) return false, err } if len(tableName) <= 0 { @@ -87,7 +86,7 @@ func (sensor GpexpandSensor) IsGpexpandRunning() (bool, error) { var status string status, err = dbconn.SelectString(sensor.postgresConn, GpexpandTemporaryTableStatusQuery) if err != nil { - gplog.Error(fmt.Sprintf("Error encountered retrieving gpexpand status: %v", err)) + gplog.Error("Error encountered retrieving gpexpand status: %v", err) return false, err } diff --git a/utils/io.go b/utils/io.go index 0985d588..a425b9c2 100644 --- a/utils/io.go +++ b/utils/io.go @@ -92,11 +92,11 @@ func CopyFile(src, dest string) error { var content []byte content, err = ioutil.ReadFile(src) if err != nil { - gplog.Error(fmt.Sprintf("Error: %v, encountered when reading file: %s", err, src)) + gplog.Error("Error: %v, encountered when reading file: %s", err, src) return err } return ioutil.WriteFile(dest, content, info.Mode()) } - gplog.Error(fmt.Sprintf("Error: %v, encountered when trying to stat file: %s", err, src)) + gplog.Error("Error: %v, encountered when trying to stat file: %s", err, src) return err } diff --git a/utils/plugin.go b/utils/plugin.go index 212ee7db..e1e9b1a2 100644 --- a/utils/plugin.go +++ b/utils/plugin.go @@ -245,10 +245,10 @@ func (plugin *PluginConfig) executeHook(c *cluster.Cluster, verboseCommandMsg st plugin.buildHookString(command, fpInfo, scope, coordinatorContentID)) if coordinatorErr != nil { if noFatal { - gplog.Error(coordinatorOutput) + gplog.Error("%s", coordinatorOutput) return } - gplog.Fatal(coordinatorErr, coordinatorOutput) + gplog.Fatal(coordinatorErr, "%s", coordinatorOutput) } // Execute command once on each segment host @@ -368,14 +368,14 @@ func (plugin *PluginConfig) createHostPluginConfig(contentIDForSegmentOnHost int if plugin.UsesEncryption() { pluginName, err := plugin.GetPluginName(c) if err != nil { - _, _ = fmt.Fprintf(operating.System.Stdout, err.Error()) - gplog.Fatal(nil, err.Error()) + _, _ = fmt.Fprintf(operating.System.Stdout, "%s", err.Error()) + gplog.Fatal(nil, "%s", err.Error()) } secret, err := GetSecretKey(pluginName, c.GetDirForContent(-1)) if err != nil { - _, _ = fmt.Fprintf(operating.System.Stdout, err.Error()) - gplog.Fatal(nil, err.Error()) + _, _ = fmt.Fprintf(operating.System.Stdout, "%s", err.Error()) + gplog.Fatal(nil, "%s", err.Error()) } plugin.Options[pluginName] = secret } diff --git a/utils/util.go b/utils/util.go index c2450c53..e773f099 100644 --- a/utils/util.go +++ b/utils/util.go @@ -252,7 +252,7 @@ func ValidateGPDBVersionCompatibility(connectionPool *dbconn.DBConn) { func LogExecutionTime(start time.Time, name string) { elapsed := time.Since(start) - gplog.Debug(fmt.Sprintf("%s took %s", name, elapsed)) + gplog.Debug("%s took %s", name, elapsed) } func Exists(slice []string, val string) bool { From 74b88257036f875b227c631aa310fefc2689d7d6 Mon Sep 17 00:00:00 2001 From: woblerr Date: Tue, 7 Apr 2026 11:49:21 +0300 Subject: [PATCH 15/39] Fix go vet issue. --- gpbackup_s3_plugin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpbackup_s3_plugin.go b/gpbackup_s3_plugin.go index c511dc1b..4060f640 100644 --- a/gpbackup_s3_plugin.go +++ b/gpbackup_s3_plugin.go @@ -113,7 +113,7 @@ func main() { err := app.Run(os.Args) if err != nil { - gplog.Error(err.Error()) + gplog.Error("%s", err.Error()) os.Exit(1) } } From 84710a5adc58fdf1c6c91e163bd7184d24026841 Mon Sep 17 00:00:00 2001 From: woblerr Date: Tue, 7 Apr 2026 12:46:12 +0300 Subject: [PATCH 16/39] Migrate golangci-lint from v1 to v2. Update golangci-lint from 1.16.0 to 2.10.1 in Makefile. Create .golangci.yml with v2 configuration format. Remove obsolete gometalinter.config. Linter mapping from gometalinter.config: - golint -> revive (with shadow check enabled) - vet -> govet - varcheck -> unused - unparam and errcheck remain unchanged TODO: consider enabling additional linters in a follow-up PR: dupl, gochecknoinits, gocritic, gocyclo, gosec, ineffassign, misspell, nakedret, prealloc, staticcheck, unconvert. Current goal is to migrate the existing config as-is. --- .golangci.yml | 42 ++++++++++++++++++++++++++++++++++++++++++ Makefile | 6 +++--- gometalinter.config | 12 ------------ 3 files changed, 45 insertions(+), 15 deletions(-) create mode 100644 .golangci.yml delete mode 100644 gometalinter.config diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 00000000..7625a743 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,42 @@ +version: "2" + +linters: + default: none + enable: + - errcheck + - govet + - revive + - unparam + - unused + settings: + govet: + enable: + - shadow + revive: + confidence: 0.1 + exclusions: + generated: lax + rules: + - linters: + - revive + text: should have comment + - linters: + - revive + text: comment on exported + - linters: + - revive + text: should not use dot imports + - linters: + - revive + text: don't use ALL_CAPS in Go names; use CamelCase + - linters: + - revive + text: and that stutters + - linters: + - revive + text: don't use an underscore in package name + paths: + - vendor + +run: + timeout: 5m diff --git a/Makefile b/Makefile index 9ec10c47..12b5fd44 100644 --- a/Makefile +++ b/Makefile @@ -54,15 +54,15 @@ $(GOSQLITE) : format : $(GOIMPORTS) @goimports -w $(shell find . -type f -name '*.go' -not -path "./vendor/*") -LINTER_VERSION=1.16.0 +LINTER_VERSION=2.10.1 $(GOLANG_LINTER) : mkdir -p $(GOPATH)/bin - curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v${LINTER_VERSION} + curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/main/install.sh | sh -s -- -b $(GOPATH)/bin v${LINTER_VERSION} .PHONY : coverage integration end_to_end lint : $(GOLANG_LINTER) - golangci-lint run --tests=false + golangci-lint run unit : $(GINKGO) TEST_DB_TYPE=CBDB TEST_DB_VERSION=2.999.0 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 diff --git a/gometalinter.config b/gometalinter.config deleted file mode 100644 index 74193c99..00000000 --- a/gometalinter.config +++ /dev/null @@ -1,12 +0,0 @@ -{ - "DisableAll": true, - "Enable": ["golint", "vet", "varcheck", "unparam", "errcheck"], - "Exclude": [ - "should have comment", - "comment on exported", - "should not use dot imports", - "don't use ALL_CAPS in Go names; use CamelCase", - "and that stutters", - "don't use an underscore in package name" - ] -} From 77650b9ce99b5428a16cb89b379fdd45436a7759 Mon Sep 17 00:00:00 2001 From: woblerr Date: Tue, 7 Apr 2026 14:56:54 +0300 Subject: [PATCH 17/39] Fix lint issues introduced by golangci-lint v2 migration. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes: - Fix variable shadowing and declaring variables before use (backup_helper.go, restore_helper.go, history.go, s3plugin.go, data.go). - Fix in testutils/functions.go where := was shadowing the host parameter, preventing PGHOST env var from being applied. - Remove unused test helper functions batchMapToString and contentMapToString from restore/data_test.go. - Add lint exclusion rules in .golangci.yml for test files: govet shadow, errcheck, and unparam — these are pre-existing issues in tests, not regressions from the v2 migration. Pre-existing issues are not fixed here and will be addressed separately. --- .golangci.yml | 10 ++++++++++ helper/backup_helper.go | 11 +++++++---- helper/restore_helper.go | 8 +++++--- history/history.go | 3 ++- plugins/s3plugin/s3plugin.go | 8 +++++--- restore/data.go | 2 +- restore/data_test.go | 22 ---------------------- testutils/functions.go | 2 +- 8 files changed, 31 insertions(+), 35 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 7625a743..60638624 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -35,6 +35,16 @@ linters: - linters: - revive text: don't use an underscore in package name + - linters: + - govet + text: "shadow:" + path: _test\.go + - linters: + - errcheck + path: _test\.go + - linters: + - unparam + path: _test\.go paths: - vendor diff --git a/helper/backup_helper.go b/helper/backup_helper.go index 56ed9a3c..1c77ee48 100644 --- a/helper/backup_helper.go +++ b/helper/backup_helper.go @@ -19,9 +19,12 @@ import ( func doBackupAgent() error { var lastRead uint64 + var numBytes int64 var ( pipeWriter BackupPipeWriterCloser writeCmd *exec.Cmd + reader io.Reader + readHandle io.ReadCloser ) tocfile := &toc.SegmentTOC{} tocfile.DataEntries = make(map[uint]toc.SegmentDataEntry) @@ -48,7 +51,7 @@ func doBackupAgent() error { if i < len(oidList)-*copyQueue { nextPipeToCreate := fmt.Sprintf("%s_%d", *pipeFile, oidList[i+*copyQueue]) logVerbose(fmt.Sprintf("Oid %d: Creating pipe %s\n", oidList[i+*copyQueue], nextPipeToCreate)) - err := createPipe(nextPipeToCreate) + err = createPipe(nextPipeToCreate) if err != nil { logError(fmt.Sprintf("Oid %d: Failed to create pipe %s\n", oidList[i+*copyQueue], nextPipeToCreate)) return err @@ -56,7 +59,7 @@ func doBackupAgent() error { } logInfo(fmt.Sprintf("Oid %d: Opening pipe %s", oid, currentPipe)) - reader, readHandle, err := getBackupPipeReader(currentPipe) + reader, readHandle, err = getBackupPipeReader(currentPipe) if err != nil { logError(fmt.Sprintf("Oid %d: Error encountered getting backup pipe reader: %v", oid, err)) return err @@ -70,7 +73,7 @@ func doBackupAgent() error { } logInfo(fmt.Sprintf("Oid %d: Backing up table with pipe %s", oid, currentPipe)) - numBytes, err := io.Copy(pipeWriter, reader) + numBytes, err = io.Copy(pipeWriter, reader) if err != nil { logError(fmt.Sprintf("Oid %d: Error encountered copying bytes from pipeWriter to reader: %v", oid, err)) return errors.Wrap(err, strings.Trim(errBuf.String(), "\x00")) @@ -96,7 +99,7 @@ func doBackupAgent() error { * written to verify the agent completed. */ logVerbose("Uploading remaining data to plugin destination") - err := writeCmd.Wait() + err = writeCmd.Wait() if err != nil { logError(fmt.Sprintf("Error encountered writing either TOC file or error file: %v", err)) return errors.Wrap(err, strings.Trim(errBuf.String(), "\x00")) diff --git a/helper/restore_helper.go b/helper/restore_helper.go index 84c0086d..50b32820 100644 --- a/helper/restore_helper.go +++ b/helper/restore_helper.go @@ -193,7 +193,7 @@ func doRestoreAgent() error { nextBatchNum := nextOidWithBatch.batch nextPipeToCreate := fmt.Sprintf("%s_%d_%d", *pipeFile, nextOid, nextBatchNum) logVerbose(fmt.Sprintf("Oid %d, Batch %d: Creating pipe %s\n", nextOid, nextBatchNum, nextPipeToCreate)) - err := createPipe(nextPipeToCreate) + err = createPipe(nextPipeToCreate) if err != nil { logError(fmt.Sprintf("Oid %d, Batch %d: Failed to create pipe %s\n", nextOid, nextBatchNum, nextPipeToCreate)) // In the case this error is hit it means we have lost the @@ -366,6 +366,8 @@ func replaceContentInFilename(filename string, content int) string { func getRestoreDataReader(fileToRead string, objToc *toc.SegmentTOC, oidList []int) (*RestoreReader, error) { var readHandle io.Reader var seekHandle io.ReadSeeker + var gzipReader *gzip.Reader + var zstdReader *zstd.Decoder var isSubset bool var err error = nil restoreReader := new(RestoreReader) @@ -402,14 +404,14 @@ func getRestoreDataReader(fileToRead string, objToc *toc.SegmentTOC, oidList []i if restoreReader.readerType == SEEKABLE { restoreReader.seekReader = seekHandle } else if strings.HasSuffix(fileToRead, ".gz") { - gzipReader, err := gzip.NewReader(readHandle) + gzipReader, err = gzip.NewReader(readHandle) if err != nil { // error logging handled by calling functions return nil, err } restoreReader.bufReader = bufio.NewReader(gzipReader) } else if strings.HasSuffix(fileToRead, ".zst") { - zstdReader, err := zstd.NewReader(readHandle) + zstdReader, err = zstd.NewReader(readHandle) if err != nil { // error logging handled by calling functions return nil, err diff --git a/history/history.go b/history/history.go index ff07e106..05e1406e 100644 --- a/history/history.go +++ b/history/history.go @@ -414,7 +414,8 @@ func GetBackupConfig(timestamp string, historyDB *sql.DB) (*BackupConfig, error) restorePlan.Timestamp = restorePlanTimestamp restorePlanTablesQuery := fmt.Sprintf("SELECT table_fqn FROM restore_plan_tables WHERE timestamp = '%s' and restore_plan_timestamp = '%s'", timestamp, restorePlanTimestamp) - restorePlanTableRows, err := historyDB.Query(restorePlanTablesQuery) + var restorePlanTableRows *sql.Rows + restorePlanTableRows, err = historyDB.Query(restorePlanTablesQuery) if err != nil { return nil, err } diff --git a/plugins/s3plugin/s3plugin.go b/plugins/s3plugin/s3plugin.go index 9a2a4e83..501bad08 100644 --- a/plugins/s3plugin/s3plugin.go +++ b/plugins/s3plugin/s3plugin.go @@ -111,6 +111,7 @@ func readAndValidatePluginConfig(configFile string) (*PluginConfig, error) { func InitializeAndValidateConfig(config *PluginConfig) error { var err error var errTxt string + var chunkSize bytesize.ByteSize opt := &config.Options // Initialize defaults @@ -155,7 +156,7 @@ func InitializeAndValidateConfig(config *PluginConfig) error { errTxt += fmt.Sprintf("Invalid value for remove_duplicate_bucket. Valid choices are true or false.\n") } if opt.BackupMultipartChunksize != "" { - chunkSize, err := bytesize.Parse(opt.BackupMultipartChunksize) + chunkSize, err = bytesize.Parse(opt.BackupMultipartChunksize) if err != nil { errTxt += fmt.Sprintf("Invalid backup_multipart_chunksize. Err: %s\n", err) } @@ -170,7 +171,7 @@ func InitializeAndValidateConfig(config *PluginConfig) error { } } if opt.RestoreMultipartChunksize != "" { - chunkSize, err := bytesize.Parse(opt.RestoreMultipartChunksize) + chunkSize, err = bytesize.Parse(opt.RestoreMultipartChunksize) if err != nil { errTxt += fmt.Sprintf("Invalid restore_multipart_chunksize. Err: %s\n", err) } @@ -363,6 +364,7 @@ func DeleteBackup(c *cli.Context) error { func ListDirectory(c *cli.Context) error { var err error + var totalBytes int64 config, sess, err := readConfigAndStartSession(c) if err != nil { return err @@ -392,7 +394,7 @@ func ListDirectory(c *cli.Context) error { u.PartSize = config.Options.DownloadChunkSize }) - totalBytes, err := getFileSize(downloader.S3, bucket, *key.Key) + totalBytes, err = getFileSize(downloader.S3, bucket, *key.Key) if err != nil { return err } diff --git a/restore/data.go b/restore/data.go index 11d2deea..4175ad96 100644 --- a/restore/data.go +++ b/restore/data.go @@ -292,7 +292,7 @@ func restoreDataFromTimestamp(fpInfo filepath.FilePathInfo, dataEntries []toc.Co var err error if MustGetFlagBool(options.INCREMENTAL) || MustGetFlagBool(options.TRUNCATE_TABLE) { gplog.Verbose("Truncating table %s prior to restoring data", tableName) - _, err := connectionPool.Exec(`TRUNCATE `+tableName, whichConn) + _, err = connectionPool.Exec(`TRUNCATE `+tableName, whichConn) if err != nil { gplog.Error("%s", err.Error()) } diff --git a/restore/data_test.go b/restore/data_test.go index 940a9927..01a03b05 100644 --- a/restore/data_test.go +++ b/restore/data_test.go @@ -1,10 +1,7 @@ package restore_test import ( - "fmt" "regexp" - "sort" - "strings" "github.com/DATA-DOG/go-sqlmock" "github.com/apache/cloudberry-backup/backup" @@ -134,22 +131,3 @@ var _ = Describe("restore/data tests", func() { }) }) }) - -func batchMapToString(m map[int]map[int]int) string { - outer := make([]string, len(m)) - for num, batch := range m { - outer[num] = fmt.Sprintf("%d: %s", num, contentMapToString(batch)) - } - return strings.Join(outer, "; ") -} - -func contentMapToString(m map[int]int) string { - inner := make([]string, len(m)) - index := 0 - for orig, dest := range m { - inner[index] = fmt.Sprintf("%d:%d", orig, dest) - index++ - } - sort.Strings(inner) - return fmt.Sprintf("{%s}", strings.Join(inner, ", ")) -} diff --git a/testutils/functions.go b/testutils/functions.go index 30e1b079..19d786aa 100644 --- a/testutils/functions.go +++ b/testutils/functions.go @@ -93,7 +93,7 @@ func SetupTestDBConnSegment(dbname string, port int, host string, gpVersion dbco username = currentUser.Username } if host == "" { - host := operating.System.Getenv("PGHOST") + host = operating.System.Getenv("PGHOST") if host == "" { host, _ = operating.System.Hostname() } From a73157c35373a48052489bb6508d153ece206be8 Mon Sep 17 00:00:00 2001 From: woblerr Date: Tue, 7 Apr 2026 19:38:59 +0300 Subject: [PATCH 18/39] Bump setup-go action to v5 and go to 1.24. --- .github/workflows/build_and_unit_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_unit_test.yml b/.github/workflows/build_and_unit_test.yml index 9727fc74..d24851a9 100644 --- a/.github/workflows/build_and_unit_test.yml +++ b/.github/workflows/build_and_unit_test.yml @@ -19,9 +19,9 @@ jobs: path: go/src/github.com/apache/cloudberry-backup - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: - go-version: 1.21 + go-version: "1.24" - name: Set Environment run: | From adba8da6ad7f99ab4d7469699b48b8e07be5d6a5 Mon Sep 17 00:00:00 2001 From: woblerr Date: Tue, 7 Apr 2026 19:45:14 +0300 Subject: [PATCH 19/39] Remove legacy `// +build` directives. The legacy `// +build` form is no longer needed. --- gpbackman.go | 1 - gpbackup.go | 1 - gpbackup_helper.go | 1 - gpbackup_s3_plugin.go | 1 - gprestore.go | 1 - tools.go | 1 - 6 files changed, 6 deletions(-) diff --git a/gpbackman.go b/gpbackman.go index 4d9a65bf..15be1158 100644 --- a/gpbackman.go +++ b/gpbackman.go @@ -1,5 +1,4 @@ //go:build gpbackman -// +build gpbackman /* Licensed to the Apache Software Foundation (ASF) under one diff --git a/gpbackup.go b/gpbackup.go index 9b75b920..e86a90b5 100644 --- a/gpbackup.go +++ b/gpbackup.go @@ -1,5 +1,4 @@ //go:build gpbackup -// +build gpbackup package main diff --git a/gpbackup_helper.go b/gpbackup_helper.go index 60e2933d..fc6b4ed9 100644 --- a/gpbackup_helper.go +++ b/gpbackup_helper.go @@ -1,5 +1,4 @@ //go:build gpbackup_helper -// +build gpbackup_helper package main diff --git a/gpbackup_s3_plugin.go b/gpbackup_s3_plugin.go index 4060f640..38a34af5 100644 --- a/gpbackup_s3_plugin.go +++ b/gpbackup_s3_plugin.go @@ -1,5 +1,4 @@ //go:build gpbackup_s3_plugin -// +build gpbackup_s3_plugin package main diff --git a/gprestore.go b/gprestore.go index ac97c1dd..df3e3db6 100644 --- a/gprestore.go +++ b/gprestore.go @@ -1,5 +1,4 @@ //go:build gprestore -// +build gprestore package main diff --git a/tools.go b/tools.go index 08dff4f6..b740aacd 100644 --- a/tools.go +++ b/tools.go @@ -1,5 +1,4 @@ //go:build tools -// +build tools package tools From 2212e80fad6d7f7602325ef230d67d5401d9d203 Mon Sep 17 00:00:00 2001 From: woblerr Date: Tue, 7 Apr 2026 22:51:30 +0300 Subject: [PATCH 20/39] Replace deprecated io/ioutil. The io/ioutil package has been deprecated since Go 1.16. All usages are replaced with their recommended alternatives from os and io packages. Changes: - ioutil.ReadFile -> os.ReadFile - ioutil.WriteFile -> os.WriteFile - ioutil.TempFile -> os.CreateTemp - ioutil.TempDir -> os.MkdirTemp - ioutil.ReadAll -> io.ReadAll - ioutil.ReadDir -> os.ReadDir The os.ReadDir returns []os.DirEntry instead of []os.FileInfo, but all call sites only use .Name(), which is available on both interfaces. --- end_to_end/end_to_end_suite_test.go | 15 +++++++-------- helper/restore_helper.go | 3 +-- history/history.go | 3 +-- integration/helper_test.go | 30 ++++++++++++++--------------- integration/utils_test.go | 3 +-- options/options_test.go | 9 ++++----- plugins/s3plugin/s3plugin.go | 3 +-- report/report_test.go | 3 +-- restore/wrappers_test.go | 9 ++++----- toc/toc.go | 6 +++--- utils/io.go | 5 ++--- utils/io_test.go | 5 ++--- utils/plugin_test.go | 15 +++++++-------- 13 files changed, 49 insertions(+), 60 deletions(-) diff --git a/end_to_end/end_to_end_suite_test.go b/end_to_end/end_to_end_suite_test.go index 6a71a0d6..a9207097 100644 --- a/end_to_end/end_to_end_suite_test.go +++ b/end_to_end/end_to_end_suite_test.go @@ -6,7 +6,6 @@ import ( "flag" "fmt" "io/fs" - "io/ioutil" "os" "os/exec" path "path/filepath" @@ -362,7 +361,7 @@ func getMetdataFileContents(backupDir string, timestamp string, fileSuffix strin } file, err := path.Glob(path.Join(backupDir, filePath)) Expect(err).ToNot(HaveOccurred()) - fileContentBytes, err := ioutil.ReadFile(file[0]) + fileContentBytes, err := os.ReadFile(file[0]) Expect(err).ToNot(HaveOccurred()) return fileContentBytes @@ -695,7 +694,7 @@ func end_to_end_setup() { _ = os.Remove(historyFilePath) // Assign a unique directory for each test - backupDir, _ = ioutil.TempDir(customBackupDir, "temp") + backupDir, _ = os.MkdirTemp(customBackupDir, "temp") } func end_to_end_teardown() { @@ -901,7 +900,7 @@ var _ = Describe("backup and restore end to end tests", func() { Expect(files).To(HaveLen(2)) Expect(files[0]).To(HaveSuffix("_data")) - contents, err := ioutil.ReadFile(files[0]) + contents, err := os.ReadFile(files[0]) Expect(err).ToNot(HaveOccurred()) tables := strings.Split(string(contents), "\n") Expect(tables).To(Equal(expectedErrorTablesData)) @@ -909,7 +908,7 @@ var _ = Describe("backup and restore end to end tests", func() { Expect(files).To(HaveLen(2)) Expect(files[1]).To(HaveSuffix("_metadata")) - contents, err = ioutil.ReadFile(files[1]) + contents, err = os.ReadFile(files[1]) Expect(err).ToNot(HaveOccurred()) tables = strings.Split(string(contents), "\n") sort.Strings(tables) @@ -932,7 +931,7 @@ var _ = Describe("backup and restore end to end tests", func() { "*-1/backups/*", "20190809230424", "*error_tables*")) Expect(files).To(HaveLen(1)) Expect(files[0]).To(HaveSuffix("_metadata")) - contents, err = ioutil.ReadFile(files[0]) + contents, err = os.ReadFile(files[0]) Expect(err).ToNot(HaveOccurred()) tables = strings.Split(string(contents), "\n") sort.Strings(tables) @@ -967,7 +966,7 @@ var _ = Describe("backup and restore end to end tests", func() { Expect(files).To(HaveLen(1)) Expect(files[0]).To(HaveSuffix("_metadata")) - contents, err := ioutil.ReadFile(files[0]) + contents, err := os.ReadFile(files[0]) Expect(err).ToNot(HaveOccurred()) tables := strings.Split(string(contents), "\n") Expect(tables).To(Equal(expectedErrorTablesMetadata)) @@ -1598,7 +1597,7 @@ var _ = Describe("backup and restore end to end tests", func() { Expect(err).ToNot(HaveOccurred()) Expect(configFile).To(HaveLen(1)) - contents, err := ioutil.ReadFile(configFile[0]) + contents, err := os.ReadFile(configFile[0]) Expect(err).ToNot(HaveOccurred()) Expect(string(contents)).To(ContainSubstring("compressed: false")) diff --git a/helper/restore_helper.go b/helper/restore_helper.go index 50b32820..e0c67857 100644 --- a/helper/restore_helper.go +++ b/helper/restore_helper.go @@ -5,7 +5,6 @@ import ( "compress/gzip" "fmt" "io" - "io/ioutil" "os" "os/exec" "path" @@ -457,7 +456,7 @@ func startRestorePluginCommand(fileToRead string, objToc *toc.SegmentTOC, oidLis } cmdStr := "" if objToc != nil && pluginConfig.CanRestoreSubset() && *isFiltered && !strings.HasSuffix(fileToRead, ".gz") && !strings.HasSuffix(fileToRead, ".zst") { - offsetsFile, _ := ioutil.TempFile("/tmp", "gprestore_offsets_") + offsetsFile, _ := os.CreateTemp("/tmp", "gprestore_offsets_") defer func() { offsetsFile.Close() }() diff --git a/history/history.go b/history/history.go index 05e1406e..1ee0c7ec 100644 --- a/history/history.go +++ b/history/history.go @@ -4,7 +4,6 @@ import ( "database/sql" "errors" "fmt" - "io/ioutil" "os" "github.com/apache/cloudberry-backup/utils" @@ -63,7 +62,7 @@ func (backup *BackupConfig) Failed() bool { func ReadConfigFile(filename string) *BackupConfig { config := &BackupConfig{} - contents, err := ioutil.ReadFile(filename) + contents, err := os.ReadFile(filename) gplog.FatalOnError(err) err = yaml.Unmarshal(contents, config) gplog.FatalOnError(err) diff --git a/integration/helper_test.go b/integration/helper_test.go index 31a79299..04ebf4ed 100644 --- a/integration/helper_test.go +++ b/integration/helper_test.go @@ -4,7 +4,7 @@ import ( "bytes" "compress/gzip" "fmt" - "io/ioutil" + "io" "math" "os" "os/exec" @@ -195,7 +195,7 @@ options: setupRestoreFiles("", false) helperCmd := gpbackupHelperRestore(gpbackupHelperPath, "--data-file", dataFileFullPath) for _, i := range []int{1, 3} { - contents, _ := ioutil.ReadFile(fmt.Sprintf("%s_%d_0", pipeFile, i)) + contents, _ := os.ReadFile(fmt.Sprintf("%s_%d_0", pipeFile, i)) Expect(string(contents)).To(Equal("here is some data\n")) } err := helperCmd.Wait() @@ -207,7 +207,7 @@ options: setupRestoreFiles("gzip", false) helperCmd := gpbackupHelperRestore(gpbackupHelperPath, "--data-file", dataFileFullPath+".gz") for _, i := range []int{1, 3} { - contents, _ := ioutil.ReadFile(fmt.Sprintf("%s_%d_0", pipeFile, i)) + contents, _ := os.ReadFile(fmt.Sprintf("%s_%d_0", pipeFile, i)) Expect(string(contents)).To(Equal("here is some data\n")) } err := helperCmd.Wait() @@ -219,7 +219,7 @@ options: setupRestoreFiles("zstd", false) helperCmd := gpbackupHelperRestore(gpbackupHelperPath, "--data-file", dataFileFullPath+".zst") for _, i := range []int{1, 3} { - contents, _ := ioutil.ReadFile(fmt.Sprintf("%s_%d_0", pipeFile, i)) + contents, _ := os.ReadFile(fmt.Sprintf("%s_%d_0", pipeFile, i)) Expect(string(contents)).To(Equal("here is some data\n")) } err := helperCmd.Wait() @@ -231,7 +231,7 @@ options: setupRestoreFiles("", true) helperCmd := gpbackupHelperRestore(gpbackupHelperPath, "--data-file", dataFileFullPath, "--plugin-config", examplePluginTestConfig) for _, i := range []int{1, 3} { - contents, _ := ioutil.ReadFile(fmt.Sprintf("%s_%d_0", pipeFile, i)) + contents, _ := os.ReadFile(fmt.Sprintf("%s_%d_0", pipeFile, i)) Expect(string(contents)).To(Equal("here is some data\n")) } err := helperCmd.Wait() @@ -243,7 +243,7 @@ options: setupRestoreFiles("gzip", true) helperCmd := gpbackupHelperRestore(gpbackupHelperPath, "--data-file", dataFileFullPath+".gz", "--plugin-config", examplePluginTestConfig) for _, i := range []int{1, 3} { - contents, _ := ioutil.ReadFile(fmt.Sprintf("%s_%d_0", pipeFile, i)) + contents, _ := os.ReadFile(fmt.Sprintf("%s_%d_0", pipeFile, i)) Expect(string(contents)).To(Equal("here is some data\n")) } err := helperCmd.Wait() @@ -255,7 +255,7 @@ options: setupRestoreFiles("zstd", true) helperCmd := gpbackupHelperRestore(gpbackupHelperPath, "--data-file", dataFileFullPath+".zst", "--plugin-config", examplePluginTestConfig) for _, i := range []int{1, 3} { - contents, _ := ioutil.ReadFile(fmt.Sprintf("%s_%d_0", pipeFile, i)) + contents, _ := os.ReadFile(fmt.Sprintf("%s_%d_0", pipeFile, i)) Expect(string(contents)).To(Equal("here is some data\n")) } err := helperCmd.Wait() @@ -329,7 +329,7 @@ options: errClose := file.Close() Expect(errClose).ToNot(HaveOccurred()) } else { - contents, err := ioutil.ReadFile(currentPipe) + contents, err := os.ReadFile(currentPipe) Expect(err).ToNot(HaveOccurred()) Expect(string(contents)).To(Equal("here is some data\n")) } @@ -404,11 +404,11 @@ func assertBackupArtifacts(withPlugin bool) { if withPlugin { dataFile = examplePluginTestDataFile } - contents, err = ioutil.ReadFile(dataFile) + contents, err = os.ReadFile(dataFile) Expect(err).ToNot(HaveOccurred()) Expect(string(contents)).To(Equal(expectedData)) - contents, err = ioutil.ReadFile(tocFile) + contents, err = os.ReadFile(tocFile) Expect(err).ToNot(HaveOccurred()) Expect(string(contents)).To(Equal(expectedTOC)) assertNoErrors() @@ -424,9 +424,9 @@ func assertBackupArtifactsWithCompression(compressionType string, withPlugin boo } if compressionType == "gzip" { - contents, err = ioutil.ReadFile(dataFile + ".gz") + contents, err = os.ReadFile(dataFile + ".gz") } else if compressionType == "zstd" { - contents, err = ioutil.ReadFile(dataFile + ".zst") + contents, err = os.ReadFile(dataFile + ".zst") } else { Fail("unknown compression type " + compressionType) } @@ -434,16 +434,16 @@ func assertBackupArtifactsWithCompression(compressionType string, withPlugin boo if compressionType == "gzip" { r, _ := gzip.NewReader(bytes.NewReader(contents)) - contents, _ = ioutil.ReadAll(r) + contents, _ = io.ReadAll(r) } else if compressionType == "zstd" { r, _ := zstd.NewReader(bytes.NewReader(contents)) - contents, _ = ioutil.ReadAll(r) + contents, _ = io.ReadAll(r) } else { Fail("unknown compression type " + compressionType) } Expect(string(contents)).To(Equal(expectedData)) - contents, err = ioutil.ReadFile(tocFile) + contents, err = os.ReadFile(tocFile) Expect(err).ToNot(HaveOccurred()) Expect(string(contents)).To(Equal(expectedTOC)) diff --git a/integration/utils_test.go b/integration/utils_test.go index efeabab0..9228e5a7 100644 --- a/integration/utils_test.go +++ b/integration/utils_test.go @@ -2,7 +2,6 @@ package integration import ( "fmt" - "io/ioutil" "os" "path/filepath" "time" @@ -21,7 +20,7 @@ import ( var _ = Describe("utils integration", func() { It("TerminateHangingCopySessions stops hanging COPY sessions", func() { - tempDir, err := ioutil.TempDir("", "temp") + tempDir, err := os.MkdirTemp("", "temp") Expect(err).To(Not(HaveOccurred())) defer os.Remove(tempDir) testPipe := filepath.Join(tempDir, "test_pipe") diff --git a/options/options_test.go b/options/options_test.go index 396c572b..fef85568 100644 --- a/options/options_test.go +++ b/options/options_test.go @@ -1,7 +1,6 @@ package options_test import ( - "io/ioutil" "os" "github.com/DATA-DOG/go-sqlmock" @@ -70,7 +69,7 @@ var _ = Describe("options", func() { Expect(includedTables[1]).To(Equal("bar.baz")) }) It("returns the text-file tables when specified", func() { - file, err := ioutil.TempFile("/tmp", "gpbackup_test_options*.txt") + file, err := os.CreateTemp("/tmp", "gpbackup_test_options*.txt") Expect(err).To(Not(HaveOccurred())) defer func() { _ = os.Remove(file.Name()) @@ -93,7 +92,7 @@ var _ = Describe("options", func() { Expect(includedTables[1]).To(Equal("myschema.mytable2")) }) It("sets the INCLUDE_RELATIONS flag from file", func() { - file, err := ioutil.TempFile("/tmp", "gpbackup_test_options*.txt") + file, err := os.CreateTemp("/tmp", "gpbackup_test_options*.txt") Expect(err).To(Not(HaveOccurred())) defer func() { _ = os.Remove(file.Name()) @@ -117,7 +116,7 @@ var _ = Describe("options", func() { Expect(includedTables[1]).To(Equal("myschema.mytable2")) }) It("skips empty lines in files provided for filtering tables", func() { - file, err := ioutil.TempFile("/tmp", "gpbackup_test_options*.txt") + file, err := os.CreateTemp("/tmp", "gpbackup_test_options*.txt") Expect(err).To(Not(HaveOccurred())) defer func() { _ = os.Remove(file.Name()) @@ -150,7 +149,7 @@ var _ = Describe("options", func() { Expect(excludedTables[1]).To(Equal("myschema.mytable2")) }) It("skips empty lines in files provided for filtering schemas", func() { - file, err := ioutil.TempFile("/tmp", "gpbackup_test_options*.txt") + file, err := os.CreateTemp("/tmp", "gpbackup_test_options*.txt") Expect(err).To(Not(HaveOccurred())) defer func() { _ = os.Remove(file.Name()) diff --git a/plugins/s3plugin/s3plugin.go b/plugins/s3plugin/s3plugin.go index 501bad08..f1b43d56 100644 --- a/plugins/s3plugin/s3plugin.go +++ b/plugins/s3plugin/s3plugin.go @@ -3,7 +3,6 @@ package s3plugin import ( "errors" "fmt" - "io/ioutil" "net/http" "net/url" "os" @@ -95,7 +94,7 @@ func GetAPIVersion(c *cli.Context) { func readAndValidatePluginConfig(configFile string) (*PluginConfig, error) { config := &PluginConfig{} - contents, err := ioutil.ReadFile(configFile) + contents, err := os.ReadFile(configFile) if err != nil { return nil, err } diff --git a/report/report_test.go b/report/report_test.go index 0d2f6409..e53cc2de 100644 --- a/report/report_test.go +++ b/report/report_test.go @@ -3,7 +3,6 @@ package report_test import ( "fmt" "io" - "io/ioutil" "os" "strings" "testing" @@ -480,7 +479,7 @@ Timestamp Key: 20170101010101`) testCluster = testutils.SetDefaultSegmentConfiguration() testFPInfo = filepath.NewFilePathInfo(testCluster, "", "20170101010101", "gpseg", false) operating.System.OpenFileRead = func(name string, flag int, perm os.FileMode) (operating.ReadCloserAt, error) { return r, nil } - operating.System.ReadFile = func(filename string) ([]byte, error) { return ioutil.ReadAll(r) } + operating.System.ReadFile = func(filename string) ([]byte, error) { return io.ReadAll(r) } operating.System.Hostname = func() (string, error) { return "localhost", nil } operating.System.Getenv = func(key string) string { if key == "HOME" { diff --git a/restore/wrappers_test.go b/restore/wrappers_test.go index 88eea0cd..86635f5e 100644 --- a/restore/wrappers_test.go +++ b/restore/wrappers_test.go @@ -2,7 +2,6 @@ package restore_test import ( "fmt" - "io/ioutil" "os" "path/filepath" @@ -236,9 +235,9 @@ timestamp: "20180415154238" withstatistics: false `, dbVersion) - tempDir, _ = ioutil.TempDir("", "temp") + tempDir, _ = os.MkdirTemp("", "temp") - err := ioutil.WriteFile(testConfigPath, []byte(sampleConfigContents), 0777) + err := os.WriteFile(testConfigPath, []byte(sampleConfigContents), 0777) Expect(err).ToNot(HaveOccurred()) err = cmdFlags.Set(options.PLUGIN_CONFIG, testConfigPath) Expect(err).ToNot(HaveOccurred()) @@ -286,7 +285,7 @@ withstatistics: false configDir := filepath.Join(mdd, "backups/20170101/20170101010101/") _ = os.MkdirAll(configDir, 0777) configPath := filepath.Join(configDir, "gpbackup_20170101010101_config.yaml") - err = ioutil.WriteFile(configPath, []byte(sampleBackupConfig), 0777) + err = os.WriteFile(configPath, []byte(sampleBackupConfig), 0777) Expect(err).ToNot(HaveOccurred()) restore.SetVersion("1.11.0+dev.28.g10571fd") @@ -298,7 +297,7 @@ withstatistics: false _ = os.Remove(testConfigPath) confDir := filepath.Dir(testConfigPath) confFileName := filepath.Base(testConfigPath) - files, _ := ioutil.ReadDir(confDir) + files, _ := os.ReadDir(confDir) for _, f := range files { match, _ := filepath.Match("*"+confFileName+"*", f.Name()) if match { diff --git a/toc/toc.go b/toc/toc.go index 42d727ca..32cb913a 100644 --- a/toc/toc.go +++ b/toc/toc.go @@ -3,7 +3,7 @@ package toc import ( "fmt" "io" - "io/ioutil" + "os" "regexp" "strings" @@ -128,7 +128,7 @@ const ( func NewTOC(filename string) *TOC { toc := &TOC{} - contents, err := ioutil.ReadFile(filename) + contents, err := os.ReadFile(filename) gplog.FatalOnError(err) err = yaml.Unmarshal(contents, toc) gplog.FatalOnError(err) @@ -137,7 +137,7 @@ func NewTOC(filename string) *TOC { func NewSegmentTOC(filename string) *SegmentTOC { toc := &SegmentTOC{} - contents, err := ioutil.ReadFile(filename) + contents, err := os.ReadFile(filename) gplog.FatalOnError(err) err = yaml.Unmarshal(contents, toc) gplog.FatalOnError(err) diff --git a/utils/io.go b/utils/io.go index a425b9c2..b2cdce1f 100644 --- a/utils/io.go +++ b/utils/io.go @@ -8,7 +8,6 @@ package utils import ( "fmt" "io" - "io/ioutil" "os" "github.com/apache/cloudberry-go-libs/gplog" @@ -90,12 +89,12 @@ func CopyFile(src, dest string) error { info, err := os.Stat(src) if err == nil { var content []byte - content, err = ioutil.ReadFile(src) + content, err = os.ReadFile(src) if err != nil { gplog.Error("Error: %v, encountered when reading file: %s", err, src) return err } - return ioutil.WriteFile(dest, content, info.Mode()) + return os.WriteFile(dest, content, info.Mode()) } gplog.Error("Error: %v, encountered when trying to stat file: %s", err, src) return err diff --git a/utils/io_test.go b/utils/io_test.go index 17f9ce61..3756dc6c 100644 --- a/utils/io_test.go +++ b/utils/io_test.go @@ -1,7 +1,6 @@ package utils_test import ( - "io/ioutil" "os" "github.com/apache/cloudberry-backup/utils" @@ -62,12 +61,12 @@ var _ = Describe("utils/io tests", func() { _ = os.Remove(destFilePath) }) It("copies source file to dest file", func() { - _ = ioutil.WriteFile(sourceFilePath, []byte{1, 2, 3, 4}, 0777) + _ = os.WriteFile(sourceFilePath, []byte{1, 2, 3, 4}, 0777) err := utils.CopyFile(sourceFilePath, destFilePath) Expect(err).ToNot(HaveOccurred()) - contents, _ := ioutil.ReadFile(destFilePath) + contents, _ := os.ReadFile(destFilePath) Expect(contents).To(Equal([]byte{1, 2, 3, 4})) }) It("returns an err when cannot read source file", func() { diff --git a/utils/plugin_test.go b/utils/plugin_test.go index b292cde6..2fc22cf9 100644 --- a/utils/plugin_test.go +++ b/utils/plugin_test.go @@ -2,7 +2,6 @@ package utils_test import ( "fmt" - "io/ioutil" "os" "path/filepath" "regexp" @@ -29,7 +28,7 @@ var _ = Describe("utils/plugin tests", func() { BeforeEach(func() { operating.InitializeSystemFunctions() - tempDir, _ = ioutil.TempDir("", "temp") + tempDir, _ = os.MkdirTemp("", "temp") operating.System.Stdout = stdout subject = utils.PluginConfig{ ExecutablePath: "/a/b/myPlugin", @@ -68,7 +67,7 @@ var _ = Describe("utils/plugin tests", func() { _ = os.Remove(subject.ConfigPath) confDir := filepath.Dir(subject.ConfigPath) confFileName := filepath.Base(subject.ConfigPath) - files, _ := ioutil.ReadDir(confDir) + files, _ := os.ReadDir(confDir) for _, f := range files { match, _ := filepath.Match(confFileName+"*", f.Name()) if match { @@ -110,7 +109,7 @@ options: field2: hello field3: 567 ` - err := ioutil.WriteFile(testConfigPath, []byte(testConfigContents), 0777) + err := os.WriteFile(testConfigPath, []byte(testConfigContents), 0777) Expect(err).To(Not(HaveOccurred())) subject.SetBackupPluginVersion("myTimestamp", "my.test.version") subject.CopyPluginConfigToAllHosts(testCluster) @@ -157,7 +156,7 @@ options: field2: hello field3: 567 ` - err := ioutil.WriteFile(testConfigPath, []byte(testConfigContents), 0777) + err := os.WriteFile(testConfigPath, []byte(testConfigContents), 0777) subject.Options["password_encryption"] = "on" mdd := testCluster.GetDirForContent(-1) _ = os.MkdirAll(mdd, 0777) @@ -286,7 +285,7 @@ options: mdd := testCluster.GetDirForContent(-1) _ = os.MkdirAll(mdd, 0777) secretFilePath := filepath.Join(mdd, utils.SecretKeyFile) - err := ioutil.WriteFile(secretFilePath, []byte(`gpbackup_fake_plugin: 0123456789`), 0777) + err := os.WriteFile(secretFilePath, []byte(`gpbackup_fake_plugin: 0123456789`), 0777) Expect(err).To(Not(HaveOccurred())) key, err := utils.GetSecretKey("gpbackup_fake_plugin", mdd) @@ -307,7 +306,7 @@ options: mdd := testCluster.GetDirForContent(-1) _ = os.MkdirAll(mdd, 0777) secretFilePath := filepath.Join(mdd, utils.SecretKeyFile) - err := ioutil.WriteFile(secretFilePath, []byte(""), 0777) + err := os.WriteFile(secretFilePath, []byte(""), 0777) Expect(err).To(Not(HaveOccurred())) pluginName := "gpbackup_fake_plugin" @@ -320,7 +319,7 @@ options: mdd := testCluster.GetDirForContent(-1) _ = os.MkdirAll(mdd, 0777) secretFilePath := filepath.Join(mdd, utils.SecretKeyFile) - err := ioutil.WriteFile(secretFilePath, []byte("improperlyFormattedYaml"), 0777) + err := os.WriteFile(secretFilePath, []byte("improperlyFormattedYaml"), 0777) Expect(err).To(Not(HaveOccurred())) pluginName := "gpbackup_fake_plugin" From ac8a9c6367b8f6f0203a34210e0d5bab3ac5be7c Mon Sep 17 00:00:00 2001 From: woblerr Date: Wed, 8 Apr 2026 10:24:45 +0300 Subject: [PATCH 21/39] Add ASF license header to golangci.yml. --- .golangci.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index 60638624..a00057b8 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + version: "2" linters: From 749a32a5da67c7d4e051d4cd638503e62464d5d5 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Thu, 9 Apr 2026 11:08:53 +0800 Subject: [PATCH 22/39] Bump VERSION to 2.2.0 for main --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7ec1d6db..ccbccc3d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.0 +2.2.0 From 75f753a0bbe7d36f7b49890ca302d89fe6226bb4 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Thu, 9 Apr 2026 22:14:24 +0800 Subject: [PATCH 23/39] CI: add smoke test for command validation (#84) Add smoke tests to verify all cloudberry-backup commands (gpbackup, gprestore, gpbackup_helper, gpbackup_s3_plugin, gpbackman) can be built and executed with --version and --help flags. - build_and_unit_test.yml: Test from GOPATH/bin after build - cloudberry-backup-ci.yml: Test from GPHOME/bin after install This provides fast feedback on basic functionality before running more expensive integration tests. --- .github/workflows/build_and_unit_test.yml | 22 ++++++++++++++++++++++ .github/workflows/cloudberry-backup-ci.yml | 22 +++++++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_unit_test.yml b/.github/workflows/build_and_unit_test.yml index d24851a9..cb50de4f 100644 --- a/.github/workflows/build_and_unit_test.yml +++ b/.github/workflows/build_and_unit_test.yml @@ -38,6 +38,28 @@ jobs: cd ${GOPATH}/src/github.com/apache/cloudberry-backup make build + - name: Smoke Test + run: | + set -euo pipefail + cd ${GOPATH}/src/github.com/apache/cloudberry-backup + echo "Running smoke tests..." + echo "=== Testing gpbackup ===" + ${GOPATH}/bin/gpbackup --version + ${GOPATH}/bin/gpbackup --help > /dev/null + echo "=== Testing gprestore ===" + ${GOPATH}/bin/gprestore --version + ${GOPATH}/bin/gprestore --help > /dev/null + echo "=== Testing gpbackup_helper ===" + ${GOPATH}/bin/gpbackup_helper --version + ${GOPATH}/bin/gpbackup_helper --help > /dev/null + echo "=== Testing gpbackup_s3_plugin ===" + ${GOPATH}/bin/gpbackup_s3_plugin --version + ${GOPATH}/bin/gpbackup_s3_plugin --help > /dev/null + echo "=== Testing gpbackman ===" + ${GOPATH}/bin/gpbackman --version + ${GOPATH}/bin/gpbackman --help > /dev/null + echo "=== All smoke tests passed ===" + - name: Unit Test run: | cd ${GOPATH}/src/github.com/apache/cloudberry-backup diff --git a/.github/workflows/cloudberry-backup-ci.yml b/.github/workflows/cloudberry-backup-ci.yml index 619afee7..05068ff8 100644 --- a/.github/workflows/cloudberry-backup-ci.yml +++ b/.github/workflows/cloudberry-backup-ci.yml @@ -187,7 +187,7 @@ jobs: strategy: fail-fast: false matrix: - test_target: [unit, integration, end_to_end, s3_plugin_e2e, regression, scale] + test_target: [smoke, unit, integration, end_to_end, s3_plugin_e2e, regression, scale] steps: - name: Free Disk Space @@ -316,6 +316,26 @@ jobs: set +e case "${TEST_TARGET}" in + smoke) + set -e + echo "Running smoke tests..." + echo "=== Testing gpbackup ===" + ${GPHOME}/bin/gpbackup --version + ${GPHOME}/bin/gpbackup --help > /dev/null + echo "=== Testing gprestore ===" + ${GPHOME}/bin/gprestore --version + ${GPHOME}/bin/gprestore --help > /dev/null + echo "=== Testing gpbackup_helper ===" + ${GPHOME}/bin/gpbackup_helper --version + ${GPHOME}/bin/gpbackup_helper --help > /dev/null + echo "=== Testing gpbackup_s3_plugin ===" + ${GPHOME}/bin/gpbackup_s3_plugin --version + ${GPHOME}/bin/gpbackup_s3_plugin --help > /dev/null + echo "=== Testing gpbackman ===" + ${GPHOME}/bin/gpbackman --version + ${GPHOME}/bin/gpbackman --help > /dev/null + echo "=== All smoke tests passed ===" | tee "${TEST_LOG_ROOT}/cloudberry-backup-smoke.log" + ;; unit) make unit 2>&1 | tee "${TEST_LOG_ROOT}/cloudberry-backup-unit.log" ;; From cd64c085f504867eb6442f4b5a997ec7f49086ee Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Thu, 16 Apr 2026 10:25:19 +0800 Subject: [PATCH 24/39] Docs: update installation command to use specific version tag (#89) Update the installation instructions in README.md to use the specific version tag @2.1.0-incubating instead of @latest. The @latest tag resolves to v1.0.2 due to Go modules semantic versioning rules, which only recognizes version tags with 'v' prefix (v1.0.0, v1.0.1, v1.0.2). The 2.1.0-incubating tag, while functional when specified directly, is not recognized as a semantic version by Go modules. Added a note to clarify why users should use the specific version tag to avoid confusion and ensure they install the correct latest version. --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e3efa584..546bd802 100644 --- a/README.md +++ b/README.md @@ -42,12 +42,18 @@ export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin ## Download & Build -1. Downloading the latest version: +1. Downloading: ```bash -go install github.com/apache/cloudberry-backup@latest +# Download the stable release version +go install github.com/apache/cloudberry-backup@2.1.0-incubating + +# Or download the latest development version from main branch +go install github.com/apache/cloudberry-backup@main ``` +**Note:** Please use the specific version `@2.1.0-incubating` or `@main` instead of `@latest`. The `@latest` tag will install an older version due to Go modules version resolution rules. + This will place the code in `$GOPATH/pkg/mod/github.com/apache/cloudberry-backup`. 2. Building and installing binaries From dcb6e4059a0d8765433e756f3ba149e724c84c28 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Mon, 20 Apr 2026 11:15:21 +0800 Subject: [PATCH 25/39] Align release artifact naming with Apache incubator conventions Use base version (without -rcN suffix) for release tarball filename to align with Apache incubator release conventions. Changes: - Modified TAR_NAME to use ${VERSION_FILE} instead of ${TAG} - Tarball filename now follows pattern: apache-cloudberry-backup-${VERSION_FILE}-incubating-src.tar.gz - Example: apache-cloudberry-backup-2.0.0-incubating-src.tar.gz (instead of apache-cloudberry-backup-2.0.0-incubating-rc1-src.tar.gz) Benefits: - Enables direct 'svn mv' to release repository after voting without renaming artifacts - Aligns with Apache release best practices where RC identifiers are used only for Git tags and voting process, not in final artifact names - Maintains consistency between tarball filename and extracted directory name The extracted directory name remains unchanged: apache-cloudberry-backup-${VERSION_FILE}-incubating/ --- cloudberry-backup-release.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cloudberry-backup-release.sh b/cloudberry-backup-release.sh index a15697db..6cf277b9 100755 --- a/cloudberry-backup-release.sh +++ b/cloudberry-backup-release.sh @@ -510,7 +510,7 @@ section "Staging release: $TAG" section "Creating Source Tarball" - TAR_NAME="apache-cloudberry-backup-${TAG}-src.tar.gz" + TAR_NAME="apache-cloudberry-backup-${VERSION_FILE}-incubating-src.tar.gz" TMP_DIR=$(mktemp -d) trap 'rm -rf "$TMP_DIR"' EXIT @@ -518,6 +518,9 @@ section "Staging release: $TAG" export COPYFILE_DISABLE=1 export COPY_EXTENDED_ATTRIBUTES_DISABLE=1 + # Use base version (without -rcN) for both tarball filename and extracted directory name. + # This allows direct svn mv to release repository after voting without renaming. + git archive --format=tar --prefix="apache-cloudberry-backup-${VERSION_FILE}-incubating/" "$TAG" | tar -x -C "$TMP_DIR" # Archive submodules if any From bde7635461a3ca9df269a47be5d3c168e44a1415 Mon Sep 17 00:00:00 2001 From: "yangyu@hashdata.cn" Date: Fri, 24 Apr 2026 18:16:50 +0800 Subject: [PATCH 26/39] Fix: gprestore --resize-cluster fails with --jobs > 1 When restoring with --resize-cluster from a larger to smaller segment count, gprestore expands the oid list into (oid, batch) pairs in oid-major order ([T1B0, T1B1, T2B0, T2B1, ...]) and preloads min(NumConns, len(oidList)) segment pipes before starting the helpers. Coordinator workers take one task per table and iterate batches sequentially inside restoreSingleTableData, so the k-th worker's first COPY targets oidList index k*batches. When batches >= 2 and --jobs > 1 (NumConns > 1), workers beyond NumConns/batches race ahead of helper pipe creation and fail with: ERROR: command error message: cat: .../__0: No such file or directory and the restore only succeeds with --jobs 1. This matches the known upstream gpbackup issue fixed in 1.30.6. Preload NumConns*batches pipes (clamped to len(oidList)) so every concurrent worker's first batch is covered. The helper then rolls its queue forward one pipe per completed batch, and since a worker must finish all batches of its current table before pulling the next task, helper progress stays ahead of worker demand. For batches == 1 the behavior is unchanged. Fixes #92 --- restore/data.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/restore/data.go b/restore/data.go index 4175ad96..44d21120 100644 --- a/restore/data.go +++ b/restore/data.go @@ -242,7 +242,7 @@ func restoreDataFromTimestamp(fpInfo filepath.FilePathInfo, dataEntries []toc.Co } utils.WriteOidListToSegments(oidList, globalCluster, fpInfo, "oid") - initialPipes := CreateInitialSegmentPipes(oidList, globalCluster, connectionPool, fpInfo) + initialPipes := CreateInitialSegmentPipes(oidList, globalCluster, connectionPool, fpInfo, batches) if wasTerminated { return 0 } @@ -339,12 +339,16 @@ func restoreDataFromTimestamp(fpInfo filepath.FilePathInfo, dataEntries []toc.Co return numErrors } -func CreateInitialSegmentPipes(oidList []string, c *cluster.Cluster, connectionPool *dbconn.DBConn, fpInfo filepath.FilePathInfo) int { - // Create min(connections, tables) segment pipes on each host - var maxPipes int - if connectionPool.NumConns < len(oidList) { - maxPipes = connectionPool.NumConns - } else { +func CreateInitialSegmentPipes(oidList []string, c *cluster.Cluster, connectionPool *dbconn.DBConn, fpInfo filepath.FilePathInfo, batches int) int { + // oidList is laid out in oid-major order: [T1B0, T1B1, ..., T2B0, T2B1, ...]. + // Workers dispatch one task per table and iterate batches sequentially within + // restoreSingleTableData, so NumConns concurrent workers may request pipes at + // oidList indices 0, batches, 2*batches, ..., (NumConns-1)*batches before the + // helper has had a chance to create any of them. Preload NumConns*batches + // pipes so every concurrent worker's first batch is covered; the helper then + // rolls the queue forward as each batch completes. + maxPipes := connectionPool.NumConns * batches + if maxPipes > len(oidList) { maxPipes = len(oidList) } for i := 0; i < maxPipes; i++ { From 0017e61900b03f0121802d5163325bc009099860 Mon Sep 17 00:00:00 2001 From: woblerr Date: Mon, 27 Apr 2026 23:40:00 +0300 Subject: [PATCH 27/39] Speed up column permissions query. Use a top-level `unnest(CASE ...) AS privileges` instead of `LEFT JOIN LATERAL`. This removes an extra join and per-array expansion, reducing planner/executor work when scanning large catalogs. Related to https://github.com/apache/cloudberry-backup/issues/88 --- backup/queries_table_defs.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/backup/queries_table_defs.go b/backup/queries_table_defs.go index 4c6bf612..49282fd2 100644 --- a/backup/queries_table_defs.go +++ b/backup/queries_table_defs.go @@ -327,9 +327,9 @@ func GetColumnDefinitions(connectionPool *dbconn.DBConn) map[uint32][]ColumnDefi ORDER BY a.attrelid, a.attnum`, relationAndSchemaFilterClause()) // In GPDB7+ we do not want to exclude child partitions, they function as separate tables. - // Cannot use unnest() in CASE statements anymore in GPDB 7+ so convert - // it to a LEFT JOIN LATERAL. We do not use LEFT JOIN LATERAL for GPDB 6 - // because the CASE unnest() logic is more performant. + // Use a top-level unnest(CASE ...) AS privileges instead of LEFT JOIN LATERAL. + // This removes an extra join and per-array expansion, reducing planner/executor work + // when scanning large catalogs. Keep the GPDB6 CASE-unnest because it was faster there. atLeast7Query := fmt.Sprintf(` SELECT a.attrelid, a.attnum, @@ -343,7 +343,11 @@ func GetColumnDefinitions(connectionPool *dbconn.DBConn) map[uint32][]ColumnDefi coalesce('('||pg_catalog.pg_get_expr(ad.adbin, ad.adrelid)||')', '') AS defaultval, coalesce(d.description, '') AS comment, a.attgenerated, - ljl_unnest AS privileges, + unnest(CASE + WHEN a.attacl IS NULL OR array_length(a.attacl, 1) IS NULL + THEN ARRAY[NULL::aclitem] + ELSE a.attacl + END) AS privileges, CASE WHEN a.attacl IS NULL THEN '' WHEN array_upper(a.attacl, 1) = 0 THEN 'Empty' @@ -365,7 +369,6 @@ func GetColumnDefinitions(connectionPool *dbconn.DBConn) map[uint32][]ColumnDefi LEFT JOIN pg_collation coll ON a.attcollation = coll.oid LEFT JOIN pg_namespace cn ON coll.collnamespace = cn.oid LEFT JOIN pg_seclabel sec ON sec.objoid = a.attrelid AND sec.classoid = 'pg_class'::regclass AND sec.objsubid = a.attnum - LEFT JOIN LATERAL unnest(a.attacl) ljl_unnest ON a.attacl IS NOT NULL AND array_length(a.attacl, 1) != 0 WHERE %s AND c.reltype <> 0 AND a.attnum > 0::pg_catalog.int2 From dfbd935223a89e50d6ff33b9077387ba6cecf4ea Mon Sep 17 00:00:00 2001 From: chenqiang Date: Wed, 29 Apr 2026 12:45:47 +0800 Subject: [PATCH 28/39] gpbackman: fall back to coordinator data dir for history db, fail loud when missing. Two improvements to how gpbackman locates and opens gpbackup_history.db: 1. When --history-db is empty, fall back (in priority order) to $COORDINATOR_DATA_DIRECTORY and $MASTER_DATA_DIRECTORY before the current directory. These variables are exported by the standard Cloudberry/Greenplum environment scripts, so users running gpbackman from a sourced cluster shell no longer need to repeat --history-db on every invocation. 2. OpenHistoryDB now pre-checks the file with os.Stat and opens the SQLite database with the rw URI mode. A missing file produces a friendly error pointing at --history-db and the env vars, instead of silently creating an empty file that later fails with "no such table: backups" when callers issue queries. The previous cwd-only fallback is preserved as a last resort. Tests cover the new env-var resolution order and the safe-open behaviour. Signed-off-by: chenqiang --- gpbackman/cmd/backup_clean.go | 2 +- gpbackman/cmd/backup_delete.go | 2 +- gpbackman/cmd/backup_info.go | 2 +- gpbackman/cmd/constants.go | 8 +++++ gpbackman/cmd/history_clean.go | 2 +- gpbackman/cmd/report_info.go | 2 +- gpbackman/cmd/root.go | 4 ++- gpbackman/cmd/wrappers.go | 16 +++++++-- gpbackman/cmd/wrappers_test.go | 45 ++++++++++++++++++++++++- gpbackman/gpbckpconfig/utils_db.go | 27 +++++++++++++-- gpbackman/gpbckpconfig/utils_db_test.go | 38 +++++++++++++++++++++ 11 files changed, 137 insertions(+), 11 deletions(-) diff --git a/gpbackman/cmd/backup_clean.go b/gpbackman/cmd/backup_clean.go index 4f5e2179..1ccaf8f8 100644 --- a/gpbackman/cmd/backup_clean.go +++ b/gpbackman/cmd/backup_clean.go @@ -77,7 +77,7 @@ For non local backups the following logic are applied: The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database will be searched in the current directory.`, +If the --history-db option is not specified, the history database is looked for under $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY); if neither variable is set, the current directory is used as a last resort.`, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { doRootFlagValidation(cmd.Flags(), checkFileExistsConst) diff --git a/gpbackman/cmd/backup_delete.go b/gpbackman/cmd/backup_delete.go index 14fa2795..202d52b3 100644 --- a/gpbackman/cmd/backup_delete.go +++ b/gpbackman/cmd/backup_delete.go @@ -84,7 +84,7 @@ For non local backups the following logic are applied: The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database will be searched in the current directory.`, +If the --history-db option is not specified, the history database is looked for under $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY); if neither variable is set, the current directory is used as a last resort.`, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { doRootFlagValidation(cmd.Flags(), checkFileExistsConst) diff --git a/gpbackman/cmd/backup_info.go b/gpbackman/cmd/backup_info.go index 199311cc..0515e402 100644 --- a/gpbackman/cmd/backup_info.go +++ b/gpbackman/cmd/backup_info.go @@ -98,7 +98,7 @@ To display the "object filtering details" column for all backups without using - The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database will be searched in the current directory.`, +If the --history-db option is not specified, the history database is looked for under $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY); if neither variable is set, the current directory is used as a last resort.`, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { doRootFlagValidation(cmd.Flags(), checkFileExistsConst) diff --git a/gpbackman/cmd/constants.go b/gpbackman/cmd/constants.go index 8259b594..14610c5a 100644 --- a/gpbackman/cmd/constants.go +++ b/gpbackman/cmd/constants.go @@ -72,4 +72,12 @@ var ( beforeTimestamp string // Timestamp to delete all backups after. afterTimestamp string + + // historyDBEnvVars lists, in priority order, the environment variables + // inspected when --history-db is not supplied. They are exported by the + // standard Cloudberry/Greenplum cluster environment scripts. + historyDBEnvVars = []string{ + "COORDINATOR_DATA_DIRECTORY", + "MASTER_DATA_DIRECTORY", + } ) diff --git a/gpbackman/cmd/history_clean.go b/gpbackman/cmd/history_clean.go index d5de643a..836991b8 100644 --- a/gpbackman/cmd/history_clean.go +++ b/gpbackman/cmd/history_clean.go @@ -50,7 +50,7 @@ Only --older-than-days or --before-timestamp option must be specified, not both. The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database will be searched in the current directory.`, +If the --history-db option is not specified, the history database is looked for under $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY); if neither variable is set, the current directory is used as a last resort.`, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { doRootFlagValidation(cmd.Flags(), checkFileExistsConst) diff --git a/gpbackman/cmd/report_info.go b/gpbackman/cmd/report_info.go index 3ac7428f..168695ae 100644 --- a/gpbackman/cmd/report_info.go +++ b/gpbackman/cmd/report_info.go @@ -78,7 +78,7 @@ It is not necessary to use the --plugin-report-file-path flag for the following The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database will be searched in the current directory.`, +If the --history-db option is not specified, the history database is looked for under $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY); if neither variable is set, the current directory is used as a last resort.`, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { doRootFlagValidation(cmd.Flags(), checkFileExistsConst) diff --git a/gpbackman/cmd/root.go b/gpbackman/cmd/root.go index 1c113b37..e8473269 100644 --- a/gpbackman/cmd/root.go +++ b/gpbackman/cmd/root.go @@ -50,7 +50,9 @@ func init() { &rootHistoryDB, historyDBFlagName, "", - "full path to the gpbackup_history.db file", + "full path to the gpbackup_history.db file (if unset, falls back to "+ + "$COORDINATOR_DATA_DIRECTORY/gpbackup_history.db, then "+ + "$MASTER_DATA_DIRECTORY/gpbackup_history.db, then the current directory)", ) rootCmd.PersistentFlags().StringVar( &rootLogFile, diff --git a/gpbackman/cmd/wrappers.go b/gpbackman/cmd/wrappers.go index 79e34f38..be1c5b59 100644 --- a/gpbackman/cmd/wrappers.go +++ b/gpbackman/cmd/wrappers.go @@ -81,12 +81,24 @@ func setLogLevelFile(level string) error { return nil } +// getHistoryDBPath resolves the path to the gpbackup_history.db file. +// When the --history-db flag is empty, fall back (in order) to the +// COORDINATOR_DATA_DIRECTORY and MASTER_DATA_DIRECTORY environment variables +// that the standard Cloudberry/Greenplum environment scripts export, so that +// users running gpbackman from a sourced cluster shell do not need to repeat +// the path on every invocation. As a last resort, return the bare filename +// (resolved against the current working directory), preserving the previous +// behaviour. func getHistoryDBPath(historyDBPath string) string { - var historyDBName = historyDBNameConst if historyDBPath != "" { return historyDBPath } - return historyDBName + for _, envVar := range historyDBEnvVars { + if dir := os.Getenv(envVar); dir != "" { + return filepath.Join(dir, historyDBNameConst) + } + } + return historyDBNameConst } func checkCompatibleFlags(flags *pflag.FlagSet, flagNames ...string) error { diff --git a/gpbackman/cmd/wrappers_test.go b/gpbackman/cmd/wrappers_test.go index e378fbf1..d7e16093 100644 --- a/gpbackman/cmd/wrappers_test.go +++ b/gpbackman/cmd/wrappers_test.go @@ -33,13 +33,56 @@ import ( var _ = Describe("wrappers tests", func() { Describe("getHistoryDBPath", func() { - It("returns default path when input is empty", func() { + // Save and restore env vars so these cases don't leak into the rest + // of the suite when run with --randomize-all. + var savedEnv map[string]string + + BeforeEach(func() { + savedEnv = make(map[string]string, len(historyDBEnvVars)) + for _, name := range historyDBEnvVars { + savedEnv[name] = os.Getenv(name) + os.Unsetenv(name) + } + }) + + AfterEach(func() { + for name, val := range savedEnv { + if val == "" { + os.Unsetenv(name) + } else { + os.Setenv(name, val) + } + } + }) + + It("returns default filename when input is empty and no env vars are set", func() { Expect(getHistoryDBPath("")).To(Equal(historyDBNameConst)) }) It("returns input path when not empty", func() { Expect(getHistoryDBPath("path/to/" + historyDBNameConst)).To(Equal("path/to/" + historyDBNameConst)) }) + + It("falls back to COORDINATOR_DATA_DIRECTORY when input is empty", func() { + os.Setenv("COORDINATOR_DATA_DIRECTORY", "/coord/data") + Expect(getHistoryDBPath("")).To(Equal(filepath.Join("/coord/data", historyDBNameConst))) + }) + + It("falls back to MASTER_DATA_DIRECTORY when COORDINATOR is unset", func() { + os.Setenv("MASTER_DATA_DIRECTORY", "/master/data") + Expect(getHistoryDBPath("")).To(Equal(filepath.Join("/master/data", historyDBNameConst))) + }) + + It("prefers COORDINATOR over MASTER when both are set", func() { + os.Setenv("COORDINATOR_DATA_DIRECTORY", "/coord/data") + os.Setenv("MASTER_DATA_DIRECTORY", "/master/data") + Expect(getHistoryDBPath("")).To(Equal(filepath.Join("/coord/data", historyDBNameConst))) + }) + + It("explicit input wins over env vars", func() { + os.Setenv("COORDINATOR_DATA_DIRECTORY", "/coord/data") + Expect(getHistoryDBPath("/explicit/path.db")).To(Equal("/explicit/path.db")) + }) }) Describe("formatBackupDuration", func() { diff --git a/gpbackman/gpbckpconfig/utils_db.go b/gpbackman/gpbckpconfig/utils_db.go index 304c0907..9cf18df5 100644 --- a/gpbackman/gpbckpconfig/utils_db.go +++ b/gpbackman/gpbckpconfig/utils_db.go @@ -21,15 +21,38 @@ package gpbckpconfig import ( "database/sql" + "errors" "fmt" + "os" "strings" "github.com/apache/cloudberry-backup/history" ) -// OpenHistoryDB opens the history backup database. +// OpenHistoryDB opens an existing gpbackup_history.db SQLite database. +// +// The path is opened with the SQLite "rw" URI mode so that a missing file +// produces a clear error rather than being silently created as an empty +// database (which would later fail with a confusing "no such table: backups" +// when callers issue queries). Existence is also pre-checked with os.Stat to +// surface a friendly error message that points the caller at the relevant +// flag and environment variables. func OpenHistoryDB(historyDBPath string) (*sql.DB, error) { - db, err := sql.Open("sqlite3", historyDBPath) + if _, err := os.Stat(historyDBPath); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf( + "gpbackup history database file not found: %s. "+ + "Specify the path via --history-db, set "+ + "COORDINATOR_DATA_DIRECTORY (or MASTER_DATA_DIRECTORY), "+ + "or run gpbackman from the directory that contains "+ + "gpbackup_history.db", + historyDBPath, + ) + } + return nil, err + } + // mode=rw opens an existing database for read+write but never creates one. + db, err := sql.Open("sqlite3", "file:"+historyDBPath+"?mode=rw") if err != nil { return nil, err } diff --git a/gpbackman/gpbckpconfig/utils_db_test.go b/gpbackman/gpbckpconfig/utils_db_test.go index 17c6530b..a89e93e9 100644 --- a/gpbackman/gpbckpconfig/utils_db_test.go +++ b/gpbackman/gpbckpconfig/utils_db_test.go @@ -20,7 +20,10 @@ under the License. package gpbckpconfig import ( + "database/sql" "fmt" + "os" + "path/filepath" "github.com/apache/cloudberry-backup/history" . "github.com/onsi/ginkgo/v2" @@ -28,6 +31,41 @@ import ( ) var _ = Describe("utils_db tests", func() { + Describe("OpenHistoryDB", func() { + It("returns a friendly error and does not create a file when the path does not exist", func() { + tempDir := GinkgoT().TempDir() + missing := filepath.Join(tempDir, "does-not-exist.db") + + db, err := OpenHistoryDB(missing) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("not found")) + Expect(err.Error()).To(ContainSubstring("--history-db")) + Expect(db).To(BeNil()) + + // Critical regression: no empty SQLite file must have been created. + _, statErr := os.Stat(missing) + Expect(os.IsNotExist(statErr)).To(BeTrue(), "OpenHistoryDB must not create the file when it is missing") + }) + + It("opens an existing SQLite history database successfully", func() { + tempDir := GinkgoT().TempDir() + path := filepath.Join(tempDir, "gpbackup_history.db") + + // Seed an existing (but empty) SQLite file via the rwc URI mode. + seed, err := sql.Open("sqlite3", "file:"+path+"?mode=rwc") + Expect(err).NotTo(HaveOccurred()) + Expect(seed.Ping()).To(Succeed()) + Expect(seed.Close()).To(Succeed()) + + db, err := OpenHistoryDB(path) + Expect(err).NotTo(HaveOccurred()) + Expect(db).NotTo(BeNil()) + Expect(db.Ping()).To(Succeed()) + Expect(db.Close()).To(Succeed()) + }) + }) + Describe("getBackupNameQuery", func() { It("returns correct query for various flag combinations", func() { tests := []struct { From 4b4912c7a92c7a84e1c90909e6d80c398560b147 Mon Sep 17 00:00:00 2001 From: chenqiang Date: Wed, 29 Apr 2026 12:45:52 +0800 Subject: [PATCH 29/39] gpbackman: document history-db env-var fallback in COMMANDS.md and README.md. Replace the "current directory" wording everywhere it appeared with the new resolution order: $COORDINATOR_DATA_DIRECTORY, $MASTER_DATA_DIRECTORY, then the current directory. Signed-off-by: chenqiang --- gpbackman/COMMANDS.md | 10 +++++----- gpbackman/README.md | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gpbackman/COMMANDS.md b/gpbackman/COMMANDS.md index c67947a3..cf82aa8b 100644 --- a/gpbackman/COMMANDS.md +++ b/gpbackman/COMMANDS.md @@ -72,7 +72,7 @@ For non local backups the following logic are applied: The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database will be searched in the current directory. +If the --history-db option is not specified, the history database is looked for under `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`); if neither variable is set, the current directory is used as a last resort. Usage: gpbackman backup-clean [flags] @@ -158,7 +158,7 @@ For non local backups the following logic are applied: The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database will be searched in the current directory. +If the --history-db option is not specified, the history database is looked for under `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`); if neither variable is set, the current directory is used as a last resort. Usage: gpbackman backup-delete [flags] @@ -256,7 +256,7 @@ To display the "object filtering details" column for all backups without using - The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database will be searched in the current directory. +If the --history-db option is not specified, the history database is looked for under `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`); if neither variable is set, the current directory is used as a last resort. Usage: gpbackman backup-info [flags] @@ -455,7 +455,7 @@ Only --older-than-days or --before-timestamp option must be specified, not both. The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database will be searched in the current directory. +If the --history-db option is not specified, the history database is looked for under `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`); if neither variable is set, the current directory is used as a last resort. Usage: gpbackman history-clean [flags] @@ -524,7 +524,7 @@ It is not necessary to use the --plugin-report-file-path flag for the following The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database will be searched in the current directory. +If the --history-db option is not specified, the history database is looked for under `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`); if neither variable is set, the current directory is used as a last resort. Usage: gpbackman report-info [flags] diff --git a/gpbackman/README.md b/gpbackman/README.md index 388ff898..923964ad 100644 --- a/gpbackman/README.md +++ b/gpbackman/README.md @@ -53,7 +53,7 @@ Available Commands: Flags: -h, --help help for gpbackman - --history-db string full path to the gpbackup_history.db file + --history-db string full path to the gpbackup_history.db file (if unset, falls back to $COORDINATOR_DATA_DIRECTORY/gpbackup_history.db, then $MASTER_DATA_DIRECTORY/gpbackup_history.db, then the current directory) --log-file string full path to log file directory, if not specified, the log file will be created in the $HOME/gpAdminLogs directory --log-level-console string level for console logging (error, info, debug, verbose) (default "info") --log-level-file string level for file logging (error, info, debug, verbose) (default "info") From cec7671c49a7f08ab57e9aa35cf93272651c1a57 Mon Sep 17 00:00:00 2001 From: chenqiang Date: Wed, 29 Apr 2026 15:35:38 +0800 Subject: [PATCH 30/39] gpbackman: gate history-db env-var fallback behind --auto-load-history-db opt-in. Address review feedback on the original PR: * The env-var fallback ($COORDINATOR_DATA_DIRECTORY then $MASTER_DATA_DIRECTORY) is now opt-in via a new persistent flag --auto-load-history-db. The default behaviour (current-directory lookup) is unchanged from upstream main, so destructive subcommands (backup-delete, backup-clean, history-clean) cannot silently target the wrong cluster's history DB on a multi-cluster jump host. * Drop the "Greenplum" mention in the comment for getHistoryDBPath to keep wording aligned with the Cloudberry environment scripts. * Trim the verbose --history-db description in gpbackman/README.md and add a separate --auto-load-history-db line. * Refine the not-found error message in OpenHistoryDB to mention the new flag rather than implying that setting env vars alone helps. * Tests updated to cover the opt-in matrix: env vars are ignored when the flag is off, honoured when on, and an explicit --history-db still wins regardless. The OpenHistoryDB safe-open change (os.Stat pre-check + mode=rw URI to avoid silently creating an empty SQLite file) is preserved as-is. Signed-off-by: chenqiang --- gpbackman/COMMANDS.md | 10 +++++----- gpbackman/README.md | 3 ++- gpbackman/cmd/backup_clean.go | 4 ++-- gpbackman/cmd/backup_delete.go | 4 ++-- gpbackman/cmd/backup_info.go | 4 ++-- gpbackman/cmd/constants.go | 1 + gpbackman/cmd/history_clean.go | 4 ++-- gpbackman/cmd/report_info.go | 4 ++-- gpbackman/cmd/root.go | 21 +++++++++++++------- gpbackman/cmd/wrappers.go | 24 ++++++++++++---------- gpbackman/cmd/wrappers_test.go | 32 ++++++++++++++++++++---------- gpbackman/gpbckpconfig/utils_db.go | 8 ++++---- 12 files changed, 70 insertions(+), 49 deletions(-) diff --git a/gpbackman/COMMANDS.md b/gpbackman/COMMANDS.md index cf82aa8b..7cb15fd0 100644 --- a/gpbackman/COMMANDS.md +++ b/gpbackman/COMMANDS.md @@ -72,7 +72,7 @@ For non local backups the following logic are applied: The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for under `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`); if neither variable is set, the current directory is used as a last resort. +If the --history-db option is not specified, the history database is looked for in the current directory. Pass `--auto-load-history-db` to resolve it from `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`) instead. Usage: gpbackman backup-clean [flags] @@ -158,7 +158,7 @@ For non local backups the following logic are applied: The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for under `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`); if neither variable is set, the current directory is used as a last resort. +If the --history-db option is not specified, the history database is looked for in the current directory. Pass `--auto-load-history-db` to resolve it from `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`) instead. Usage: gpbackman backup-delete [flags] @@ -256,7 +256,7 @@ To display the "object filtering details" column for all backups without using - The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for under `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`); if neither variable is set, the current directory is used as a last resort. +If the --history-db option is not specified, the history database is looked for in the current directory. Pass `--auto-load-history-db` to resolve it from `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`) instead. Usage: gpbackman backup-info [flags] @@ -455,7 +455,7 @@ Only --older-than-days or --before-timestamp option must be specified, not both. The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for under `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`); if neither variable is set, the current directory is used as a last resort. +If the --history-db option is not specified, the history database is looked for in the current directory. Pass `--auto-load-history-db` to resolve it from `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`) instead. Usage: gpbackman history-clean [flags] @@ -524,7 +524,7 @@ It is not necessary to use the --plugin-report-file-path flag for the following The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for under `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`); if neither variable is set, the current directory is used as a last resort. +If the --history-db option is not specified, the history database is looked for in the current directory. Pass `--auto-load-history-db` to resolve it from `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`) instead. Usage: gpbackman report-info [flags] diff --git a/gpbackman/README.md b/gpbackman/README.md index 923964ad..930e570b 100644 --- a/gpbackman/README.md +++ b/gpbackman/README.md @@ -53,7 +53,8 @@ Available Commands: Flags: -h, --help help for gpbackman - --history-db string full path to the gpbackup_history.db file (if unset, falls back to $COORDINATOR_DATA_DIRECTORY/gpbackup_history.db, then $MASTER_DATA_DIRECTORY/gpbackup_history.db, then the current directory) + --auto-load-history-db resolve gpbackup_history.db from $COORDINATOR_DATA_DIRECTORY (or $MASTER_DATA_DIRECTORY) when --history-db is unset + --history-db string full path to the gpbackup_history.db file --log-file string full path to log file directory, if not specified, the log file will be created in the $HOME/gpAdminLogs directory --log-level-console string level for console logging (error, info, debug, verbose) (default "info") --log-level-file string level for file logging (error, info, debug, verbose) (default "info") diff --git a/gpbackman/cmd/backup_clean.go b/gpbackman/cmd/backup_clean.go index 1ccaf8f8..eacd3418 100644 --- a/gpbackman/cmd/backup_clean.go +++ b/gpbackman/cmd/backup_clean.go @@ -77,7 +77,7 @@ For non local backups the following logic are applied: The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for under $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY); if neither variable is set, the current directory is used as a last resort.`, +If the --history-db option is not specified, the history database is looked for in the current directory. To resolve it from $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY) instead, pass the --auto-load-history-db flag.`, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { doRootFlagValidation(cmd.Flags(), checkFileExistsConst) @@ -205,7 +205,7 @@ func doCleanBackup() { } func cleanBackup() error { - hDB, err := gpbckpconfig.OpenHistoryDB(getHistoryDBPath(rootHistoryDB)) + hDB, err := gpbckpconfig.OpenHistoryDB(getHistoryDBPath(rootHistoryDB, rootAutoLoadHistoryDB)) if err != nil { gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("open", err)) return err diff --git a/gpbackman/cmd/backup_delete.go b/gpbackman/cmd/backup_delete.go index 202d52b3..2d4b95f8 100644 --- a/gpbackman/cmd/backup_delete.go +++ b/gpbackman/cmd/backup_delete.go @@ -84,7 +84,7 @@ For non local backups the following logic are applied: The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for under $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY); if neither variable is set, the current directory is used as a last resort.`, +If the --history-db option is not specified, the history database is looked for in the current directory. To resolve it from $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY) instead, pass the --auto-load-history-db flag.`, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { doRootFlagValidation(cmd.Flags(), checkFileExistsConst) @@ -205,7 +205,7 @@ func doDeleteBackup() { } func deleteBackup() error { - hDB, err := gpbckpconfig.OpenHistoryDB(getHistoryDBPath(rootHistoryDB)) + hDB, err := gpbckpconfig.OpenHistoryDB(getHistoryDBPath(rootHistoryDB, rootAutoLoadHistoryDB)) if err != nil { gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("open", err)) return err diff --git a/gpbackman/cmd/backup_info.go b/gpbackman/cmd/backup_info.go index 0515e402..adb5c118 100644 --- a/gpbackman/cmd/backup_info.go +++ b/gpbackman/cmd/backup_info.go @@ -98,7 +98,7 @@ To display the "object filtering details" column for all backups without using - The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for under $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY); if neither variable is set, the current directory is used as a last resort.`, +If the --history-db option is not specified, the history database is looked for in the current directory. To resolve it from $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY) instead, pass the --auto-load-history-db flag.`, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { doRootFlagValidation(cmd.Flags(), checkFileExistsConst) @@ -226,7 +226,7 @@ func backupInfo() error { } t := tablewriter.NewWriter(os.Stdout) initTable(t, opts.ShowDetails) - hDB, err := gpbckpconfig.OpenHistoryDB(getHistoryDBPath(rootHistoryDB)) + hDB, err := gpbckpconfig.OpenHistoryDB(getHistoryDBPath(rootHistoryDB, rootAutoLoadHistoryDB)) if err != nil { gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("open", err)) return err diff --git a/gpbackman/cmd/constants.go b/gpbackman/cmd/constants.go index 14610c5a..7b5b59ed 100644 --- a/gpbackman/cmd/constants.go +++ b/gpbackman/cmd/constants.go @@ -35,6 +35,7 @@ const ( // Flags. historyDBFlagName = "history-db" + autoLoadHistoryDBFlagName = "auto-load-history-db" logFileFlagName = "log-file" logLevelConsoleFlagName = "log-level-console" logLevelFileFlagName = "log-level-file" diff --git a/gpbackman/cmd/history_clean.go b/gpbackman/cmd/history_clean.go index 836991b8..361f4ee0 100644 --- a/gpbackman/cmd/history_clean.go +++ b/gpbackman/cmd/history_clean.go @@ -50,7 +50,7 @@ Only --older-than-days or --before-timestamp option must be specified, not both. The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for under $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY); if neither variable is set, the current directory is used as a last resort.`, +If the --history-db option is not specified, the history database is looked for in the current directory. To resolve it from $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY) instead, pass the --auto-load-history-db flag.`, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { doRootFlagValidation(cmd.Flags(), checkFileExistsConst) @@ -106,7 +106,7 @@ func doCleanHistory() { } func cleanHistory() error { - hDB, err := gpbckpconfig.OpenHistoryDB(getHistoryDBPath(rootHistoryDB)) + hDB, err := gpbckpconfig.OpenHistoryDB(getHistoryDBPath(rootHistoryDB, rootAutoLoadHistoryDB)) if err != nil { gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("open", err)) return err diff --git a/gpbackman/cmd/report_info.go b/gpbackman/cmd/report_info.go index 168695ae..042a3da5 100644 --- a/gpbackman/cmd/report_info.go +++ b/gpbackman/cmd/report_info.go @@ -78,7 +78,7 @@ It is not necessary to use the --plugin-report-file-path flag for the following The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for under $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY); if neither variable is set, the current directory is used as a last resort.`, +If the --history-db option is not specified, the history database is looked for in the current directory. To resolve it from $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY) instead, pass the --auto-load-history-db flag.`, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { doRootFlagValidation(cmd.Flags(), checkFileExistsConst) @@ -174,7 +174,7 @@ func doReportInfo() { } func reportInfo() error { - hDB, err := gpbckpconfig.OpenHistoryDB(getHistoryDBPath(rootHistoryDB)) + hDB, err := gpbckpconfig.OpenHistoryDB(getHistoryDBPath(rootHistoryDB, rootAutoLoadHistoryDB)) if err != nil { gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("open", err)) return err diff --git a/gpbackman/cmd/root.go b/gpbackman/cmd/root.go index e8473269..50884771 100644 --- a/gpbackman/cmd/root.go +++ b/gpbackman/cmd/root.go @@ -33,10 +33,11 @@ var version string // Flags for the gpbackman command (rootCmd) var ( - rootHistoryDB string - rootLogFile string - rootLogLevelConsole string - rootLogLevelFile string + rootHistoryDB string + rootAutoLoadHistoryDB bool + rootLogFile string + rootLogLevelConsole string + rootLogLevelFile string ) var rootCmd = &cobra.Command{ @@ -50,9 +51,15 @@ func init() { &rootHistoryDB, historyDBFlagName, "", - "full path to the gpbackup_history.db file (if unset, falls back to "+ - "$COORDINATOR_DATA_DIRECTORY/gpbackup_history.db, then "+ - "$MASTER_DATA_DIRECTORY/gpbackup_history.db, then the current directory)", + "full path to the gpbackup_history.db file", + ) + rootCmd.PersistentFlags().BoolVar( + &rootAutoLoadHistoryDB, + autoLoadHistoryDBFlagName, + false, + "when --history-db is unset, look up gpbackup_history.db under "+ + "$COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY) before "+ + "falling back to the current directory", ) rootCmd.PersistentFlags().StringVar( &rootLogFile, diff --git a/gpbackman/cmd/wrappers.go b/gpbackman/cmd/wrappers.go index be1c5b59..f809ce4c 100644 --- a/gpbackman/cmd/wrappers.go +++ b/gpbackman/cmd/wrappers.go @@ -82,20 +82,22 @@ func setLogLevelFile(level string) error { } // getHistoryDBPath resolves the path to the gpbackup_history.db file. -// When the --history-db flag is empty, fall back (in order) to the -// COORDINATOR_DATA_DIRECTORY and MASTER_DATA_DIRECTORY environment variables -// that the standard Cloudberry/Greenplum environment scripts export, so that -// users running gpbackman from a sourced cluster shell do not need to repeat -// the path on every invocation. As a last resort, return the bare filename -// (resolved against the current working directory), preserving the previous -// behaviour. -func getHistoryDBPath(historyDBPath string) string { +// An explicit --history-db value always wins. Otherwise, when the caller +// asks for auto-load (--auto-load-history-db), look up the file under the +// COORDINATOR_DATA_DIRECTORY / MASTER_DATA_DIRECTORY environment variables +// exported by the standard Cloudberry environment scripts. As a final +// fallback, return the bare filename so it is resolved against the current +// working directory, preserving the original behaviour for the default +// invocation. +func getHistoryDBPath(historyDBPath string, autoLoad bool) string { if historyDBPath != "" { return historyDBPath } - for _, envVar := range historyDBEnvVars { - if dir := os.Getenv(envVar); dir != "" { - return filepath.Join(dir, historyDBNameConst) + if autoLoad { + for _, envVar := range historyDBEnvVars { + if dir := os.Getenv(envVar); dir != "" { + return filepath.Join(dir, historyDBNameConst) + } } } return historyDBNameConst diff --git a/gpbackman/cmd/wrappers_test.go b/gpbackman/cmd/wrappers_test.go index d7e16093..401f2962 100644 --- a/gpbackman/cmd/wrappers_test.go +++ b/gpbackman/cmd/wrappers_test.go @@ -55,33 +55,43 @@ var _ = Describe("wrappers tests", func() { } }) - It("returns default filename when input is empty and no env vars are set", func() { - Expect(getHistoryDBPath("")).To(Equal(historyDBNameConst)) + It("returns default filename when input is empty (auto-load off, no env)", func() { + Expect(getHistoryDBPath("", false)).To(Equal(historyDBNameConst)) }) It("returns input path when not empty", func() { - Expect(getHistoryDBPath("path/to/" + historyDBNameConst)).To(Equal("path/to/" + historyDBNameConst)) + Expect(getHistoryDBPath("path/to/"+historyDBNameConst, false)).To(Equal("path/to/" + historyDBNameConst)) }) - It("falls back to COORDINATOR_DATA_DIRECTORY when input is empty", func() { + It("ignores env vars by default (auto-load off)", func() { os.Setenv("COORDINATOR_DATA_DIRECTORY", "/coord/data") - Expect(getHistoryDBPath("")).To(Equal(filepath.Join("/coord/data", historyDBNameConst))) + os.Setenv("MASTER_DATA_DIRECTORY", "/master/data") + Expect(getHistoryDBPath("", false)).To(Equal(historyDBNameConst)) + }) + + It("falls back to COORDINATOR_DATA_DIRECTORY when auto-load is on", func() { + os.Setenv("COORDINATOR_DATA_DIRECTORY", "/coord/data") + Expect(getHistoryDBPath("", true)).To(Equal(filepath.Join("/coord/data", historyDBNameConst))) }) - It("falls back to MASTER_DATA_DIRECTORY when COORDINATOR is unset", func() { + It("falls back to MASTER_DATA_DIRECTORY when COORDINATOR is unset and auto-load is on", func() { os.Setenv("MASTER_DATA_DIRECTORY", "/master/data") - Expect(getHistoryDBPath("")).To(Equal(filepath.Join("/master/data", historyDBNameConst))) + Expect(getHistoryDBPath("", true)).To(Equal(filepath.Join("/master/data", historyDBNameConst))) }) - It("prefers COORDINATOR over MASTER when both are set", func() { + It("prefers COORDINATOR over MASTER when both are set and auto-load is on", func() { os.Setenv("COORDINATOR_DATA_DIRECTORY", "/coord/data") os.Setenv("MASTER_DATA_DIRECTORY", "/master/data") - Expect(getHistoryDBPath("")).To(Equal(filepath.Join("/coord/data", historyDBNameConst))) + Expect(getHistoryDBPath("", true)).To(Equal(filepath.Join("/coord/data", historyDBNameConst))) + }) + + It("returns the cwd default when auto-load is on but no env vars are set", func() { + Expect(getHistoryDBPath("", true)).To(Equal(historyDBNameConst)) }) - It("explicit input wins over env vars", func() { + It("explicit input wins over env vars even when auto-load is on", func() { os.Setenv("COORDINATOR_DATA_DIRECTORY", "/coord/data") - Expect(getHistoryDBPath("/explicit/path.db")).To(Equal("/explicit/path.db")) + Expect(getHistoryDBPath("/explicit/path.db", true)).To(Equal("/explicit/path.db")) }) }) diff --git a/gpbackman/gpbckpconfig/utils_db.go b/gpbackman/gpbckpconfig/utils_db.go index 9cf18df5..07e307d9 100644 --- a/gpbackman/gpbckpconfig/utils_db.go +++ b/gpbackman/gpbckpconfig/utils_db.go @@ -42,10 +42,10 @@ func OpenHistoryDB(historyDBPath string) (*sql.DB, error) { if errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf( "gpbackup history database file not found: %s. "+ - "Specify the path via --history-db, set "+ - "COORDINATOR_DATA_DIRECTORY (or MASTER_DATA_DIRECTORY), "+ - "or run gpbackman from the directory that contains "+ - "gpbackup_history.db", + "Specify the path via --history-db, run gpbackman from "+ + "the directory that contains gpbackup_history.db, or "+ + "pass --auto-load-history-db to resolve it from "+ + "$COORDINATOR_DATA_DIRECTORY (or $MASTER_DATA_DIRECTORY)", historyDBPath, ) } From 017b644b6ef4c9423df8ca854bb1cced1c0bf757 Mon Sep 17 00:00:00 2001 From: chenqiang Date: Wed, 29 Apr 2026 16:17:57 +0800 Subject: [PATCH 31/39] gpbackman: wrap non-NotExist os.Stat error with path context in OpenHistoryDB. Previously a permission-denied or I/O failure on the history DB path returned the bare os.Stat error, which gave no hint about which operation or path failed. Wrap it with fmt.Errorf using %w so errors.Is/As still work while surfacing "stat history db " in the message, matching the friendly tone of the ErrNotExist branch. --- gpbackman/gpbckpconfig/utils_db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpbackman/gpbckpconfig/utils_db.go b/gpbackman/gpbckpconfig/utils_db.go index 07e307d9..6a834010 100644 --- a/gpbackman/gpbckpconfig/utils_db.go +++ b/gpbackman/gpbckpconfig/utils_db.go @@ -49,7 +49,7 @@ func OpenHistoryDB(historyDBPath string) (*sql.DB, error) { historyDBPath, ) } - return nil, err + return nil, fmt.Errorf("stat history db %q: %w", historyDBPath, err) } // mode=rw opens an existing database for read+write but never creates one. db, err := sql.Open("sqlite3", "file:"+historyDBPath+"?mode=rw") From 022be5c2d274b73a1bc2f7adcc7c4e052a3b27cb Mon Sep 17 00:00:00 2001 From: chenqiang Date: Wed, 29 Apr 2026 20:48:53 +0800 Subject: [PATCH 32/39] gpbackman: drop MASTER_DATA_DIRECTORY from history-db env-var fallback. Cloudberry switched from MASTER_DATA_DIRECTORY to COORDINATOR_DATA_DIRECTORY starting around release 1.5.4, and the standard environment scripts no longer export the legacy variable. Maintaining a fallback for it in gpbackman just adds surface area (constants, help text, docs, tests) for a value that should not exist in a current Cloudberry install. Drop MASTER_DATA_DIRECTORY entirely from the resolution chain, the flag help, the per-command long help strings, README/COMMANDS docs, and the corresponding test cases. The flag description is shortened in root.go to match the concise wording already used in README. The lingering "Cloudberry/Greenplum" comment in constants.go is updated to "Cloudberry" in the same pass. Resolution chain after this change: explicit --history-db -> $COORDINATOR_DATA_DIRECTORY (only when --auto-load-history-db is set) -> current working directory (default) Addresses review feedback from woblerr, tuhaihe, and MisterRaindrop on PR #97. The pre-existing MASTER_DATA_DIRECTORY reference in .github/workflows/cloudberry-backup-ci.yml predates this PR and is unrelated to history-db resolution; it is left for a separate cleanup. --- gpbackman/COMMANDS.md | 10 +++++----- gpbackman/README.md | 2 +- gpbackman/cmd/backup_clean.go | 2 +- gpbackman/cmd/backup_delete.go | 2 +- gpbackman/cmd/backup_info.go | 2 +- gpbackman/cmd/constants.go | 7 +++---- gpbackman/cmd/history_clean.go | 2 +- gpbackman/cmd/report_info.go | 2 +- gpbackman/cmd/root.go | 4 +--- gpbackman/cmd/wrappers.go | 9 ++++----- gpbackman/cmd/wrappers_test.go | 12 ------------ gpbackman/gpbckpconfig/utils_db.go | 2 +- 12 files changed, 20 insertions(+), 36 deletions(-) diff --git a/gpbackman/COMMANDS.md b/gpbackman/COMMANDS.md index 7cb15fd0..d9e43d47 100644 --- a/gpbackman/COMMANDS.md +++ b/gpbackman/COMMANDS.md @@ -72,7 +72,7 @@ For non local backups the following logic are applied: The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for in the current directory. Pass `--auto-load-history-db` to resolve it from `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`) instead. +If the --history-db option is not specified, the history database is looked for in the current directory. Pass `--auto-load-history-db` to resolve it from `$COORDINATOR_DATA_DIRECTORY` instead. Usage: gpbackman backup-clean [flags] @@ -158,7 +158,7 @@ For non local backups the following logic are applied: The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for in the current directory. Pass `--auto-load-history-db` to resolve it from `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`) instead. +If the --history-db option is not specified, the history database is looked for in the current directory. Pass `--auto-load-history-db` to resolve it from `$COORDINATOR_DATA_DIRECTORY` instead. Usage: gpbackman backup-delete [flags] @@ -256,7 +256,7 @@ To display the "object filtering details" column for all backups without using - The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for in the current directory. Pass `--auto-load-history-db` to resolve it from `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`) instead. +If the --history-db option is not specified, the history database is looked for in the current directory. Pass `--auto-load-history-db` to resolve it from `$COORDINATOR_DATA_DIRECTORY` instead. Usage: gpbackman backup-info [flags] @@ -455,7 +455,7 @@ Only --older-than-days or --before-timestamp option must be specified, not both. The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for in the current directory. Pass `--auto-load-history-db` to resolve it from `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`) instead. +If the --history-db option is not specified, the history database is looked for in the current directory. Pass `--auto-load-history-db` to resolve it from `$COORDINATOR_DATA_DIRECTORY` instead. Usage: gpbackman history-clean [flags] @@ -524,7 +524,7 @@ It is not necessary to use the --plugin-report-file-path flag for the following The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for in the current directory. Pass `--auto-load-history-db` to resolve it from `$COORDINATOR_DATA_DIRECTORY` (then `$MASTER_DATA_DIRECTORY`) instead. +If the --history-db option is not specified, the history database is looked for in the current directory. Pass `--auto-load-history-db` to resolve it from `$COORDINATOR_DATA_DIRECTORY` instead. Usage: gpbackman report-info [flags] diff --git a/gpbackman/README.md b/gpbackman/README.md index 930e570b..d9bd9ae8 100644 --- a/gpbackman/README.md +++ b/gpbackman/README.md @@ -53,7 +53,7 @@ Available Commands: Flags: -h, --help help for gpbackman - --auto-load-history-db resolve gpbackup_history.db from $COORDINATOR_DATA_DIRECTORY (or $MASTER_DATA_DIRECTORY) when --history-db is unset + --auto-load-history-db resolve gpbackup_history.db from $COORDINATOR_DATA_DIRECTORY when --history-db is unset --history-db string full path to the gpbackup_history.db file --log-file string full path to log file directory, if not specified, the log file will be created in the $HOME/gpAdminLogs directory --log-level-console string level for console logging (error, info, debug, verbose) (default "info") diff --git a/gpbackman/cmd/backup_clean.go b/gpbackman/cmd/backup_clean.go index eacd3418..46c9548b 100644 --- a/gpbackman/cmd/backup_clean.go +++ b/gpbackman/cmd/backup_clean.go @@ -77,7 +77,7 @@ For non local backups the following logic are applied: The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for in the current directory. To resolve it from $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY) instead, pass the --auto-load-history-db flag.`, +If the --history-db option is not specified, the history database is looked for in the current directory. To resolve it from $COORDINATOR_DATA_DIRECTORY instead, pass the --auto-load-history-db flag.`, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { doRootFlagValidation(cmd.Flags(), checkFileExistsConst) diff --git a/gpbackman/cmd/backup_delete.go b/gpbackman/cmd/backup_delete.go index 2d4b95f8..4eb525a5 100644 --- a/gpbackman/cmd/backup_delete.go +++ b/gpbackman/cmd/backup_delete.go @@ -84,7 +84,7 @@ For non local backups the following logic are applied: The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for in the current directory. To resolve it from $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY) instead, pass the --auto-load-history-db flag.`, +If the --history-db option is not specified, the history database is looked for in the current directory. To resolve it from $COORDINATOR_DATA_DIRECTORY instead, pass the --auto-load-history-db flag.`, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { doRootFlagValidation(cmd.Flags(), checkFileExistsConst) diff --git a/gpbackman/cmd/backup_info.go b/gpbackman/cmd/backup_info.go index adb5c118..96674b48 100644 --- a/gpbackman/cmd/backup_info.go +++ b/gpbackman/cmd/backup_info.go @@ -98,7 +98,7 @@ To display the "object filtering details" column for all backups without using - The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for in the current directory. To resolve it from $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY) instead, pass the --auto-load-history-db flag.`, +If the --history-db option is not specified, the history database is looked for in the current directory. To resolve it from $COORDINATOR_DATA_DIRECTORY instead, pass the --auto-load-history-db flag.`, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { doRootFlagValidation(cmd.Flags(), checkFileExistsConst) diff --git a/gpbackman/cmd/constants.go b/gpbackman/cmd/constants.go index 7b5b59ed..03b13d36 100644 --- a/gpbackman/cmd/constants.go +++ b/gpbackman/cmd/constants.go @@ -74,11 +74,10 @@ var ( // Timestamp to delete all backups after. afterTimestamp string - // historyDBEnvVars lists, in priority order, the environment variables - // inspected when --history-db is not supplied. They are exported by the - // standard Cloudberry/Greenplum cluster environment scripts. + // historyDBEnvVars lists the environment variables inspected when + // --history-db is not supplied. Exported by the standard Cloudberry + // cluster environment scripts. historyDBEnvVars = []string{ "COORDINATOR_DATA_DIRECTORY", - "MASTER_DATA_DIRECTORY", } ) diff --git a/gpbackman/cmd/history_clean.go b/gpbackman/cmd/history_clean.go index 361f4ee0..42366421 100644 --- a/gpbackman/cmd/history_clean.go +++ b/gpbackman/cmd/history_clean.go @@ -50,7 +50,7 @@ Only --older-than-days or --before-timestamp option must be specified, not both. The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for in the current directory. To resolve it from $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY) instead, pass the --auto-load-history-db flag.`, +If the --history-db option is not specified, the history database is looked for in the current directory. To resolve it from $COORDINATOR_DATA_DIRECTORY instead, pass the --auto-load-history-db flag.`, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { doRootFlagValidation(cmd.Flags(), checkFileExistsConst) diff --git a/gpbackman/cmd/report_info.go b/gpbackman/cmd/report_info.go index 042a3da5..36bf1207 100644 --- a/gpbackman/cmd/report_info.go +++ b/gpbackman/cmd/report_info.go @@ -78,7 +78,7 @@ It is not necessary to use the --plugin-report-file-path flag for the following The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. -If the --history-db option is not specified, the history database is looked for in the current directory. To resolve it from $COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY) instead, pass the --auto-load-history-db flag.`, +If the --history-db option is not specified, the history database is looked for in the current directory. To resolve it from $COORDINATOR_DATA_DIRECTORY instead, pass the --auto-load-history-db flag.`, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { doRootFlagValidation(cmd.Flags(), checkFileExistsConst) diff --git a/gpbackman/cmd/root.go b/gpbackman/cmd/root.go index 50884771..79156cea 100644 --- a/gpbackman/cmd/root.go +++ b/gpbackman/cmd/root.go @@ -57,9 +57,7 @@ func init() { &rootAutoLoadHistoryDB, autoLoadHistoryDBFlagName, false, - "when --history-db is unset, look up gpbackup_history.db under "+ - "$COORDINATOR_DATA_DIRECTORY (then $MASTER_DATA_DIRECTORY) before "+ - "falling back to the current directory", + "resolve gpbackup_history.db from $COORDINATOR_DATA_DIRECTORY when --history-db is unset", ) rootCmd.PersistentFlags().StringVar( &rootLogFile, diff --git a/gpbackman/cmd/wrappers.go b/gpbackman/cmd/wrappers.go index f809ce4c..c36a5256 100644 --- a/gpbackman/cmd/wrappers.go +++ b/gpbackman/cmd/wrappers.go @@ -84,11 +84,10 @@ func setLogLevelFile(level string) error { // getHistoryDBPath resolves the path to the gpbackup_history.db file. // An explicit --history-db value always wins. Otherwise, when the caller // asks for auto-load (--auto-load-history-db), look up the file under the -// COORDINATOR_DATA_DIRECTORY / MASTER_DATA_DIRECTORY environment variables -// exported by the standard Cloudberry environment scripts. As a final -// fallback, return the bare filename so it is resolved against the current -// working directory, preserving the original behaviour for the default -// invocation. +// COORDINATOR_DATA_DIRECTORY environment variable exported by the standard +// Cloudberry environment scripts. As a final fallback, return the bare +// filename so it is resolved against the current working directory, +// preserving the original behaviour for the default invocation. func getHistoryDBPath(historyDBPath string, autoLoad bool) string { if historyDBPath != "" { return historyDBPath diff --git a/gpbackman/cmd/wrappers_test.go b/gpbackman/cmd/wrappers_test.go index 401f2962..10c0d0c5 100644 --- a/gpbackman/cmd/wrappers_test.go +++ b/gpbackman/cmd/wrappers_test.go @@ -65,7 +65,6 @@ var _ = Describe("wrappers tests", func() { It("ignores env vars by default (auto-load off)", func() { os.Setenv("COORDINATOR_DATA_DIRECTORY", "/coord/data") - os.Setenv("MASTER_DATA_DIRECTORY", "/master/data") Expect(getHistoryDBPath("", false)).To(Equal(historyDBNameConst)) }) @@ -74,17 +73,6 @@ var _ = Describe("wrappers tests", func() { Expect(getHistoryDBPath("", true)).To(Equal(filepath.Join("/coord/data", historyDBNameConst))) }) - It("falls back to MASTER_DATA_DIRECTORY when COORDINATOR is unset and auto-load is on", func() { - os.Setenv("MASTER_DATA_DIRECTORY", "/master/data") - Expect(getHistoryDBPath("", true)).To(Equal(filepath.Join("/master/data", historyDBNameConst))) - }) - - It("prefers COORDINATOR over MASTER when both are set and auto-load is on", func() { - os.Setenv("COORDINATOR_DATA_DIRECTORY", "/coord/data") - os.Setenv("MASTER_DATA_DIRECTORY", "/master/data") - Expect(getHistoryDBPath("", true)).To(Equal(filepath.Join("/coord/data", historyDBNameConst))) - }) - It("returns the cwd default when auto-load is on but no env vars are set", func() { Expect(getHistoryDBPath("", true)).To(Equal(historyDBNameConst)) }) diff --git a/gpbackman/gpbckpconfig/utils_db.go b/gpbackman/gpbckpconfig/utils_db.go index 6a834010..48cc28d7 100644 --- a/gpbackman/gpbckpconfig/utils_db.go +++ b/gpbackman/gpbckpconfig/utils_db.go @@ -45,7 +45,7 @@ func OpenHistoryDB(historyDBPath string) (*sql.DB, error) { "Specify the path via --history-db, run gpbackman from "+ "the directory that contains gpbackup_history.db, or "+ "pass --auto-load-history-db to resolve it from "+ - "$COORDINATOR_DATA_DIRECTORY (or $MASTER_DATA_DIRECTORY)", + "$COORDINATOR_DATA_DIRECTORY", historyDBPath, ) } From 8fd4287ac0bc171cd58bc080605dd128aa60cd3f Mon Sep 17 00:00:00 2001 From: Anton Kurochkin <45575813+woblerr@users.noreply.github.com> Date: Tue, 12 May 2026 08:33:06 +0300 Subject: [PATCH 33/39] =?UTF-8?q?Add=20gpbackup=5Fexporter=20=E2=80=94=20a?= =?UTF-8?q?=20Prometheus=20exporter=20for=20gpbackup=20history=20database.?= =?UTF-8?q?=20(#87)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Port gpbackup_exporter to cloudberry-backup. Introduce the Prometheus metrics exporter as a new standalone binary within the repository. - Create gpbackup_exporter.go entry point with CLI flags and collection loop. - Port core exporter logic to the new `exporter/` package. - Adapt type system to use `*history.BackupConfig` directly, aligning with the cloudberry-backup architecture. - Switch receiver methods to standalone gpbckpconfig helpers. - Add Prometheus and Kingpin dependencies to `go.mod`. - Update Makefile to support building, testing, and packaging. - Port unit tests to Ginkgo/Gomega. * Add end-to-end tests for gpbackup_exporter. Add e2e test that runs gpbackup commands and validates that gpbackup_exporter correctly reads history database and exposes metrics. Also remove dead gpbackmanPath assignment from useOldBackupVersion in e2e tests. And suppress noisy test logger output by writing to bytes.Buffer instead of os.Stdout in exporter unit tests. * Add README for gpbackup_exporter. * Add additional tools section to README. * Add smoke tests for gpbackup_exporter in CI. * Fix smoke test for gpbackup_exporter in install target. * Graceful shutdown with exit status 0. SIGINT means "interrupt," signaling a user wants to stop the current operation. SIGTERM means "terminate," requesting a polite program shutdown. So' it's correct to consider as graceful shutdown * Fix port in test. * Prevent panic and make exit. To prevent a panic with an error like "panic: http: invalid pattern", exit in case of an empty endpoint. * Remove unused getDataSuccessStatus and simplify GetGPBackupInfo. Remove unused getDataSuccessStatus. Clarify include/exclude handling: * DB present in both dbInclude and dbExclude - warn and emit gpbackup_exporter_status=0. * DB only in dbExclude - skip, emit no metrics. * dbInclude empty or DB in dbInclude - process and emit full backup metrics. Add unit tests. * Ensure command starts successfully in gpbackup_exporter end-to-end tests. * Update gpbackup_exporter_status metric description. * Fix expected error message in exporter test for missing history DB. This aligns the test with the recent behavior change in commit 1035fb84. `OpenHistoryDB()` now fails fast on missing files during the open stage, rather than failing later during the first SQL query. --- .github/workflows/build_and_unit_test.yml | 3 + .github/workflows/cloudberry-backup-ci.yml | 3 + Makefile | 16 +- README.md | 8 + end_to_end/end_to_end_suite_test.go | 3 +- end_to_end/exporter_test.go | 143 ++++++++++ exporter/README.md | 122 +++++++++ exporter/exporter_suite_test.go | 32 +++ exporter/gpbckp_backup_metrics.go | 177 ++++++++++++ exporter/gpbckp_backup_metrics_test.go | 109 ++++++++ exporter/gpbckp_converters.go | 46 ++++ exporter/gpbckp_converters_test.go | 56 ++++ exporter/gpbckp_exporter.go | 186 +++++++++++++ exporter/gpbckp_exporter_metrics.go | 52 ++++ exporter/gpbckp_exporter_test.go | 286 ++++++++++++++++++++ exporter/gpbckp_last_backup_metrics.go | 59 ++++ exporter/gpbckp_last_backup_metrics_test.go | 96 +++++++ exporter/gpbckp_parser.go | 170 ++++++++++++ exporter/gpbckp_parser_test.go | 235 ++++++++++++++++ go.mod | 40 ++- go.sum | 103 +++++-- gpbackup_exporter.go | 172 ++++++++++++ 22 files changed, 2083 insertions(+), 34 deletions(-) create mode 100644 end_to_end/exporter_test.go create mode 100644 exporter/README.md create mode 100644 exporter/exporter_suite_test.go create mode 100644 exporter/gpbckp_backup_metrics.go create mode 100644 exporter/gpbckp_backup_metrics_test.go create mode 100644 exporter/gpbckp_converters.go create mode 100644 exporter/gpbckp_converters_test.go create mode 100644 exporter/gpbckp_exporter.go create mode 100644 exporter/gpbckp_exporter_metrics.go create mode 100644 exporter/gpbckp_exporter_test.go create mode 100644 exporter/gpbckp_last_backup_metrics.go create mode 100644 exporter/gpbckp_last_backup_metrics_test.go create mode 100644 exporter/gpbckp_parser.go create mode 100644 exporter/gpbckp_parser_test.go create mode 100644 gpbackup_exporter.go diff --git a/.github/workflows/build_and_unit_test.yml b/.github/workflows/build_and_unit_test.yml index cb50de4f..ae140f9f 100644 --- a/.github/workflows/build_and_unit_test.yml +++ b/.github/workflows/build_and_unit_test.yml @@ -58,6 +58,9 @@ jobs: echo "=== Testing gpbackman ===" ${GOPATH}/bin/gpbackman --version ${GOPATH}/bin/gpbackman --help > /dev/null + echo "=== Testing gpbackup_exporter ===" + ${GOPATH}/bin/gpbackup_exporter --version + ${GOPATH}/bin/gpbackup_exporter --help > /dev/null echo "=== All smoke tests passed ===" - name: Unit Test diff --git a/.github/workflows/cloudberry-backup-ci.yml b/.github/workflows/cloudberry-backup-ci.yml index 05068ff8..5753ec66 100644 --- a/.github/workflows/cloudberry-backup-ci.yml +++ b/.github/workflows/cloudberry-backup-ci.yml @@ -334,6 +334,9 @@ jobs: echo "=== Testing gpbackman ===" ${GPHOME}/bin/gpbackman --version ${GPHOME}/bin/gpbackman --help > /dev/null + echo "=== Testing gpbackup_exporter ===" + ${GPHOME}/bin/gpbackup_exporter --version + ${GPHOME}/bin/gpbackup_exporter --help > /dev/null echo "=== All smoke tests passed ===" | tee "${TEST_LOG_ROOT}/cloudberry-backup-smoke.log" ;; unit) diff --git a/Makefile b/Makefile index 12b5fd44..fa27092c 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ RESTORE=gprestore HELPER=gpbackup_helper S3PLUGIN=gpbackup_s3_plugin GPBACKMAN=gpbackman +EXPORTER=gpbackup_exporter BIN_DIR=$(shell echo $${GOPATH:-~/go} | awk -F':' '{ print $$1 "/bin"}') GINKGO_FLAGS := -r --keep-going --randomize-suites --randomize-all --no-color GIT_VERSION := $(shell v=$$(git describe --tags 2>/dev/null); if [ -n "$$v" ]; then echo $$v | perl -pe 's/(.*)-([0-9]*)-(g[0-9a-f]*)/\1+dev.\2.\3/'; else cat VERSION 2>/dev/null || echo "dev"; fi) @@ -18,9 +19,13 @@ RESTORE_VERSION_STR=github.com/apache/cloudberry-backup/restore.version=$(GIT_VE HELPER_VERSION_STR=github.com/apache/cloudberry-backup/helper.version=$(GIT_VERSION) S3PLUGIN_VERSION_STR=github.com/apache/cloudberry-backup/plugins/s3plugin.version=$(GIT_VERSION) GPBACKMAN_VERSION_STR=github.com/apache/cloudberry-backup/gpbackman/cmd.version=$(GIT_VERSION) +GIT_REVISION := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown") +GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown") +BUILD_DATE := $(shell date -u '+%Y%m%d-%H:%M:%S') +EXPORTER_VERSION_STR=-X github.com/prometheus/common/version.Version=$(GIT_VERSION) -X github.com/prometheus/common/version.Revision=$(GIT_REVISION) -X github.com/prometheus/common/version.Branch=$(GIT_BRANCH) -X github.com/prometheus/common/version.BuildDate=$(BUILD_DATE) # note that /testutils is not a production directory, but has unit tests to validate testing tools -SUBDIRS_HAS_UNIT=backup/ filepath/ history/ helper/ options/ report/ restore/ toc/ utils/ testutils/ plugins/s3plugin/ gpbackman/cmd/ gpbackman/gpbckpconfig/ gpbackman/textmsg/ +SUBDIRS_HAS_UNIT=backup/ filepath/ history/ helper/ options/ report/ restore/ toc/ utils/ testutils/ plugins/s3plugin/ gpbackman/cmd/ gpbackman/gpbckpconfig/ gpbackman/textmsg/ exporter/ SUBDIRS_ALL=$(SUBDIRS_HAS_UNIT) integration/ end_to_end/ GOLANG_LINTER=$(GOPATH)/bin/golangci-lint GINKGO=$(GOPATH)/bin/ginkgo @@ -90,6 +95,7 @@ build : $(GOSQLITE) CGO_ENABLED=1 $(GO_BUILD) -tags '$(HELPER)' -o $(BIN_DIR)/$(HELPER) --ldflags '-X $(HELPER_VERSION_STR)' CGO_ENABLED=1 $(GO_BUILD) -tags '$(S3PLUGIN)' -o $(BIN_DIR)/$(S3PLUGIN) --ldflags '-X $(S3PLUGIN_VERSION_STR)' CGO_ENABLED=1 $(GO_BUILD) -tags '$(GPBACKMAN)' -o $(BIN_DIR)/$(GPBACKMAN) --ldflags '-X $(GPBACKMAN_VERSION_STR)' + CGO_ENABLED=1 $(GO_BUILD) -tags '$(EXPORTER)' -o $(BIN_DIR)/$(EXPORTER) -ldflags "$(EXPORTER_VERSION_STR)" debug : CGO_ENABLED=1 $(GO_BUILD) -tags '$(BACKUP)' -o $(BIN_DIR)/$(BACKUP) -ldflags "-X $(BACKUP_VERSION_STR)" $(DEBUG) @@ -97,6 +103,7 @@ debug : CGO_ENABLED=1 $(GO_BUILD) -tags '$(HELPER)' -o $(BIN_DIR)/$(HELPER) -ldflags "-X $(HELPER_VERSION_STR)" $(DEBUG) CGO_ENABLED=1 $(GO_BUILD) -tags '$(S3PLUGIN)' -o $(BIN_DIR)/$(S3PLUGIN) -ldflags "-X $(S3PLUGIN_VERSION_STR)" $(DEBUG) CGO_ENABLED=1 $(GO_BUILD) -tags '$(GPBACKMAN)' -o $(BIN_DIR)/$(GPBACKMAN) -ldflags "-X $(GPBACKMAN_VERSION_STR)" $(DEBUG) + CGO_ENABLED=1 $(GO_BUILD) -tags '$(EXPORTER)' -o $(BIN_DIR)/$(EXPORTER) -ldflags "$(EXPORTER_VERSION_STR)" $(DEBUG) build_linux : env GOOS=linux GOARCH=amd64 $(GO_BUILD) -tags '$(BACKUP)' -o $(BACKUP) -ldflags "-X $(BACKUP_VERSION_STR)" @@ -104,9 +111,10 @@ build_linux : env GOOS=linux GOARCH=amd64 $(GO_BUILD) -tags '$(HELPER)' -o $(HELPER) -ldflags "-X $(HELPER_VERSION_STR)" env GOOS=linux GOARCH=amd64 $(GO_BUILD) -tags '$(S3PLUGIN)' -o $(S3PLUGIN) -ldflags "-X $(S3PLUGIN_VERSION_STR)" env GOOS=linux GOARCH=amd64 $(GO_BUILD) -tags '$(GPBACKMAN)' -o $(GPBACKMAN) -ldflags "-X $(GPBACKMAN_VERSION_STR)" + env GOOS=linux GOARCH=amd64 $(GO_BUILD) -tags '$(EXPORTER)' -o $(EXPORTER) -ldflags "$(EXPORTER_VERSION_STR)" install : - cp $(BIN_DIR)/$(BACKUP) $(BIN_DIR)/$(RESTORE) $(BIN_DIR)/$(GPBACKMAN) $(GPHOME)/bin + cp $(BIN_DIR)/$(BACKUP) $(BIN_DIR)/$(RESTORE) $(BIN_DIR)/$(GPBACKMAN) $(BIN_DIR)/$(EXPORTER) $(GPHOME)/bin @psql -X -t -d template1 -c 'select distinct hostname from gp_segment_configuration where content != -1' > /tmp/seg_hosts 2>/dev/null; \ if [ $$? -eq 0 ]; then \ $(COPYUTIL) -f /tmp/seg_hosts $(helper_path) $(s3plugin_path) =:$(GPHOME)/bin/; \ @@ -124,7 +132,7 @@ install : clean : # Build artifacts - rm -f $(BIN_DIR)/$(BACKUP) $(BACKUP) $(BIN_DIR)/$(RESTORE) $(RESTORE) $(BIN_DIR)/$(HELPER) $(HELPER) $(BIN_DIR)/$(S3PLUGIN) $(S3PLUGIN) $(BIN_DIR)/$(GPBACKMAN) $(GPBACKMAN) + rm -f $(BIN_DIR)/$(BACKUP) $(BACKUP) $(BIN_DIR)/$(RESTORE) $(RESTORE) $(BIN_DIR)/$(HELPER) $(HELPER) $(BIN_DIR)/$(S3PLUGIN) $(S3PLUGIN) $(BIN_DIR)/$(GPBACKMAN) $(GPBACKMAN) $(BIN_DIR)/$(EXPORTER) $(EXPORTER) # Test artifacts rm -rf /tmp/go-build* /tmp/gexec_artifacts* /tmp/ginkgo* docker stop s3-minio # stop minio before removing its data directories @@ -184,6 +192,7 @@ package: @GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=$(CGO) go build -tags '$(HELPER)' -o $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/bin/$(HELPER) --ldflags '-X $(HELPER_VERSION_STR)' @GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=$(CGO) go build -tags '$(S3PLUGIN)' -o $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/bin/$(S3PLUGIN) --ldflags '-X $(S3PLUGIN_VERSION_STR)' @GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=$(CGO) go build -tags '$(GPBACKMAN)' -o $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/bin/$(GPBACKMAN) --ldflags '-X $(GPBACKMAN_VERSION_STR)' + @GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=$(CGO) go build -tags '$(EXPORTER)' -o $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/bin/$(EXPORTER) -ldflags "$(EXPORTER_VERSION_STR)" @echo "Copying Apache compliance files..." @cp LICENSE $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/ @cp NOTICE $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/ @@ -214,6 +223,7 @@ package: @echo 'sudo chmod 755 "$${INSTALL_DIR}/bin/$(HELPER)"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh @echo 'sudo chmod 755 "$${INSTALL_DIR}/bin/$(S3PLUGIN)"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh @echo 'sudo chmod 755 "$${INSTALL_DIR}/bin/$(GPBACKMAN)"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh + @echo 'sudo chmod 755 "$${INSTALL_DIR}/bin/$(EXPORTER)"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh @echo '' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh @echo 'echo "Installation complete!"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh @echo 'echo "$(PACKAGE_NAME) binaries installed to $${INSTALL_DIR}/bin/"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh diff --git a/README.md b/README.md index 546bd802..d41b4b26 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,14 @@ gprestore --timestamp Run `--help` with either command for a complete list of options. +## Additional tools + +This repository also includes the following tools: + +* [gpbackup_s3_plugin](./plugins/s3plugin/README.md) — S3 storage plugin for gpbackup and gprestore. +* [gpBackMan](./gpbackman/README.md) — utility for managing backups created by gpbackup. +* [gpbackup_exporter](./exporter/README.md) — Prometheus exporter for collecting metrics from gpbackup history database. + ## Validation and code quality ### Test setup diff --git a/end_to_end/end_to_end_suite_test.go b/end_to_end/end_to_end_suite_test.go index a9207097..a21a5977 100644 --- a/end_to_end/end_to_end_suite_test.go +++ b/end_to_end/end_to_end_suite_test.go @@ -71,6 +71,7 @@ var ( backupDir string segmentCount int gpbackmanPath string + exporterPath string ) const ( @@ -558,7 +559,6 @@ options: oldBackupVersionStr := os.Getenv("OLD_BACKUP_VERSION") _, restoreHelperPath, gprestorePath = buildAndInstallBinaries() - gpbackmanPath = fmt.Sprintf("%s/go/bin/gpbackman", operating.System.Getenv("HOME")) // Precompiled binaries will exist when running the ci job, `backward-compatibility` if _, err := os.Stat(fmt.Sprintf("/tmp/%s", oldBackupVersionStr)); err == nil { @@ -580,6 +580,7 @@ options: backupHelperPath = fmt.Sprintf("%s/gpbackup_helper", binDir) restoreHelperPath = backupHelperPath gpbackmanPath = fmt.Sprintf("%s/gpbackman", binDir) + exporterPath = fmt.Sprintf("%s/gpbackup_exporter", binDir) } segConfig := cluster.MustGetSegmentConfiguration(backupConn) backupCluster = cluster.NewCluster(segConfig) diff --git a/end_to_end/exporter_test.go b/end_to_end/exporter_test.go new file mode 100644 index 00000000..37ecc761 --- /dev/null +++ b/end_to_end/exporter_test.go @@ -0,0 +1,143 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package end_to_end_test + +import ( + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "regexp" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func getFreeTCPPort() int { + listener, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + port := listener.Addr().(*net.TCPAddr).Port + listener.Close() + return port +} + +type exporterMetricCheck struct { + pattern string + count int +} + +func validateExporterMetrics(body string, checks []exporterMetricCheck) { + lines := strings.Split(body, "\n") + for _, em := range checks { + re := regexp.MustCompile(em.pattern) + count := 0 + for _, line := range lines { + if re.MatchString(line) { + count++ + } + } + Expect(count).To(Equal(em.count), + fmt.Sprintf("metric pattern %q: got %d, want %d", em.pattern, count, em.count)) + } +} + +func fetchMetrics(url string, client *http.Client) (string, error) { + if client == nil { + client = &http.Client{Timeout: 5 * time.Second} + } + resp, err := client.Get(url) + if err != nil { + return "", err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + if resp.StatusCode != http.StatusOK { + return string(body), fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + return string(body), nil +} + +var _ = Describe("gpbackup_exporter end to end tests", func() { + BeforeEach(func() { + if useOldBackupVersion { + Skip("exporter tests are not applicable in old backup version mode") + } + if _, err := os.Stat(exporterPath); os.IsNotExist(err) { + Skip("gpbackup_exporter binary not found, skipping exporter e2e tests") + } + }) + + It("collects and exposes metrics without web config", func() { + expectedMetrics := []exporterMetricCheck{ + {`^gpbackup_backup_deletion_status\{`, 3}, + {`^gpbackup_backup_duration_seconds\{`, 3}, + {`^gpbackup_backup_info\{`, 3}, + {`^gpbackup_backup_status\{`, 3}, + {`^gpbackup_backup_status\{.*\} 0$`, 3}, + {`^gpbackup_backup_duration_seconds\{.*object_filtering="none",plugin="none".*\}`, 3}, + {`^gpbackup_backup_info\{.*database_name="testdb".*\}`, 3}, + {`^gpbackup_backup_since_last_completion_seconds\{`, 3}, + {`^gpbackup_backup_since_last_completion_seconds\{.*backup_type="full",database_name="testdb".*\}`, 1}, + {`^gpbackup_backup_since_last_completion_seconds\{.*backup_type="incremental",database_name="testdb".*\}`, 1}, + {`^gpbackup_backup_since_last_completion_seconds\{.*backup_type="metadata-only",database_name="testdb".*\}`, 1}, + {`^gpbackup_exporter_status\{database_name="testdb"\} 1$`, 1}, + {`^gpbackup_exporter_build_info\{.*\} 1$`, 1}, + } + end_to_end_setup() + defer end_to_end_teardown() + historyDB := getHistoryDBPathForCluster() + gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir, + "--leaf-partition-data") + gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir, + "--incremental", + "--leaf-partition-data") + gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir, + "--metadata-only") + port := getFreeTCPPort() + cmd := exec.Command(exporterPath, + "--gpbackup.history-file", historyDB, + "--collect.interval", "600", + "--web.listen-address", fmt.Sprintf("127.0.0.1:%d", port), + ) + Expect(cmd.Start()).To(Succeed()) + defer cmd.Process.Kill() + metricsURL := fmt.Sprintf("http://127.0.0.1:%d/metrics", port) + var body string + Eventually(func() string { + b, err := fetchMetrics(metricsURL, nil) + if err != nil { + return "" + } + body = b + return b + }, 10*time.Second, 500*time.Millisecond).Should(ContainSubstring("gpbackup_backup_status")) + validateExporterMetrics(body, expectedMetrics) + }) +}) diff --git a/exporter/README.md b/exporter/README.md new file mode 100644 index 00000000..640cb192 --- /dev/null +++ b/exporter/README.md @@ -0,0 +1,122 @@ + + +# gpbackup_exporter + +**gpbackup_exporter** is a Prometheus exporter for collecting metrics from gpbackup history database (`gpbackup_history.db`). + +## Metrics +### Backup metrics +| Metric | Description | Labels | Additional Info | +| ----------- | ------------------ | ------------- | --------------- | +| `gpbackup_backup_status` | backup status | backup_type, database_name, object_filtering, plugin, timestamp | Values description:
`0` - success,
`1` - failure.| +| `gpbackup_backup_deletion_status` | backup deletion status | backup_type, database_name, date_deleted, object_filtering, plugin, timestamp | Values description:
`0` - backup still exists,
`1` - backup was successfully deleted,
`2` - the deletion is in progress,
`3` - last delete attempt failed to delete backup from plugin storage,
`4` - last delete attempt failed to delete backup from local storage.| +| `gpbackup_backup_info` | backup info | backup_dir, backup_ver, backup_type, compression_type, database_name, database_ver, object_filtering, plugin, plugin_ver, timestamp, with_statistic | Values description:
`1` - info about backup is exist.| +| `gpbackup_backup_duration_seconds` | backup duration in seconds| backup_type, database_name, end_time, object_filtering, plugin, timestamp || + +### Last backup metrics +| Metric | Description | Labels | Additional Info | +| ----------- | ------------------ | ------------- | --------------- | +| `gpbackup_backup_since_last_completion_seconds`| seconds since the last completed backup | backup_type, database_name || + +### Exporter metrics + +| Metric | Description | Labels | Additional Info | +| ----------- | ------------------ | ------------- | --------------- | +| `gpbackup_exporter_build_info` | information about gpbackup exporter | branch, goarch, goos, goversion, revision, tags, version | | +| `gpbackup_exporter_status` | gpbackup exporter data collection status for a specific database | database_name | Values description:
`0` — exporter marked this database as not collected,
`1` — information successfully fetched from history database.| + +## Getting Started +Available configuration flags: + +```bash +./gpbackup_exporter --help +usage: gpbackup_exporter [] + + +Flags: + -h, --[no-]help Show context-sensitive help (also try --help-long and --help-man). + --web.telemetry-path="/metrics" + Path under which to expose metrics. + --web.listen-address=:19854 ... + Addresses on which to expose metrics and web interface. Repeatable for multiple addresses. Examples: `:9100` or `[::1]:9100` for http, `vsock://:9100` for vsock + --web.config.file="" Path to configuration file that can enable TLS or authentication. See: https://github.com/prometheus/exporter-toolkit/blob/master/docs/web-configuration.md + --collect.interval=600 Collecting metrics interval in seconds. + --collect.depth=0 Metrics depth collection in days. Metrics for backup older than this interval will not be collected. 0 - disable. + --gpbackup.history-file="" + Path to gpbackup_history.db. + --gpbackup.db-include="" ... + Specific db for collecting metrics. Can be specified several times. + --gpbackup.db-exclude="" ... + Specific db to exclude from collecting metrics. Can be specified several times. + --gpbackup.backup-type="" Specific backup type for collecting metrics. One of: [full, incremental, data-only, metadata-only]. + --[no-]gpbackup.collect-deleted + Collecting metrics for deleted backups. + --[no-]gpbackup.collect-failed + Collecting metrics for failed backups. + --log.level=info Only log messages with the given severity or above. One of: [debug, info, warn, error] + --log.format=logfmt Output format of log messages. One of: [logfmt, json] + --[no-]version Show application version. +``` + +### Additional description of flags. + +It's necessary to specify the `gpbackup_history.db` file location via `--gpbackup.history-file` flag. + +By default, metrics a collected only for active backups. The flag `--gpbackup.collect-deleted` allows to collect metrics for deleted backups. The flag `--gpbackup.collect-failed` allows to collect metrics for failed backups. + +Custom database for collecting metrics can be specified via `--gpbackup.db-include` flag. You can specify several databases.
+For example, `--gpbackup.db-include=demo1 --gpbackup.db-include=demo2`.
+For this case, metrics will be collected only for `demo1` and `demo2` databases. + +Custom database to exclude from collecting metrics can be specified via `--gpbackup.db-exclude` flag. You can specify several databases.
+For example, `--gpbackup.db-exclude=demo1 --gpbackup.db-exclude=demo2`.
+For this case, metrics **will not be collected** for `demo1` and `demo2` databases.
+If the same database is specified for include and exclude flags, then metrics for this database will not be collected. +The flag `--gpbackup.db-exclude` has a higher priority.
+For example, `--gpbackup.db-include=demo1 --gpbackup.db-exclude=demo1`.
+For this case, metrics **will not be collected** for `demo1` database. + +Custom `backup type` for collecting metrics can be specified via `--gpbackup.backup-type` flag. Valid values: `full`, `incremental`, `data-only`, `metadata-only`.
+For example, `--gpbackup.backup-type=full`.
+For this case, metrics will be collected only for `full` backups.
+ +Custom metrics depth collection in days can be specified via `--collect.depth` flag. Since gpbackup doesn't have regular options for removing info about outdated backups from history file, it is possible to limit the depth of collection metrics.
+For example, `--collect.depth=14`.
+For this case, metrics will be collected for backups not older then 14 days from current time.
+Value `0` - disable this functionality. + +When `--log.level=debug` is specified - information of values and labels for metrics is printing to the log. + +The flag `--web.config.file` allows to specify the path to the configuration for TLS and/or basic authentication. + +## Running + +```bash +./gpbackup_exporter \ + --gpbackup.history-file=/data/master/gpseg-1/gpbackup_history.db \ + --gpbackup.collect-deleted \ + --gpbackup.collect-failed +``` + +After starting, metrics are available at `http://localhost:19854/metrics`. + +## About + +gpbackup_exporter is part of the Apache Cloudberry Backup (Incubating) toolset. It is based on the original [gpbackup_exporter](https://github.com/woblerr/gpbackup_exporter) project. diff --git a/exporter/exporter_suite_test.go b/exporter/exporter_suite_test.go new file mode 100644 index 00000000..1b90b423 --- /dev/null +++ b/exporter/exporter_suite_test.go @@ -0,0 +1,32 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package exporter + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestExporter(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Exporter Suite") +} diff --git a/exporter/gpbckp_backup_metrics.go b/exporter/gpbckp_backup_metrics.go new file mode 100644 index 00000000..f0439cd0 --- /dev/null +++ b/exporter/gpbckp_backup_metrics.go @@ -0,0 +1,177 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package exporter + +import ( + "log/slog" + "strconv" + + "github.com/apache/cloudberry-backup/gpbackman/gpbckpconfig" + "github.com/apache/cloudberry-backup/history" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + gpbckpBackupStatusMetric = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "gpbackup_backup_status", + Help: "Backup status.", + }, + []string{ + "backup_type", + "database_name", + "object_filtering", + "plugin", + "timestamp"}) + gpbckpBackupDataDeletedStatusMetric = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "gpbackup_backup_deletion_status", + Help: "Backup deletion status.", + }, + []string{ + "backup_type", + "database_name", + "date_deleted", + "object_filtering", + "plugin", + "timestamp"}) + gpbckpBackupInfoMetric = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "gpbackup_backup_info", + Help: "Backup info.", + }, + []string{ + "backup_dir", + "backup_ver", + "backup_type", + "compression_type", + "database_name", + "database_ver", + "object_filtering", + "plugin", + "plugin_ver", + "timestamp", + "with_statistic"}) + gpbckpBackupDurationMetric = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "gpbackup_backup_duration_seconds", + Help: "Backup duration.", + }, + []string{ + "backup_type", + "database_name", + "end_time", + "object_filtering", + "plugin", + "timestamp"}) +) + +// Set backup metrics: +// - gpbackup_backup_status +// - gpbackup_backup_deletion_status +// - gpbackup_backup_info +// - gpbackup_backup_duration_seconds +func getBackupMetrics(backupData *history.BackupConfig, setUpMetricValueFun setUpMetricValueFunType, logger *slog.Logger) { + var ( + bckpDuration float64 + err error + ) + bckpType, err := gpbckpconfig.GetBackupType(backupData) + if err != nil { + logger.Error("Parse backup type value failed", "err", err) + } + backpObjectFiltering, err := gpbckpconfig.GetObjectFilteringInfo(backupData) + if err != nil { + logger.Error("Parse object filtering value failed", "err", err) + } + bckpDuration, err = gpbckpconfig.GetBackupDuration(backupData) + if err != nil { + logger.Error( + "Failed to parse dates to calculate duration", + "err", err, + ) + } + bckpDateDeleted, bckpDeletedStatus := getDeletedStatusCode(backupData.DateDeleted) + // Backup status. + setUpMetric( + gpbckpBackupStatusMetric, + "gpbackup_backup_status", + convertStatusFloat64(backupData.Status), + setUpMetricValueFun, + logger, + bckpType, + backupData.DatabaseName, + convertEmptyLabel(backpObjectFiltering), + convertEmptyLabel(backupData.Plugin), + backupData.Timestamp, + ) + // Backup deletion status. + setUpMetric( + gpbckpBackupDataDeletedStatusMetric, + "gpbackup_backup_deletion_status", + bckpDeletedStatus, + setUpMetricValueFun, + logger, + bckpType, + backupData.DatabaseName, + bckpDateDeleted, + convertEmptyLabel(backpObjectFiltering), + convertEmptyLabel(backupData.Plugin), + backupData.Timestamp, + ) + // Backup info. + setUpMetric( + gpbckpBackupInfoMetric, + "gpbackup_backup_info", + 1, + setUpMetricValueFun, + logger, + convertEmptyLabel(backupData.BackupDir), + backupData.BackupVersion, + bckpType, + convertEmptyLabel(backupData.CompressionType), + backupData.DatabaseName, + backupData.DatabaseVersion, + convertEmptyLabel(backpObjectFiltering), + convertEmptyLabel(backupData.Plugin), + convertEmptyLabel(backupData.PluginVersion), + backupData.Timestamp, + strconv.FormatBool(backupData.WithStatistics), + ) + // Backup duration. + setUpMetric( + gpbckpBackupDurationMetric, + "gpbackup_backup_duration_seconds", + bckpDuration, + setUpMetricValueFun, + logger, + bckpType, + backupData.DatabaseName, + // End time may be not set, if backup in progress. + convertEmptyLabel(backupData.EndTime), + convertEmptyLabel(backpObjectFiltering), + convertEmptyLabel(backupData.Plugin), + backupData.Timestamp, + ) +} + +func resetBackupMetrics() { + gpbckpBackupStatusMetric.Reset() + gpbckpBackupDataDeletedStatusMetric.Reset() + gpbckpBackupInfoMetric.Reset() + gpbckpBackupDurationMetric.Reset() +} diff --git a/exporter/gpbckp_backup_metrics_test.go b/exporter/gpbckp_backup_metrics_test.go new file mode 100644 index 00000000..17ca87d1 --- /dev/null +++ b/exporter/gpbckp_backup_metrics_test.go @@ -0,0 +1,109 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package exporter + +import ( + "bytes" + "fmt" + "log/slog" + "strings" + + "github.com/apache/cloudberry-backup/history" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/expfmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("BackupMetrics", func() { + // All metrics exist and all labels are corrected. + // gpbackup version >= 1.23.0 + Describe("getBackupMetrics", func() { + It("sets all backup metrics correctly", func() { + resetBackupMetrics() + getBackupMetrics(templateBackupConfig(), setUpMetricValue, getLogger()) + reg := prometheus.NewRegistry() + reg.MustRegister( + gpbckpBackupStatusMetric, + gpbckpBackupDataDeletedStatusMetric, + gpbckpBackupInfoMetric, + gpbckpBackupDurationMetric, + ) + metricFamily, err := reg.Gather() + if err != nil { + fmt.Println(err) + } + out := &bytes.Buffer{} + for _, mf := range metricFamily { + if _, err := expfmt.MetricFamilyToText(out, mf); err != nil { + panic(err) + } + } + templateMetrics := `# HELP gpbackup_backup_deletion_status Backup deletion status. +# TYPE gpbackup_backup_deletion_status gauge +gpbackup_backup_deletion_status{backup_type="full",database_name="test",date_deleted="none",object_filtering="none",plugin="none",timestamp="20230118152654"} 0 +# HELP gpbackup_backup_duration_seconds Backup duration. +# TYPE gpbackup_backup_duration_seconds gauge +gpbackup_backup_duration_seconds{backup_type="full",database_name="test",end_time="20230118152656",object_filtering="none",plugin="none",timestamp="20230118152654"} 2 +# HELP gpbackup_backup_info Backup info. +# TYPE gpbackup_backup_info gauge +gpbackup_backup_info{backup_dir="/data/backups",backup_type="full",backup_ver="1.30.5",compression_type="gzip",database_name="test",database_ver="6.23.0",object_filtering="none",plugin="none",plugin_ver="none",timestamp="20230118152654",with_statistic="false"} 1 +# HELP gpbackup_backup_status Backup status. +# TYPE gpbackup_backup_status gauge +gpbackup_backup_status{backup_type="full",database_name="test",object_filtering="none",plugin="none",timestamp="20230118152654"} 0 +` + Expect(out.String()).To(Equal(templateMetrics)) + }) + }) + + Describe("getBackupMetrics errors and debugs", func() { + DescribeTable("counts errors and debugs correctly", + func(backupData *history.BackupConfig, errorsCount, debugsCount int) { + resetBackupMetrics() + out := &bytes.Buffer{} + lc := slog.New(slog.NewTextHandler(out, &slog.HandlerOptions{Level: slog.LevelDebug})) + getBackupMetrics(backupData, fakeSetUpMetricValue, lc) + errorsOutputCount := strings.Count(out.String(), "level=ERROR") + debugsOutputCount := strings.Count(out.String(), "level=DEBUG") + Expect(errorsOutputCount).To(Equal(errorsCount)) + Expect(debugsOutputCount).To(Equal(debugsCount)) + }, + Entry("GetBackupMetricsErrorGetDurationGood", + templateBackupConfig(), + 4, 4, + ), + Entry("GetBackupMetricsErrorGetDurationError", + &history.BackupConfig{}, + 5, 4, + ), + Entry("GetBackupMetricsErrorGetBackupTypeAndObjectFilteringError", + // Fake example for testing. + &history.BackupConfig{ + DataOnly: true, + Incremental: true, + IncludeSchemaFiltered: true, + ExcludeSchemaFiltered: true, + }, + 7, 4, + ), + ) + }) +}) diff --git a/exporter/gpbckp_converters.go b/exporter/gpbckp_converters.go new file mode 100644 index 00000000..a4836c86 --- /dev/null +++ b/exporter/gpbckp_converters.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package exporter + +import "github.com/apache/cloudberry-backup/history" + +// Convert bool to float64. +func convertBoolToFloat64(value bool) float64 { + if value { + return 1 + } + return 0 +} + +// Convert backup status to float64. +func convertStatusFloat64(valueStatus string) float64 { + if valueStatus == history.BackupStatusFailed { + return 1 + } + return 0 +} + +// Convert empty string to empty label ("none" value). +func convertEmptyLabel(str string) string { + if str == "" { + return emptyLabel + } + return str +} diff --git a/exporter/gpbckp_converters_test.go b/exporter/gpbckp_converters_test.go new file mode 100644 index 00000000..75db69a0 --- /dev/null +++ b/exporter/gpbckp_converters_test.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package exporter + +import ( + "github.com/apache/cloudberry-backup/history" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Converters", func() { + Describe("convertBoolToFloat64", func() { + It("returns 1 for true", func() { + Expect(convertBoolToFloat64(true)).To(Equal(float64(1))) + }) + It("returns 0 for false", func() { + Expect(convertBoolToFloat64(false)).To(Equal(float64(0))) + }) + }) + + Describe("convertEmptyLabel", func() { + It("returns 'none' for empty string", func() { + Expect(convertEmptyLabel("")).To(Equal("none")) + }) + It("returns original string for non-empty string", func() { + Expect(convertEmptyLabel("text")).To(Equal("text")) + }) + }) + + Describe("convertStatusFloat64", func() { + It("returns 1 for Failure status", func() { + Expect(convertStatusFloat64(history.BackupStatusFailed)).To(Equal(float64(1))) + }) + It("returns 0 for non-Failure status", func() { + Expect(convertStatusFloat64("text")).To(Equal(float64(0))) + }) + }) +}) diff --git a/exporter/gpbckp_exporter.go b/exporter/gpbckp_exporter.go new file mode 100644 index 00000000..de505786 --- /dev/null +++ b/exporter/gpbckp_exporter.go @@ -0,0 +1,186 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package exporter + +import ( + "log/slog" + "net/http" + "os" + "time" + + "github.com/apache/cloudberry-backup/gpbackman/gpbckpconfig" + "github.com/apache/cloudberry-backup/history" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/exporter-toolkit/web" +) + +var ( + webFlagsConfig web.FlagConfig + webEndpoint string +) + +// SetPromPortAndPath sets HTTP endpoint parameters. +func SetPromPortAndPath(flagsConfig web.FlagConfig, endpoint string) { + webFlagsConfig = flagsConfig + webEndpoint = endpoint +} + +// StartPromEndpoint run HTTP endpoint. +func StartPromEndpoint(version string, logger *slog.Logger) { + go func(logger *slog.Logger) { + if webEndpoint == "" { + logger.Error("Metric endpoint is empty; aborting exporter startup", "endpoint", webEndpoint) + os.Exit(1) + } + http.Handle(webEndpoint, promhttp.Handler()) + if webEndpoint != "/" { + landingConfig := web.LandingConfig{ + Name: "cloudberry-backup exporter", + Description: "Prometheus exporter for Apache Cloudberry (Incubating) backup utility", + HeaderColor: "#476b6b", + Version: version, + Profiling: "false", + Links: []web.LandingLinks{ + { + Address: webEndpoint, + Text: "Metrics", + }, + }, + } + landingPage, err := web.NewLandingPage(landingConfig) + if err != nil { + logger.Error("Error creating landing page", "err", err) + os.Exit(1) + } + http.Handle("/", landingPage) + } + server := &http.Server{ + ReadHeaderTimeout: 5 * time.Second, + } + if err := web.ListenAndServe(server, &webFlagsConfig, logger); err != nil { + logger.Error("Run web endpoint failed", "err", err) + os.Exit(1) + } + }(logger) +} + +// GetGPBackupInfo get and parse gpbackup history database. +func GetGPBackupInfo(historyFile, backupType string, collectDeleted, collectFailed bool, dbInclude, dbExclude []string, collectDepth int, logger *slog.Logger) { + // To calculate the time elapsed since the last completed backup for specific database. + // For all databases values are calculated relative to one value. + // To calculate the time elapsed since the last completed backup for specific database. + // For all databases values are calculated relative to one value. + currentTime := time.Now() + currentUnixTime := currentTime.Unix() + // Calculate metrics collection depth. + // For backups with timestamp older than this - metrics doesn't collect. + collectDepthTime := currentTime.AddDate(0, 0, -collectDepth) + // The backup number can be reduced using filters for deleted and failed backups. + backupConfigs, err := parseBackupData(historyFile, collectDeleted, collectFailed, logger) + if err != nil { + logger.Error("Get data failed", "err", err) + } + // Reset metrics. + resetMetrics() + if len(backupConfigs) != 0 { + // Like lastbackups["testDB"]["full"] = time + lastBackups := make(lastBackupMap) + dbStatus := make(dbStatusMap) + for i := 0; i < len(backupConfigs); i++ { + db := backupConfigs[i].DatabaseName + // If the same database is specified in include and exclude list, + // then metrics for this database will not be collected. + if !dbInList(db, dbExclude) { + if listEmpty(dbInclude) || dbInList(db, dbInclude) { + dbStatus[db] = true + bckpType, err := gpbckpconfig.GetBackupType(backupConfigs[i]) + if err != nil { + logger.Error("Parse backup type value failed", "err", err) + } + // Check backup type and compare with backup type filter. + if backupType == "" || backupType == bckpType { + // History file contains backup timestamp and endtime with timezone information. + // It is necessary to take this into account when calculating time intervals. + // With a high probability, the exporter will work in the same timezone as Greenplum cluster. + // If this is not the case, then there are many questions about the backup process. + bckpStartTime, err := time.ParseInLocation(gpbckpconfig.Layout, backupConfigs[i].Timestamp, time.Local) + if err != nil { + logger.Error("Parse backup timestamp value failed", "err", err) + } + bckpStopTime, err := time.ParseInLocation(gpbckpconfig.Layout, backupConfigs[i].EndTime, time.Local) + if err != nil { + logger.Error("Parse backup end time value failed", "err", err) + } + // Only if set correct value for collectDepth. + if collectDepth > 0 { + // gpbackup_history.db is sorted by timestamp values. + // The data of the most recent backup is always located at the beginning. + // So as soon as we get the first value that is older than collectDepthTime, + // the cycle can be broken. + // If this behavior ever changes, then this code needs to be refactored. + if collectDepthTime.Before(bckpStartTime) { + getBackupMetrics(backupConfigs[i], setUpMetricValue, logger) + } else { + break + } + } else { + getBackupMetrics(backupConfigs[i], setUpMetricValue, logger) + } + if backupConfigs[i].Status == history.BackupStatusSucceed { + // Check specific database key already exist. + if dbLastBackups, ok := lastBackups[db]; ok { + // Check specific backup type key already exist. + if _, ok := dbLastBackups[bckpType]; !ok { + dbLastBackups[bckpType] = bckpStopTime + } + // A small note on the code above. + // Since the history file is already sorted, the first occurrence will be the last backup. + // However, if sorting is suddenly removed in the future, the code should be something like this: + // if curLastTime, ok := dbLastBackups[bckpType]; ok { + // if curLastTime.Before(bckpStopTime) { + // dbLastBackups[bckpType] = bckpStopTime + // } + // } else { + // dbLastBackups[bckpType] = bckpStopTime + // } + } else { + lastBackups[db] = backupMap{bckpType: bckpStopTime} + } + } + } + } + } else if dbInList(db, dbInclude) { + // When db is specified in both include and exclude lists, a warning is displayed in the log + // and data for this db is not collected. + // Set zero metric value for this db. + dbStatus[db] = false + logger.Warn("DB is specified in include and exclude lists", "DB", db) + } + } + if len(lastBackups) != 0 { + getBackupLastMetrics(lastBackups, currentUnixTime, setUpMetricValue, logger) + } else { + logger.Warn("No succeed backups") + } + getExporterStatusMetrics(dbStatus, setUpMetricValue, logger) + } else { + logger.Warn("No backup data returned") + } +} diff --git a/exporter/gpbckp_exporter_metrics.go b/exporter/gpbckp_exporter_metrics.go new file mode 100644 index 00000000..a995b95d --- /dev/null +++ b/exporter/gpbckp_exporter_metrics.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package exporter + +import ( + "log/slog" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var gpbckpExporterStatusMetric = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "gpbackup_exporter_status", + Help: "gpbackup exporter get data status.", +}, + []string{"database_name"}) + +// Set exporter metrics: +// - gpbackup_exporter_status +func getExporterStatusMetrics(dbStatus dbStatusMap, setUpMetricValueFun setUpMetricValueFunType, logger *slog.Logger) { + for dbName, status := range dbStatus { + setUpMetric( + gpbckpExporterStatusMetric, + "gpbackup_exporter_status", + convertBoolToFloat64(status), + setUpMetricValueFun, + logger, + dbName, + ) + } +} + +func resetExporterMetrics() { + gpbckpExporterStatusMetric.Reset() +} diff --git a/exporter/gpbckp_exporter_test.go b/exporter/gpbckp_exporter_test.go new file mode 100644 index 00000000..c4016afc --- /dev/null +++ b/exporter/gpbckp_exporter_test.go @@ -0,0 +1,286 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package exporter + +import ( + "bytes" + "log/slog" + "os" + "strings" + + "github.com/apache/cloudberry-backup/history" + "github.com/prometheus/exporter-toolkit/web" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Create a test history database from backup configs. +func fakeHistoryFileData(backupConfigs []*history.BackupConfig) (string, error) { + tempFile, err := os.CreateTemp("", "gpbackup_history*.db") + if err != nil { + return "", err + } + tempFile.Close() + hDB, err := history.InitializeHistoryDatabase(tempFile.Name()) + if err != nil { + return "", err + } + defer hDB.Close() + for _, config := range backupConfigs { + err = history.StoreBackupHistory(hDB, config) + if err != nil { + return "", err + } + } + return tempFile.Name(), nil +} + +var _ = Describe("Exporter", func() { + Describe("SetPromPortAndPath", func() { + It("sets web flags config and endpoint", func() { + testFlagsConfig := web.FlagConfig{ + WebListenAddresses: &([]string{":19854"}), + WebSystemdSocket: func(i bool) *bool { return &i }(false), + WebConfigFile: func(i string) *string { return &i }(""), + } + testEndpoint := "/metrics" + SetPromPortAndPath(testFlagsConfig, testEndpoint) + Expect(webFlagsConfig.WebListenAddresses).To(BeIdenticalTo(testFlagsConfig.WebListenAddresses)) + Expect(webFlagsConfig.WebSystemdSocket).To(BeIdenticalTo(testFlagsConfig.WebSystemdSocket)) + Expect(webFlagsConfig.WebConfigFile).To(BeIdenticalTo(testFlagsConfig.WebConfigFile)) + Expect(webEndpoint).To(Equal(testEndpoint)) + }) + }) + + Describe("fakeHistoryFileData", func() { + It("creates valid db from backup configs", func() { + configs := []*history.BackupConfig{templateBackupConfig()} + dbFile, err := fakeHistoryFileData(configs) + Expect(err).ToNot(HaveOccurred()) + Expect(dbFile).ToNot(BeEmpty()) + defer os.Remove(dbFile) + }) + It("creates empty db from empty config list", func() { + configs := []*history.BackupConfig{} + dbFile, err := fakeHistoryFileData(configs) + Expect(err).ToNot(HaveOccurred()) + Expect(dbFile).ToNot(BeEmpty()) + defer os.Remove(dbFile) + }) + }) + + Describe("GetGPBackupInfo", func() { + It("returns good data for valid backups", func() { + metadataOnlyConfig := templateBackupConfig() + metadataOnlyConfig.MetadataOnly = true + metadataOnlyConfig.Timestamp = "20230118162454" + metadataOnlyConfig.EndTime = "20230118162456" + backupConfigs := []*history.BackupConfig{ + templateBackupConfig(), + metadataOnlyConfig, + } + dbFile, err := fakeHistoryFileData(backupConfigs) + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(dbFile) + resetMetrics() + out := &bytes.Buffer{} + lc := slog.New(slog.NewTextHandler(out, &slog.HandlerOptions{ + Level: slog.LevelDebug, + ReplaceAttr: func(_ []string, a slog.Attr) slog.Attr { + if a.Key == slog.TimeKey { + return slog.Attr{} + } + return a + }, + })) + GetGPBackupInfo(dbFile, "", false, false, []string{""}, []string{""}, 0, lc) + logOutput := out.String() + // Metadata-only backup (later timestamp) should appear first. + Expect(logOutput).To(ContainSubstring( + `level=DEBUG msg="Set up metric" metric=gpbackup_backup_status value=0 labels=metadata-only,test,none,none,20230118162454`)) + Expect(logOutput).To(ContainSubstring( + `level=DEBUG msg="Set up metric" metric=gpbackup_backup_deletion_status value=0 labels=metadata-only,test,none,none,none,20230118162454`)) + Expect(logOutput).To(ContainSubstring( + `level=DEBUG msg="Set up metric" metric=gpbackup_backup_info value=1 labels=/data/backups,1.30.5,metadata-only,gzip,test,6.23.0,none,none,none,20230118162454,false`)) + Expect(logOutput).To(ContainSubstring( + `level=DEBUG msg="Set up metric" metric=gpbackup_backup_duration_seconds value=2 labels=metadata-only,test,20230118162456,none,none,20230118162454`)) + // Full backup. + Expect(logOutput).To(ContainSubstring( + `level=DEBUG msg="Set up metric" metric=gpbackup_backup_status value=0 labels=full,test,none,none,20230118152654`)) + Expect(logOutput).To(ContainSubstring( + `level=DEBUG msg="Set up metric" metric=gpbackup_backup_deletion_status value=0 labels=full,test,none,none,none,20230118152654`)) + Expect(logOutput).To(ContainSubstring( + `level=DEBUG msg="Set up metric" metric=gpbackup_backup_info value=1 labels=/data/backups,1.30.5,full,gzip,test,6.23.0,none,none,none,20230118152654,false`)) + Expect(logOutput).To(ContainSubstring( + `level=DEBUG msg="Set up metric" metric=gpbackup_backup_duration_seconds value=2 labels=full,test,20230118152656,none,none,20230118152654`)) + }) + + It("warns when no data is returned", func() { + backupConfigs := []*history.BackupConfig{} + dbFile, err := fakeHistoryFileData(backupConfigs) + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(dbFile) + resetMetrics() + out := &bytes.Buffer{} + lc := slog.New(slog.NewTextHandler(out, &slog.HandlerOptions{ + Level: slog.LevelDebug, + ReplaceAttr: func(_ []string, a slog.Attr) slog.Attr { + if a.Key == slog.TimeKey { + return slog.Attr{} + } + return a + }, + })) + GetGPBackupInfo(dbFile, "", false, false, []string{""}, []string{""}, 0, lc) + Expect(out.String()).To(ContainSubstring(`No backup data returned`)) + }) + + It("logs parse error and emits no metrics", func() { + tempFile, err := os.CreateTemp("", "test*.yaml") + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tempFile.Name()) + resetMetrics() + out := &bytes.Buffer{} + lc := slog.New(slog.NewTextHandler(out, &slog.HandlerOptions{ + Level: slog.LevelDebug, + ReplaceAttr: func(_ []string, a slog.Attr) slog.Attr { + if a.Key == slog.TimeKey { + return slog.Attr{} + } + return a + }, + })) + GetGPBackupInfo(tempFile.Name(), "", false, false, []string{""}, []string{""}, 0, lc) + logOutput := out.String() + Expect(logOutput).To(ContainSubstring("Get data failed")) + Expect(logOutput).To(ContainSubstring("No backup data returned")) + Expect(logOutput).ToNot(ContainSubstring("Set up metric")) + }) + + It("warns when using depth and backup is older than depth interval", func() { + backupConfigs := []*history.BackupConfig{templateBackupConfig()} + dbFile, err := fakeHistoryFileData(backupConfigs) + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(dbFile) + resetMetrics() + out := &bytes.Buffer{} + lc := slog.New(slog.NewTextHandler(out, &slog.HandlerOptions{ + Level: slog.LevelDebug, + ReplaceAttr: func(_ []string, a slog.Attr) slog.Attr { + if a.Key == slog.TimeKey { + return slog.Attr{} + } + return a + }, + })) + GetGPBackupInfo(dbFile, "", false, false, []string{""}, []string{""}, 14, lc) + Expect(out.String()).To(ContainSubstring(`No succeed backups`)) + }) + + It("warns when db is in both include and exclude lists", func() { + backupConfigs := []*history.BackupConfig{templateBackupConfig()} + dbFile, err := fakeHistoryFileData(backupConfigs) + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(dbFile) + resetMetrics() + out := &bytes.Buffer{} + lc := slog.New(slog.NewTextHandler(out, &slog.HandlerOptions{ + Level: slog.LevelDebug, + ReplaceAttr: func(_ []string, a slog.Attr) slog.Attr { + if a.Key == slog.TimeKey { + return slog.Attr{} + } + return a + }, + })) + GetGPBackupInfo(dbFile, "", false, false, []string{"test"}, []string{"test"}, 0, lc) + Expect(out.String()).To(ContainSubstring(`DB is specified in include and exclude lists`)) + Expect(out.String()).To(ContainSubstring(`DB=test`)) + }) + + It("sets exporter status metric only for conflicting DB", func() { + goodConfig := templateBackupConfig() + goodConfig.DatabaseName = "good" + goodConfig.Timestamp = "20230118152654" + goodConfig.EndTime = "20230118152656" + + badConfig := templateBackupConfig() + badConfig.DatabaseName = "bad" + badConfig.Timestamp = "20230118152655" + badConfig.EndTime = "20230118152657" + + backupConfigs := []*history.BackupConfig{goodConfig, badConfig} + dbFile, err := fakeHistoryFileData(backupConfigs) + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(dbFile) + resetMetrics() + out := &bytes.Buffer{} + lc := slog.New(slog.NewTextHandler(out, &slog.HandlerOptions{ + Level: slog.LevelDebug, + ReplaceAttr: func(_ []string, a slog.Attr) slog.Attr { + if a.Key == slog.TimeKey { + return slog.Attr{} + } + return a + }, + })) + // Include both good and bad, but exclude only bad -> conflict for bad + GetGPBackupInfo(dbFile, "", false, false, []string{"good", "bad"}, []string{"bad"}, 0, lc) + logOutput := out.String() + Expect(logOutput).To(ContainSubstring(`metric=gpbackup_exporter_status value=1 labels=good`)) + Expect(logOutput).To(ContainSubstring(`metric=gpbackup_exporter_status value=0 labels=bad`)) + }) + + It("logs errors for invalid backup values", func() { + invalidConfig := &history.BackupConfig{ + DatabaseName: "test", + DataOnly: true, + Incremental: true, + IncludeSchemaFiltered: true, + ExcludeSchemaFiltered: true, + Timestamp: "test", + EndTime: "test", + Status: history.BackupStatusSucceed, + } + backupConfigs := []*history.BackupConfig{invalidConfig} + dbFile, err := fakeHistoryFileData(backupConfigs) + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(dbFile) + resetMetrics() + out := &bytes.Buffer{} + lc := slog.New(slog.NewTextHandler(out, &slog.HandlerOptions{ + Level: slog.LevelDebug, + ReplaceAttr: func(_ []string, a slog.Attr) slog.Attr { + if a.Key == slog.TimeKey { + return slog.Attr{} + } + return a + }, + })) + GetGPBackupInfo(dbFile, "", false, false, []string{""}, []string{""}, 0, lc) + logOutput := out.String() + Expect(logOutput).To(ContainSubstring(`Parse backup timestamp value failed`)) + // Verify errors were logged (parsing errors + metric setup errors). + errorsCount := strings.Count(logOutput, "level=ERROR") + Expect(errorsCount).To(BeNumerically(">", 0)) + }) + }) +}) diff --git a/exporter/gpbckp_last_backup_metrics.go b/exporter/gpbckp_last_backup_metrics.go new file mode 100644 index 00000000..a96e9262 --- /dev/null +++ b/exporter/gpbckp_last_backup_metrics.go @@ -0,0 +1,59 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package exporter + +import ( + "log/slog" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var gpbckpBackupSinceLastCompletionSecondsMetric = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "gpbackup_backup_since_last_completion_seconds", + Help: "Seconds since the last completed backup.", +}, + []string{ + "backup_type", + "database_name"}) + +// Set backup metrics: +// - gpbackup_backup_since_last_completion_seconds +func getBackupLastMetrics(lastBackups lastBackupMap, currentUnixTime int64, setUpMetricValueFun setUpMetricValueFunType, logger *slog.Logger) { + for db, bckps := range lastBackups { + for bckpType, endTime := range bckps { + // Seconds since the last completed backups. + setUpMetric( + gpbckpBackupSinceLastCompletionSecondsMetric, + "gpbackup_backup_since_last_completion_seconds", + time.Unix(currentUnixTime, 0).Sub(endTime).Seconds(), + setUpMetricValueFun, + logger, + bckpType, + db, + ) + } + } +} + +func resetLastBackupMetrics() { + gpbckpBackupSinceLastCompletionSecondsMetric.Reset() +} diff --git a/exporter/gpbckp_last_backup_metrics_test.go b/exporter/gpbckp_last_backup_metrics_test.go new file mode 100644 index 00000000..af2c1557 --- /dev/null +++ b/exporter/gpbckp_last_backup_metrics_test.go @@ -0,0 +1,96 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package exporter + +import ( + "bytes" + "fmt" + "log/slog" + "strings" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/expfmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("LastBackupMetrics", func() { + Describe("getBackupLastMetrics", func() { + It("sets last backup metrics correctly", func() { + resetLastBackupMetrics() + getBackupLastMetrics( + lastBackupMap{ + "test": backupMap{ + "full": returnTimeTime("20230118150000"), + "incremental": returnTimeTime("20230118160000"), + "metadata-only": returnTimeTime("20230118170000"), + "data-only": returnTimeTime("20230118180000"), + }, + }, + templateUnixTime(), + setUpMetricValue, + getLogger(), + ) + reg := prometheus.NewRegistry() + reg.MustRegister(gpbckpBackupSinceLastCompletionSecondsMetric) + metricFamily, err := reg.Gather() + if err != nil { + fmt.Println(err) + } + out := &bytes.Buffer{} + for _, mf := range metricFamily { + if _, err := expfmt.MetricFamilyToText(out, mf); err != nil { + panic(err) + } + } + templateMetrics := `# HELP gpbackup_backup_since_last_completion_seconds Seconds since the last completed backup. +# TYPE gpbackup_backup_since_last_completion_seconds gauge +gpbackup_backup_since_last_completion_seconds{backup_type="data-only",database_name="test"} 7200 +gpbackup_backup_since_last_completion_seconds{backup_type="full",database_name="test"} 18000 +gpbackup_backup_since_last_completion_seconds{backup_type="incremental",database_name="test"} 14400 +gpbackup_backup_since_last_completion_seconds{backup_type="metadata-only",database_name="test"} 10800 +` + Expect(out.String()).To(Equal(templateMetrics)) + }) + }) + + Describe("getBackupLastMetrics errors and debugs", func() { + It("counts errors and debugs correctly", func() { + resetLastBackupMetrics() + out := &bytes.Buffer{} + lc := slog.New(slog.NewTextHandler(out, &slog.HandlerOptions{Level: slog.LevelDebug})) + getBackupLastMetrics( + lastBackupMap{ + "test": backupMap{ + "full": returnTimeTime("20230118150000"), + }, + }, + templateUnixTime(), + fakeSetUpMetricValue, + lc, + ) + errorsOutputCount := strings.Count(out.String(), "level=ERROR") + debugsOutputCount := strings.Count(out.String(), "level=DEBUG") + Expect(errorsOutputCount).To(Equal(1)) + Expect(debugsOutputCount).To(Equal(1)) + }) + }) +}) diff --git a/exporter/gpbckp_parser.go b/exporter/gpbckp_parser.go new file mode 100644 index 00000000..aaa7e268 --- /dev/null +++ b/exporter/gpbckp_parser.go @@ -0,0 +1,170 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package exporter + +import ( + "errors" + "log/slog" + "path/filepath" + "strings" + "time" + + "github.com/apache/cloudberry-backup/gpbackman/gpbckpconfig" + "github.com/apache/cloudberry-backup/history" + "github.com/prometheus/client_golang/prometheus" +) + +const emptyLabel = "none" + +type setUpMetricValueFunType func(metric *prometheus.GaugeVec, value float64, labels ...string) error + +type backupMap map[string]time.Time +type lastBackupMap map[string]backupMap +type dbStatusMap map[string]bool + +func setUpMetricValue(metric *prometheus.GaugeVec, value float64, labels ...string) error { + metricVec, err := metric.GetMetricWithLabelValues(labels...) + if err != nil { + return err + } + // The situation should be handled by the prometheus libraries. + // But, anything is possible. + if metricVec == nil { + err := errors.New("metric is nil") + return err + } + metricVec.Set(value) + return nil +} + +// Get status code about backup deletion status. +// Based on available statuses from gpbackman utility documentation, +// but not limited to that. +// - 0 - backup still exists; +// - 1 - backup was successfully deleted; +// - 2 - the deletion is in progress; +// - 3 - last delete attempt failed to delete backup from plugin storage; +// - 4 - last delete attempt failed to delete backup from local storage; +func getDeletedStatusCode(valueDateDeleted string) (string, float64) { + var ( + dateDeleted string + deletedStatus float64 + ) + switch valueDateDeleted { + case "": + dateDeleted = emptyLabel + deletedStatus = 0 + case gpbckpconfig.DateDeletedInProgress: + dateDeleted = emptyLabel + deletedStatus = 2 + case gpbckpconfig.DateDeletedPluginFailed: + dateDeleted = emptyLabel + deletedStatus = 3 + case gpbckpconfig.DateDeletedLocalFailed: + dateDeleted = emptyLabel + deletedStatus = 4 + default: + dateDeleted = valueDateDeleted + deletedStatus = 1 + } + return dateDeleted, deletedStatus +} + +// Reset all metrics. +func resetMetrics() { + resetBackupMetrics() + resetLastBackupMetrics() + resetExporterMetrics() +} + +func setUpMetric(metric *prometheus.GaugeVec, metricName string, value float64, setUpMetricValueFun setUpMetricValueFunType, logger *slog.Logger, labels ...string) { + logger.Debug( + "Set up metric", + "metric", metricName, + "value", value, + "labels", strings.Join(labels, ","), + ) + err := setUpMetricValueFun(metric, value, labels...) + if err != nil { + logger.Error( + "Metric set up failed", + "metric", metricName, + "err", err, + ) + } +} + +func dbInList(db string, list []string) bool { + if listEmpty(list) { + return false + } + for _, val := range list { + if val == db { + return true + } + } + return false +} + +// Check list not empty. +func listEmpty(list []string) bool { + return strings.Join(list, "") == "" +} + +// Get and parse data from history database: +// - file with extension .db (sqlite). +// +// Returns parsed data or error. +func parseBackupData(historyFile string, collectDeleted, collectFailed bool, logger *slog.Logger) ([]*history.BackupConfig, error) { + if filepath.Ext(historyFile) != ".db" { + return nil, errors.New("file has an extension other than db (sqlite)") + } + return getDataFromHistoryDB(historyFile, collectDeleted, collectFailed, logger) +} + +func getDataFromHistoryDB(historyFile string, collectDeleted, collectFailed bool, logger *slog.Logger) ([]*history.BackupConfig, error) { + hDB, err := gpbckpconfig.OpenHistoryDB(historyFile) + if err != nil { + logger.Error("Open gpbackup history db failed", "err", err) + return nil, err + } + defer func() { + errClose := hDB.Close() + if errClose != nil { + logger.Error("Close gpbackup history db failed", "err", errClose) + } + }() + backupList, err := gpbckpconfig.GetBackupNamesDB(collectDeleted, collectFailed, hDB) + if err != nil { + logger.Error("Get backups from history db failed", "err", err) + return nil, err + } + // Get data for selected backups. + var backupConfigs []*history.BackupConfig + for _, backupName := range backupList { + backupData, err := gpbckpconfig.GetBackupDataDB(backupName, hDB) + if err != nil { + logger.Error("Get backup data from history db failed", "err", err) + return nil, err + } + backupConfigs = append(backupConfigs, backupData) + } + return backupConfigs, nil +} diff --git a/exporter/gpbckp_parser_test.go b/exporter/gpbckp_parser_test.go new file mode 100644 index 00000000..d7cacb71 --- /dev/null +++ b/exporter/gpbckp_parser_test.go @@ -0,0 +1,235 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package exporter + +import ( + "bytes" + "errors" + "log/slog" + "os" + "time" + + "github.com/apache/cloudberry-backup/gpbackman/gpbckpconfig" + "github.com/apache/cloudberry-backup/history" + "github.com/prometheus/client_golang/prometheus" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func getLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{ + Level: slog.LevelInfo, + })) +} + +func fakeSetUpMetricValue(_ *prometheus.GaugeVec, _ float64, _ ...string) error { + return errors.New("custom error for test") +} + +// Create a SQLite database file with missing tables. +func createCorruptedDBFile() string { + tempFile, err := os.CreateTemp("", "test_corrupted_*.db") + Expect(err).ToNot(HaveOccurred()) + defer tempFile.Close() + return tempFile.Name() +} + +// Create a database with invalid backup name. +func createDBWithInvalidBackupName() string { + tempFile, err := os.CreateTemp("", "test_invalid_backup_*.db") + Expect(err).ToNot(HaveOccurred()) + defer tempFile.Close() + db, err := gpbckpconfig.OpenHistoryDB(tempFile.Name()) + Expect(err).ToNot(HaveOccurred()) + defer db.Close() + _, err = db.Exec(`CREATE TABLE IF NOT EXISTS backups ( + timestamp TEXT PRIMARY KEY, + date_deleted TEXT, + database_name TEXT, + status TEXT + )`) + Expect(err).ToNot(HaveOccurred()) + // Insert a backup with an invalid timestamp. + _, err = db.Exec(`INSERT INTO backups (timestamp, date_deleted, database_name, status) VALUES + ('invalid_backup_name', '', 'testdb', 'Success')`) + Expect(err).ToNot(HaveOccurred()) + return tempFile.Name() +} + +func templateBackupConfig() *history.BackupConfig { + return &history.BackupConfig{ + BackupDir: "/data/backups", + BackupVersion: "1.30.5", + Compressed: true, + CompressionType: "gzip", + DatabaseName: "test", + DatabaseVersion: "6.23.0", + DataOnly: false, + DateDeleted: "", + ExcludeRelations: []string{}, + ExcludeSchemaFiltered: false, + ExcludeSchemas: []string{}, + ExcludeTableFiltered: false, + IncludeRelations: []string{}, + IncludeSchemaFiltered: false, + IncludeSchemas: []string{}, + IncludeTableFiltered: false, + Incremental: false, + LeafPartitionData: false, + MetadataOnly: false, + Plugin: "", + PluginVersion: "", + RestorePlan: []history.RestorePlanEntry{}, + SingleDataFile: false, + Timestamp: "20230118152654", + EndTime: "20230118152656", + WithoutGlobals: false, + WithStatistics: false, + Status: history.BackupStatusSucceed, + } +} + +func templateUnixTime() int64 { + // Thu Jan 18 2023 20:00:00 UTC + var curUnixTime int64 = 1674072000 + return curUnixTime +} + +func returnTimeTime(sTime string) time.Time { + rTime, err := time.Parse(gpbckpconfig.Layout, sTime) + if err != nil { + panic(err) + } + return rTime +} + +var _ = Describe("Parser", func() { + Describe("getDeletedStatusCode", func() { + It("returns 0 for existing backup (empty date)", func() { + dateDeleted, status := getDeletedStatusCode("") + Expect(dateDeleted).To(Equal("none")) + Expect(status).To(Equal(float64(0))) + }) + It("returns 2 for In Progress", func() { + dateDeleted, status := getDeletedStatusCode(gpbckpconfig.DateDeletedInProgress) + Expect(dateDeleted).To(Equal("none")) + Expect(status).To(Equal(float64(2))) + }) + It("returns 3 for Plugin Backup Delete Failed", func() { + dateDeleted, status := getDeletedStatusCode(gpbckpconfig.DateDeletedPluginFailed) + Expect(dateDeleted).To(Equal("none")) + Expect(status).To(Equal(float64(3))) + }) + It("returns 4 for Local Delete Failed", func() { + dateDeleted, status := getDeletedStatusCode(gpbckpconfig.DateDeletedLocalFailed) + Expect(dateDeleted).To(Equal("none")) + Expect(status).To(Equal(float64(4))) + }) + It("returns 1 for valid deletion date", func() { + dateDeleted, status := getDeletedStatusCode("20230118150331") + Expect(dateDeleted).To(Equal("20230118150331")) + Expect(status).To(Equal(float64(1))) + }) + }) + + Describe("setUpMetricValue", func() { + It("returns error when labels don't match", func() { + err := setUpMetricValue(gpbckpExporterStatusMetric, 0, "demo", "bad") + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("dbInList", func() { + It("returns true when db is in list", func() { + Expect(dbInList("test", []string{"test"})).To(BeTrue()) + }) + It("returns false when db is not in list", func() { + Expect(dbInList("test", []string{"demo"})).To(BeFalse()) + }) + It("returns false for empty list", func() { + Expect(dbInList("test", []string{""})).To(BeFalse()) + }) + }) + + Describe("listEmpty", func() { + It("returns true for empty list", func() { + Expect(listEmpty([]string{})).To(BeTrue()) + }) + It("returns false for non-empty list", func() { + Expect(listEmpty([]string{"a", "b", "c"})).To(BeFalse()) + }) + }) + + Describe("parseBackupData", func() { + It("returns error for yaml file", func() { + tempFile, err := os.CreateTemp("", "test*.yaml") + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tempFile.Name()) + got, err := parseBackupData(tempFile.Name(), false, false, getLogger()) + Expect(err).To(HaveOccurred()) + Expect(got).To(BeNil()) + }) + It("returns error for empty db file", func() { + tempFile, err := os.CreateTemp("", "test*.db") + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tempFile.Name()) + got, err := parseBackupData(tempFile.Name(), false, false, getLogger()) + Expect(err).To(HaveOccurred()) + Expect(got).To(BeNil()) + }) + It("returns error for unknown file extension", func() { + tempFile, err := os.CreateTemp("", "test*.txt") + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tempFile.Name()) + got, err := parseBackupData(tempFile.Name(), false, false, getLogger()) + Expect(err).To(HaveOccurred()) + Expect(got).To(BeNil()) + }) + }) + + Describe("getDataFromHistoryDB", func() { + It("returns error for invalid db file path", func() { + out := &bytes.Buffer{} + logger := slog.New(slog.NewTextHandler(out, &slog.HandlerOptions{Level: slog.LevelError})) + _, err := getDataFromHistoryDB("/nonexistent/path/to/db.db", false, false, logger) + Expect(err).To(HaveOccurred()) + Expect(out.String()).To(ContainSubstring("Open gpbackup history db failed")) + }) + It("returns error for corrupted db with invalid backup data", func() { + dbFile := createCorruptedDBFile() + defer os.Remove(dbFile) + out := &bytes.Buffer{} + logger := slog.New(slog.NewTextHandler(out, &slog.HandlerOptions{Level: slog.LevelError})) + _, err := getDataFromHistoryDB(dbFile, false, false, logger) + Expect(err).To(HaveOccurred()) + Expect(out.String()).To(ContainSubstring("Get backups from history db failed")) + }) + It("returns error for db with invalid backup name", func() { + dbFile := createDBWithInvalidBackupName() + defer os.Remove(dbFile) + out := &bytes.Buffer{} + logger := slog.New(slog.NewTextHandler(out, &slog.HandlerOptions{Level: slog.LevelError})) + _, err := getDataFromHistoryDB(dbFile, false, false, logger) + Expect(err).To(HaveOccurred()) + Expect(out.String()).To(ContainSubstring("Get backup data from history db failed")) + }) + }) +}) diff --git a/go.mod b/go.mod index cec552ff..c39faca4 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.24.0 require ( github.com/DATA-DOG/go-sqlmock v1.5.0 + github.com/alecthomas/kingpin/v2 v2.4.0 github.com/apache/cloudberry-go-libs v1.0.12-0.20250910014224-fc376e8a1056 github.com/aws/aws-sdk-go v1.44.257 github.com/blang/semver v3.5.1+incompatible @@ -11,7 +12,7 @@ require ( github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf github.com/jackc/pgconn v1.14.3 github.com/jmoiron/sqlx v1.3.5 - github.com/klauspost/compress v1.15.15 + github.com/klauspost/compress v1.18.0 github.com/lib/pq v1.10.7 github.com/mattn/go-sqlite3 v1.14.19 github.com/nightlyone/lockfile v1.0.0 @@ -19,23 +20,32 @@ require ( github.com/onsi/ginkgo/v2 v2.13.0 github.com/onsi/gomega v1.27.10 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/common v0.67.5 + github.com/prometheus/exporter-toolkit v0.15.1 github.com/sergi/go-diff v1.3.1 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 github.com/urfave/cli v1.22.13 - golang.org/x/sys v0.18.0 - golang.org/x/tools v0.12.0 + golang.org/x/sys v0.39.0 + golang.org/x/tools v0.39.0 gopkg.in/cheggaaa/pb.v1 v1.0.28 gopkg.in/yaml.v2 v2.4.0 ) require ( + github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/coreos/go-systemd/v22 v22.6.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/fatih/color v1.14.1 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/golang-jwt/jwt/v5 v5.3.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgio v1.0.0 // indirect @@ -45,13 +55,27 @@ require ( github.com/jackc/pgtype v1.14.0 // indirect github.com/jackc/pgx/v4 v4.18.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/jpillora/backoff v1.0.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/mdlayher/socket v0.4.1 // indirect + github.com/mdlayher/vsock v1.2.1 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/text v0.14.0 // indirect + github.com/xhit/go-str2duration/v2 v2.1.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/mod v0.30.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/oauth2 v0.34.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54 // indirect + golang.org/x/text v0.32.0 // indirect + golang.org/x/time v0.14.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index ca12ca24..9f21e075 100644 --- a/go.sum +++ b/go.sum @@ -4,14 +4,22 @@ github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20O github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY= +github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= +github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= +github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= github.com/apache/cloudberry-go-libs v1.0.12-0.20250910014224-fc376e8a1056 h1:ycrFztmYATpidbSAU1rw60XuhuDxgBHtLD3Sueu947c= github.com/apache/cloudberry-go-libs v1.0.12-0.20250910014224-fc376e8a1056/go.mod h1:lfHWkNYsno/lV+Nee0OoCmlOlBz5yvT6EW8WQEOUI5c= github.com/aws/aws-sdk-go v1.44.257 h1:HwelXYZZ8c34uFFhgVw3ybu2gB5fkk8KLj2idTvzZb8= github.com/aws/aws-sdk-go v1.44.257/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/vfs v1.0.0 h1:AUZUgulCDzbaNjTRWEP45X7m/J10brAptZpSRKRZBZc= github.com/blang/vfs v1.0.0/go.mod h1:jjuNUc/IKcRNNWC9NUCvz4fR9PZLPIKxEygtPs/4tSI= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -19,6 +27,8 @@ github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo= +github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= @@ -39,13 +49,17 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -104,17 +118,23 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= -github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -137,6 +157,14 @@ github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= +github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA= +github.com/mdlayher/vsock v1.2.1 h1:pC1mTJTvjo1r9n9fbm7S1j04rCgCzhCOS5DY0zqHlnQ= +github.com/mdlayher/vsock v1.2.1/go.mod h1:NRfCibel++DgeMD8z/hP+PPTjlNJsdPOmxcnENvE+SE= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nightlyone/lockfile v1.0.0 h1:RHep2cFKK4PonZJDdEl4GmkabuhbsRMgk/k3uAmxBiA= github.com/nightlyone/lockfile v1.0.0/go.mod h1:rywoIealpdNse2r832aiD9jRk8ErCatROs6LzC841CI= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -150,9 +178,21 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= +github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= +github.com/prometheus/exporter-toolkit v0.15.1 h1:XrGGr/qWl8Gd+pqJqTkNLww9eG8vR/CoRk0FubOKfLE= +github.com/prometheus/exporter-toolkit v0.15.1/go.mod h1:P/NR9qFRGbCFgpklyhix9F6v6fFr/VQB/CVsrMDGKo4= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -175,6 +215,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -183,16 +224,23 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/urfave/cli v1.22.13 h1:wsLILXG8qCJNse/qAgLNf23737Cx05GflHg/PJGe1Ok= github.com/urfave/cli v1.22.13/go.mod h1:VufqObjsMTF2BBwKawpx9R8eAneNEWhoO0yx8Vd+FkE= +github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -200,6 +248,8 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -210,14 +260,14 @@ golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -225,10 +275,14 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -246,8 +300,10 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54 h1:E2/AqCUMZGgd73TQkxUMcMla25GB9i/5HOdLr+uH7Vo= +golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54/go.mod h1:hKdjCMrbv9skySur+Nek8Hd0uJ0GuxJIoIX2payrIdQ= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -259,8 +315,10 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -271,19 +329,20 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= diff --git a/gpbackup_exporter.go b/gpbackup_exporter.go new file mode 100644 index 00000000..fc9f9da3 --- /dev/null +++ b/gpbackup_exporter.go @@ -0,0 +1,172 @@ +//go:build gpbackup_exporter + +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package main + +import ( + "log/slog" + "os" + "os/signal" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/alecthomas/kingpin/v2" + "github.com/apache/cloudberry-backup/exporter" + "github.com/prometheus/client_golang/prometheus" + version_collector "github.com/prometheus/client_golang/prometheus/collectors/version" + "github.com/prometheus/common/promslog" + "github.com/prometheus/common/promslog/flag" + "github.com/prometheus/common/version" + "github.com/prometheus/exporter-toolkit/web/kingpinflag" +) + +const exporterName = "gpbackup_exporter" + +func main() { + var ( + webPath = kingpin.Flag( + "web.telemetry-path", + "Path under which to expose metrics.", + ).Default("/metrics").String() + webAdditionalToolkitFlags = kingpinflag.AddFlags(kingpin.CommandLine, ":19854") + collectionInterval = kingpin.Flag( + "collect.interval", + "Collecting metrics interval in seconds.", + ).Default("600").Int() + collectionDepth = kingpin.Flag( + "collect.depth", + "Metrics depth collection in days. Metrics for backup older than this interval will not be collected. 0 - disable.", + ).Default("0").Int() + gpbckpHistoryFilePath = kingpin.Flag( + "gpbackup.history-file", + "Path to gpbackup_history.db.", + ).Default("").String() + gpbckpIncludeDB = kingpin.Flag( + "gpbackup.db-include", + "Specific db for collecting metrics. Can be specified several times.", + ).Default("").PlaceHolder("\"\"").Strings() + gpbckpExcludeDB = kingpin.Flag( + "gpbackup.db-exclude", + "Specific db to exclude from collecting metrics. Can be specified several times.", + ).Default("").PlaceHolder("\"\"").Strings() + gpbckpBackupType = kingpin.Flag( + "gpbackup.backup-type", + "Specific backup type for collecting metrics. One of: [full, incremental, data-only, metadata-only].", + ).Default("").String() + gpbckpBackupCollectDeleted = kingpin.Flag( + "gpbackup.collect-deleted", + "Collecting metrics for deleted backups.", + ).Default("false").Bool() + gpbckpBackupCollectFailed = kingpin.Flag( + "gpbackup.collect-failed", + "Collecting metrics for failed backups.", + ).Default("false").Bool() + ) + // Set logger config. + promslogConfig := &promslog.Config{} + // Add flags log.level and log.format from promslog package. + flag.AddFlags(kingpin.CommandLine, promslogConfig) + kingpin.Version(version.Print(exporterName)) + // Add short help flag. + kingpin.HelpFlag.Short('h') + // Load command line arguments. + kingpin.Parse() + // Setup signal catching. + sigs := make(chan os.Signal, 1) + // Catch listed signals. + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + // Set logger. + logger := promslog.New(promslogConfig) + // Method invoked upon seeing signal. + go func(logger *slog.Logger) { + s := <-sigs + logger.Info( + "Stopping exporter", + "name", filepath.Base(os.Args[0]), + "signal", s) + os.Exit(0) + }(logger) + logger.Info( + "Starting exporter", + "name", filepath.Base(os.Args[0]), + "version", version.Info()) + logger.Info("Build context", "build_context", version.BuildContext()) + logger.Info( + "History database file path", + "file", *gpbckpHistoryFilePath) + logger.Info( + "Collecting metrics for deleted and failed backups", + "deleted", *gpbckpBackupCollectDeleted, + "failed", *gpbckpBackupCollectFailed, + ) + if *collectionDepth > 0 { + logger.Info( + "Metrics depth collection in days", + "depth", *collectionDepth) + } + if strings.Join(*gpbckpIncludeDB, "") != "" { + for _, db := range *gpbckpIncludeDB { + logger.Info( + "Collecting metrics for specific DB", + "DB", db) + } + } + if strings.Join(*gpbckpExcludeDB, "") != "" { + for _, db := range *gpbckpExcludeDB { + logger.Info( + "Exclude collecting metrics for specific DB", + "DB", db) + } + } + if *gpbckpBackupType != "" { + logger.Info( + "Collecting metrics for specific backup type", + "type", *gpbckpBackupType) + } + // Setup parameters for exporter. + exporter.SetPromPortAndPath(*webAdditionalToolkitFlags, *webPath) + logger.Info( + "Use exporter parameters", + "endpoint", *webPath, + "config.file", *webAdditionalToolkitFlags.WebConfigFile, + ) + // Exporter build info metric. + prometheus.MustRegister(version_collector.NewCollector(exporterName)) + // Start web server. + exporter.StartPromEndpoint(version.Info(), logger) + for { + // Get information from gpbackup_history.db. + exporter.GetGPBackupInfo( + *gpbckpHistoryFilePath, + *gpbckpBackupType, + *gpbckpBackupCollectDeleted, + *gpbckpBackupCollectFailed, + *gpbckpIncludeDB, + *gpbckpExcludeDB, + *collectionDepth, + logger, + ) + // Sleep for 'collection.interval' seconds. + time.Sleep(time.Duration(*collectionInterval) * time.Second) + } +} From da93e97e68213b41ba31e75e5febe53fda65a553 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Fri, 15 May 2026 17:31:44 +0800 Subject: [PATCH 34/39] Package: extract install script generation to template file (#95) Extract install script generation from Makefile to a separate template file for better maintainability and readability. - Add install.sh.template with Apache License header - Simplify Makefile package target using sed for template substitution --- .github/workflows/cloudberry-backup-ci.yml | 49 +++++++++++++++++++++- Makefile | 37 ++++------------ install.sh.template | 48 +++++++++++++++++++++ 3 files changed, 104 insertions(+), 30 deletions(-) create mode 100644 install.sh.template diff --git a/.github/workflows/cloudberry-backup-ci.yml b/.github/workflows/cloudberry-backup-ci.yml index 5753ec66..4f025b81 100644 --- a/.github/workflows/cloudberry-backup-ci.yml +++ b/.github/workflows/cloudberry-backup-ci.yml @@ -187,7 +187,7 @@ jobs: strategy: fail-fast: false matrix: - test_target: [smoke, unit, integration, end_to_end, s3_plugin_e2e, regression, scale] + test_target: [smoke, unit, integration, end_to_end, s3_plugin_e2e, regression, scale, package_install] steps: - name: Free Disk Space @@ -477,6 +477,53 @@ jobs: chmod +x "${CLOUDBERRY_BACKUP_SRC}/.github/workflows/scale-tests-cloudberry-ci.bash" "${CLOUDBERRY_BACKUP_SRC}/.github/workflows/scale-tests-cloudberry-ci.bash" 2>&1 | tee "${TEST_LOG_ROOT}/cloudberry-backup-scale.log" ;; + package_install) + set -e + echo "Running package and install tests..." + + # Clean up previously installed binaries + echo "=== Cleaning up existing installations ===" + for binary in gpbackup gprestore gpbackup_helper gpbackup_s3_plugin gpbackman gpbackup_exporter; do + rm -f "${GPHOME}/bin/${binary}" + done + echo "Cleanup complete" + + # Create package + echo "=== Creating package ===" + make package 2>&1 | tee "${TEST_LOG_ROOT}/cloudberry-backup-package.log" + + # Find and extract package + package_file=$(ls -1 build/*.tar.gz 2>/dev/null | head -n 1) + if [ -z "${package_file}" ]; then + echo "ERROR: Package file not found" + exit 1 + fi + echo "Package created: ${package_file}" + + tar -xzf "${package_file}" + extracted_dir=$(ls -1d apache-cloudberry-backup-incubating-*/ 2>/dev/null | head -n 1) + echo "Package extracted" + ls -lh "${extracted_dir}" + + # Install using install.sh + echo "=== Installing via install.sh ===" + cd "${extracted_dir}" + chmod +x install.sh + GPHOME="${GPHOME}" ./install.sh 2>&1 | tee "${TEST_LOG_ROOT}/cloudberry-backup-install.log" + + # Test installed binaries + echo "=== Testing installed binaries ===" + for binary in gpbackup gprestore gpbackup_helper gpbackup_s3_plugin gpbackman gpbackup_exporter; do + version_output=$("${GPHOME}/bin/${binary}" --version 2>&1) + if [ $? -ne 0 ]; then + echo "ERROR: ${binary} --version failed" + exit 1 + fi + echo "${binary}: ${version_output}" + done + + echo "=== Package and install tests passed ===" + ;; *) echo "unknown test target: ${TEST_TARGET}" exit 2 diff --git a/Makefile b/Makefile index fa27092c..69675e9d 100644 --- a/Makefile +++ b/Makefile @@ -198,35 +198,14 @@ package: @cp NOTICE $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/ @cp DISCLAIMER $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/ @echo "Creating install script..." - @echo '#!/bin/bash' > $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo 'set -e' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo '' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo '# Use GPHOME if set, otherwise use default path' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo 'if [ -n "$$GPHOME" ]; then' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo ' INSTALL_DIR="$$GPHOME"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo 'elif [ -n "$$INSTALL_DIR" ]; then' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo ' INSTALL_DIR="$$INSTALL_DIR"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo 'else' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo ' INSTALL_DIR="/usr/local"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo 'fi' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo '' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo 'SCRIPT_DIR="$$(cd "$$(dirname "$${BASH_SOURCE[0]}")" && pwd)"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo '' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo 'echo "Installing $(PACKAGE_NAME) to $$INSTALL_DIR..."' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo '' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo '# Install binary files' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo 'sudo cp "$${SCRIPT_DIR}/bin/"* "$${INSTALL_DIR}/bin/"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo '' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo '# Set permissions' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo 'sudo chmod 755 "$${INSTALL_DIR}/bin/$(BACKUP)"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo 'sudo chmod 755 "$${INSTALL_DIR}/bin/$(RESTORE)"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo 'sudo chmod 755 "$${INSTALL_DIR}/bin/$(HELPER)"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo 'sudo chmod 755 "$${INSTALL_DIR}/bin/$(S3PLUGIN)"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo 'sudo chmod 755 "$${INSTALL_DIR}/bin/$(GPBACKMAN)"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo 'sudo chmod 755 "$${INSTALL_DIR}/bin/$(EXPORTER)"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo '' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo 'echo "Installation complete!"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh - @echo 'echo "$(PACKAGE_NAME) binaries installed to $${INSTALL_DIR}/bin/"' >> $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh + @sed -e 's/__PACKAGE_NAME__/$(PACKAGE_NAME)/g' \ + -e 's/__BACKUP__/$(BACKUP)/g' \ + -e 's/__RESTORE__/$(RESTORE)/g' \ + -e 's/__HELPER__/$(HELPER)/g' \ + -e 's/__S3PLUGIN__/$(S3PLUGIN)/g' \ + -e 's/__GPBACKMAN__/$(GPBACKMAN)/g' \ + -e 's/__EXPORTER__/$(EXPORTER)/g' \ + install.sh.template > $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh @chmod +x $(BUILD_DIR)/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/install.sh @echo "Creating tar.gz package..." @cd $(BUILD_DIR) && tar -czf $(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH).tar.gz $(PACKAGE_NAME)-$(PACKAGE_VERSION)-$(GOOS)-$(GOARCH)/ diff --git a/install.sh.template b/install.sh.template new file mode 100644 index 00000000..d61963a1 --- /dev/null +++ b/install.sh.template @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# ====================================================================== +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ====================================================================== + +set -e + +# Use GPHOME if set, otherwise use default path +if [ -n "$GPHOME" ]; then + INSTALL_DIR="$GPHOME" +elif [ -n "$INSTALL_DIR" ]; then + INSTALL_DIR="$INSTALL_DIR" +else + INSTALL_DIR="/usr/local" +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +echo "Installing __PACKAGE_NAME__ to $INSTALL_DIR..." + +# Install binary files +sudo cp "${SCRIPT_DIR}/bin/"* "${INSTALL_DIR}/bin/" + +# Set permissions +sudo chmod 755 "${INSTALL_DIR}/bin/__BACKUP__" +sudo chmod 755 "${INSTALL_DIR}/bin/__RESTORE__" +sudo chmod 755 "${INSTALL_DIR}/bin/__HELPER__" +sudo chmod 755 "${INSTALL_DIR}/bin/__S3PLUGIN__" +sudo chmod 755 "${INSTALL_DIR}/bin/__GPBACKMAN__" +sudo chmod 755 "${INSTALL_DIR}/bin/__EXPORTER__" + +echo "Installation complete!" +echo "__PACKAGE_NAME__ binaries installed to ${INSTALL_DIR}/bin/" From 0d994f6d75a0fb988d22d13de48c40c5f92a57bf Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Thu, 11 Jun 2026 16:08:49 +0800 Subject: [PATCH 35/39] CI: pin Cloudberry branch to `REL_2_STABLE` Switch the CI Cloudberry checkout from `main` to `REL_2_STABLE branch`. Cloudberry main has upgraded to the PG16 kernel, whose tightened role grant permission checks (GRANT ... GRANTED BY now requires ADMIN OPTION) break multiple backup integration tests. `REL_2_STABLE` still runs PG14 and passes the full test suite without these failures. Add a TODO to revert to main once Cloudberry 3.x stabilizes with PG16. See: https://github.com/apache/cloudberry-backup/issues/103 From d55b16cadda6ca27698b636e1d80bcbd19eaebd0 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Thu, 11 Jun 2026 14:51:19 +0800 Subject: [PATCH 36/39] NOTICE: add the original gpbackup NOTICE In our 2.1.0-rc2 review round, Jean Baptiste Onofre suggested we can include the original gpbackup NOTICE file content to the NOTICE. I also check the ASF docs, and find that we should do as the suggestion. You can see more details here: - https://lists.apache.org/thread/s3r9zmkt90cjnx8921mvmojw2zdfgh8k - https://infra.apache.org/licensing-howto.html#mod-notice --- NOTICE | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/NOTICE b/NOTICE index 1bc302b6..cf7f007c 100644 --- a/NOTICE +++ b/NOTICE @@ -3,3 +3,23 @@ Copyright 2024-2026 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). + +--------------------------------------------------------------------------- + +This product includes software originally developed by VMware. + +Greenplum Database Backup + +Copyright 2017-Present VMware, Inc. or its affiliates. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. From f7f05a0290f42eae07867a4cfda7b2a8a277cd87 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Mon, 29 Jun 2026 15:41:42 +0800 Subject: [PATCH 37/39] Bump Go from 1.24 to 1.25 (#99) * Bump Go from 1.24 to 1.25 * Update go version in README.md * Upgrade go-libs to the latest * Fix unit end_to_end test * Fix deadlock e2e test transaction handling. The test held locks with raw BEGIN/COMMIT SQL on DBConn and then reused the same connection for polling. With pgx v5 this can release the raw transaction lock, so gpbackup was not blocked and the test observed 0 deadlock traps. Use explicit DBConn transactions, separate polling connections, wait for goroutines before closing connections, and wait for the expected 2 blocked AccessExclusiveLock requests. * Fix pgx v5 PgError handling. After moving to pgx v5, database errors are returned as pgx/v5/pgconn.PgError. The old jackc/pgconn type assertions no longer matched, so backup lock handling could treat unexpected errors as lock-not-available, and restore COPY errors could lose their CONTEXT. Use errors.As with pgx/v5 pgconn in backup and restore code, update the restore test, and remove the old pgconn dependency. * Fix lock e2e test transaction handling. The deadlock and signal handler e2e tests still held locks with raw BEGIN/COMMIT SQL on DBConn and reused those connections for pg_locks polling. With pgx v5 this can release the raw transaction lock or leave goroutines polling on closed connections, causing missing blocked-lock counts and DBConn.Get panics. Use explicit transactions, dedicated lock and polling connections, and synchronize goroutines before closing test connections. * Fix gp_segment_configuration lock e2e test. The test intended to verify that gpbackup does not keep a lock on pg_catalog.gp_segment_configuration after passing that metadata query and blocking later on pg_catalog.pg_trigger. However, the old test did not actually exercise that state: it used raw BEGIN/COMMIT SQL through DBConn, committed before gpbackup was started, queried pg_locks in an unsynchronized goroutine, and ignored query errors while scanning SELECT * into an int. Use an explicit DBConn transaction on a dedicated lock connection, start gpbackup while the pg_trigger lock is still held, wait until gpbackup is visibly blocked on pg_trigger, and then synchronously assert that no gp_segment_configuration locks remain. Finally release pg_trigger and verify gpbackup completes successfully. --------- Co-authored-by: woblerr --- .github/workflows/build_and_unit_test.yml | 2 +- README.md | 2 +- backup/backup.go | 2 +- backup/data.go | 12 +- end_to_end/end_to_end_suite_test.go | 64 ++++++-- end_to_end/incremental_test.go | 2 +- end_to_end/locks_test.go | 180 +++++++++++++++++----- end_to_end/signal_handler_test.go | 129 ++++++---------- go.mod | 16 +- go.sum | 158 ++----------------- integration/data_backup_test.go | 2 +- integration/integration_suite_test.go | 2 +- report/report.go | 2 +- report/report_test.go | 2 +- restore/data.go | 5 +- restore/data_test.go | 2 +- testutils/functions.go | 2 +- utils/plugin.go | 2 +- utils/plugin_test.go | 2 +- 19 files changed, 286 insertions(+), 302 deletions(-) diff --git a/.github/workflows/build_and_unit_test.yml b/.github/workflows/build_and_unit_test.yml index ae140f9f..c4e64a24 100644 --- a/.github/workflows/build_and_unit_test.yml +++ b/.github/workflows/build_and_unit_test.yml @@ -21,7 +21,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.24" + go-version: "1.25" - name: Set Environment run: | diff --git a/README.md b/README.md index d41b4b26..eb2fafaa 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ repo is a fork of gpbackup, dedicated to supporting Cloudberry. ## Pre-Requisites -The project requires the Go Programming language version 1.21 or higher. +The project requires the Go Programming language version 1.25 or higher. Follow the directions [here](https://golang.org/doc/) for installation, usage and configuration instructions. Make sure to set the [Go PATH environment variable](https://go.dev/doc/install) before starting the following steps. diff --git a/backup/backup.go b/backup/backup.go index 2711f83c..9e12918d 100644 --- a/backup/backup.go +++ b/backup/backup.go @@ -624,7 +624,7 @@ type TableLocks struct { } func getTableLocks(table Table) []TableLocks { - conn := dbconn.NewDBConnFromEnvironment(MustGetFlagString(options.DBNAME)) + conn := dbconn.NewDBConnFromEnvironment(connectionPool.DBName) conn.MustConnect(1) var query string defer conn.Close() diff --git a/backup/data.go b/backup/data.go index 9d937bae..e4571408 100644 --- a/backup/data.go +++ b/backup/data.go @@ -16,7 +16,7 @@ import ( "github.com/apache/cloudberry-backup/utils" "github.com/apache/cloudberry-go-libs/dbconn" "github.com/apache/cloudberry-go-libs/gplog" - "github.com/jackc/pgconn" + "github.com/jackc/pgx/v5/pgconn" "gopkg.in/cheggaaa/pb.v1" ) @@ -214,13 +214,19 @@ func BackupDataForAllTables(tables []Table) []map[uint32]int64 { // tables before the metadata dumping part. err := LockTableNoWait(table, whichConn) if err != nil { - if pgErr, ok := err.(*pgconn.PgError); ok && pgErr.Code != PG_LOCK_NOT_AVAILABLE { + lockErr := err + var pgErr *pgconn.PgError + if !errors.As(lockErr, &pgErr) || pgErr.Code != PG_LOCK_NOT_AVAILABLE { isErroredBackup.Store(true) err = connectionPool.Rollback(whichConn) if err != nil { gplog.Warn("Worker %d: %s", whichConn, err) } - gplog.Fatal(fmt.Errorf("Unexpectedly unable to take lock on table %s, %s", table.FQN(), pgErr.Error()), "") + errMsg := lockErr.Error() + if pgErr != nil { + errMsg = pgErr.Error() + } + gplog.Fatal(fmt.Errorf("Unexpectedly unable to take lock on table %s, %s", table.FQN(), errMsg), "") } if gplog.GetVerbosity() < gplog.LOGVERBOSE { // Add a newline to interrupt the progress bar so that diff --git a/end_to_end/end_to_end_suite_test.go b/end_to_end/end_to_end_suite_test.go index a21a5977..cbbfc90f 100644 --- a/end_to_end/end_to_end_suite_test.go +++ b/end_to_end/end_to_end_suite_test.go @@ -29,7 +29,7 @@ import ( "github.com/apache/cloudberry-go-libs/operating" "github.com/apache/cloudberry-go-libs/structmatcher" "github.com/apache/cloudberry-go-libs/testhelper" - "github.com/blang/semver" + "github.com/blang/semver/v4" _ "github.com/mattn/go-sqlite3" "github.com/pkg/errors" "github.com/spf13/pflag" @@ -1993,7 +1993,16 @@ LANGUAGE plpgsql NO SQL;`) Skip("This test is not needed for old backup versions") } // Block on pg_trigger, which gpbackup queries after gp_segment_configuration - backupConn.MustExec("BEGIN; LOCK TABLE pg_trigger IN ACCESS EXCLUSIVE MODE") + lockConn := testutils.SetupTestDbConn("testdb") + defer lockConn.Close() + lockConn.MustBegin() + lockReleased := false + defer func() { + if !lockReleased { + _ = lockConn.Rollback() + } + }() + lockConn.MustExec("LOCK TABLE pg_catalog.pg_trigger IN ACCESS EXCLUSIVE MODE") args := []string{ "--dbname", "testdb", @@ -2001,19 +2010,54 @@ LANGUAGE plpgsql NO SQL;`) "--verbose"} cmd := exec.Command(gpbackupPath, args...) - backupConn.MustExec("COMMIT") - anotherConn := testutils.SetupTestDbConn("testdb") - defer anotherConn.Close() - var lockCount int + type commandResult struct { + output []byte + err error + } + gpbackupResultChan := make(chan commandResult, 1) go func() { - gpSegConfigQuery := `SELECT * FROM pg_locks l, pg_class c, pg_namespace n WHERE l.relation = c.oid AND n.oid = c.relnamespace AND c.relname = 'gp_segment_configuration';` - _ = anotherConn.Get(&lockCount, gpSegConfigQuery) + output, err := cmd.CombinedOutput() + gpbackupResultChan <- commandResult{output: output, err: err} }() + pollConn := testutils.SetupTestDbConn("testdb") + defer pollConn.Close() + + triggerLockQuery := `SELECT count(*) FROM pg_locks l, pg_class c, pg_namespace n WHERE l.relation = c.oid AND n.oid = c.relnamespace AND n.nspname = 'pg_catalog' AND c.relname = 'pg_trigger' AND l.granted = 'f' AND l.mode = 'AccessShareLock'` + var gpbackupBlockedLockCount int + var earlyResult *commandResult + for iterations := 100; iterations > 0; iterations-- { + select { + case result := <-gpbackupResultChan: + earlyResult = &result + default: + } + if earlyResult != nil { + break + } + + Expect(pollConn.Get(&gpbackupBlockedLockCount, triggerLockQuery)).To(Succeed()) + if gpbackupBlockedLockCount > 0 { + break + } + time.Sleep(100 * time.Millisecond) + } + if earlyResult != nil { + Fail(fmt.Sprintf("gpbackup finished before blocking on pg_trigger: %v\n%s", earlyResult.err, string(earlyResult.output))) + } + Expect(gpbackupBlockedLockCount).To(BeNumerically(">", 0), "gpbackup did not block on pg_trigger") + + var lockCount int + gpSegConfigQuery := `SELECT count(*) FROM pg_locks l, pg_class c, pg_namespace n WHERE l.relation = c.oid AND n.oid = c.relnamespace AND n.nspname = 'pg_catalog' AND c.relname = 'gp_segment_configuration'` + Expect(pollConn.Get(&lockCount, gpSegConfigQuery)).To(Succeed()) Expect(lockCount).To(Equal(0)) - output, _ := cmd.CombinedOutput() - stdout := string(output) + lockConn.MustCommit() + lockReleased = true + + result := <-gpbackupResultChan + stdout := string(result.output) + Expect(result.err).ToNot(HaveOccurred(), "%s", stdout) Expect(stdout).To(ContainSubstring("Backup completed successfully")) }) It("properly handles various implicit casts on pg_catalog.text", func() { diff --git a/end_to_end/incremental_test.go b/end_to_end/incremental_test.go index 50a15802..6434250f 100644 --- a/end_to_end/incremental_test.go +++ b/end_to_end/incremental_test.go @@ -9,7 +9,7 @@ import ( "github.com/apache/cloudberry-backup/history" "github.com/apache/cloudberry-go-libs/dbconn" "github.com/apache/cloudberry-go-libs/testhelper" - "github.com/blang/semver" + "github.com/blang/semver/v4" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" diff --git a/end_to_end/locks_test.go b/end_to_end/locks_test.go index 8b7ee35d..046a5730 100644 --- a/end_to_end/locks_test.go +++ b/end_to_end/locks_test.go @@ -3,6 +3,7 @@ package end_to_end_test import ( "fmt" "os/exec" + "sync" "time" "github.com/apache/cloudberry-backup/backup" @@ -27,7 +28,11 @@ var _ = Describe("Deadlock handling", func() { } // Acquire AccessExclusiveLock on public.foo to block gpbackup when it attempts // to grab AccessShareLocks before its metadata dump section. - backupConn.MustExec("BEGIN; LOCK TABLE public.foo IN ACCESS EXCLUSIVE MODE") + initialLockConn := testutils.SetupTestDbConn("testdb") + defer initialLockConn.Close() + initialLockConn.MustBegin() + defer func() { _ = initialLockConn.Rollback() }() + initialLockConn.MustExec("LOCK TABLE public.foo IN ACCESS EXCLUSIVE MODE") // Execute gpbackup with --jobs 10 since there are 10 tables to back up args := []string{ @@ -36,13 +41,25 @@ var _ = Describe("Deadlock handling", func() { "--jobs", "10", "--verbose"} cmd := exec.Command(gpbackupPath, args...) + + var wg sync.WaitGroup + releaseTriggerLock := make(chan struct{}) + var releaseTriggerLockOnce sync.Once + defer func() { + releaseTriggerLockOnce.Do(func() { close(releaseTriggerLock) }) + wg.Wait() + }() + // Concurrently wait for gpbackup to block when it requests an AccessShareLock on public.foo. Once // that happens, acquire an AccessExclusiveLock on pg_catalog.pg_trigger to block gpbackup during its // trigger metadata dump. Then release the initial AccessExclusiveLock on public.foo (from the // beginning of the test) to unblock gpbackup and let gpbackup move forward to the trigger metadata dump. - anotherConn := testutils.SetupTestDbConn("testdb") - defer anotherConn.Close() + wg.Add(1) go func() { + defer wg.Done() + triggerLockConn := testutils.SetupTestDbConn("testdb") + defer triggerLockConn.Close() + // Query to see if gpbackup's AccessShareLock request on public.foo is blocked checkLockQuery := `SELECT count(*) FROM pg_locks l, pg_class c, pg_namespace n WHERE l.relation = c.oid AND n.oid = c.relnamespace AND n.nspname = 'public' AND c.relname = 'foo' AND l.granted = 'f' AND l.mode = 'AccessShareLock'` @@ -50,7 +67,7 @@ var _ = Describe("Deadlock handling", func() { var gpbackupBlockedLockCount int iterations := 100 for iterations > 0 { - _ = anotherConn.Get(&gpbackupBlockedLockCount, checkLockQuery) + _ = triggerLockConn.Get(&gpbackupBlockedLockCount, checkLockQuery) if gpbackupBlockedLockCount < 1 { time.Sleep(100 * time.Millisecond) iterations-- @@ -63,8 +80,12 @@ var _ = Describe("Deadlock handling", func() { // during the trigger metadata dump so that the test can queue a bunch of // AccessExclusiveLock requests against the test tables. Afterwards, release the // AccessExclusiveLock on public.foo to let gpbackup go to the trigger metadata dump. - anotherConn.MustExec(`BEGIN; LOCK TABLE pg_catalog.pg_trigger IN ACCESS EXCLUSIVE MODE`) - backupConn.MustExec("COMMIT") + triggerLockConn.MustBegin() + defer func() { _ = triggerLockConn.Rollback() }() + triggerLockConn.MustExec(`LOCK TABLE pg_catalog.pg_trigger IN ACCESS EXCLUSIVE MODE`) + initialLockConn.MustCommit() + <-releaseTriggerLock + triggerLockConn.MustCommit() }() // Concurrently wait for gpbackup to block on the trigger metadata dump section. Once we @@ -73,7 +94,9 @@ var _ = Describe("Deadlock handling", func() { dataTables := []string{`public."FOObar"`, "public.foo", "public.holds", "public.sales", "public.bigtable", "schema2.ao1", "schema2.ao2", "schema2.foo2", "schema2.foo3", "schema2.returns"} for _, dataTable := range dataTables { + wg.Add(1) go func(dataTable string) { + defer wg.Done() accessExclusiveLockConn := testutils.SetupTestDbConn("testdb") defer accessExclusiveLockConn.Close() @@ -95,23 +118,32 @@ var _ = Describe("Deadlock handling", func() { // Queue an AccessExclusiveLock request on a test table which will later // result in a detected deadlock during the gpbackup data dump section. - accessExclusiveLockConn.MustExec(fmt.Sprintf(`BEGIN; LOCK TABLE %s IN ACCESS EXCLUSIVE MODE; COMMIT`, dataTable)) + accessExclusiveLockConn.MustBegin() + defer func() { _ = accessExclusiveLockConn.Rollback() }() + accessExclusiveLockConn.MustExec(fmt.Sprintf(`LOCK TABLE %s IN ACCESS EXCLUSIVE MODE`, dataTable)) + accessExclusiveLockConn.MustCommit() }(dataTable) } // Concurrently wait for all AccessExclusiveLock requests on all 10 test tables to block. // Once that happens, release the AccessExclusiveLock on pg_catalog.pg_trigger to unblock // gpbackup and let gpbackup move forward to the data dump section. - var accessExclBlockedLockCount int + accessExclBlockedLockCountChan := make(chan int, 1) + wg.Add(1) go func() { + defer wg.Done() + accessExclBlockedLockConn := testutils.SetupTestDbConn("testdb") + defer accessExclBlockedLockConn.Close() + // Query to check for ungranted AccessExclusiveLock requests on our test tables checkLockQuery := `SELECT count(*) FROM pg_locks WHERE granted = 'f' AND mode = 'AccessExclusiveLock'` // Wait up to 10 seconds + var accessExclBlockedLockCount int iterations := 100 for iterations > 0 { - _ = backupConn.Get(&accessExclBlockedLockCount, checkLockQuery) - if accessExclBlockedLockCount < 10 { + _ = accessExclBlockedLockConn.Get(&accessExclBlockedLockCount, checkLockQuery) + if accessExclBlockedLockCount < len(dataTables) { time.Sleep(100 * time.Millisecond) iterations-- } else { @@ -120,15 +152,18 @@ var _ = Describe("Deadlock handling", func() { } // Unblock gpbackup by releasing AccessExclusiveLock on pg_catalog.pg_trigger - anotherConn.MustExec("COMMIT") + accessExclBlockedLockCountChan <- accessExclBlockedLockCount + releaseTriggerLockOnce.Do(func() { close(releaseTriggerLock) }) }() // gpbackup has finished - output, _ := cmd.CombinedOutput() + output, err := cmd.CombinedOutput() stdout := string(output) + accessExclBlockedLockCount := <-accessExclBlockedLockCountChan + Expect(err).ToNot(HaveOccurred(), "%s", stdout) // Check that 10 deadlock traps were placed during the test - Expect(accessExclBlockedLockCount).To(Equal(10)) + Expect(accessExclBlockedLockCount).To(Equal(len(dataTables))) // No non-main worker should have been able to run COPY due to deadlock detection for i := 1; i < 10; i++ { expectedLockString := fmt.Sprintf("[DEBUG]:-Worker %d: LOCK TABLE ", i) @@ -155,7 +190,11 @@ var _ = Describe("Deadlock handling", func() { } // Acquire AccessExclusiveLock on public.foo to block gpbackup when it attempts // to grab AccessShareLocks before its metadata dump section. - backupConn.MustExec("BEGIN; LOCK TABLE public.foo IN ACCESS EXCLUSIVE MODE") + initialLockConn := testutils.SetupTestDbConn("testdb") + defer initialLockConn.Close() + initialLockConn.MustBegin() + defer func() { _ = initialLockConn.Rollback() }() + initialLockConn.MustExec("LOCK TABLE public.foo IN ACCESS EXCLUSIVE MODE") // Execute gpbackup with --copy-queue-size 2 args := []string{ @@ -166,13 +205,24 @@ var _ = Describe("Deadlock handling", func() { "--verbose"} cmd := exec.Command(gpbackupPath, args...) + var wg sync.WaitGroup + releaseTriggerLock := make(chan struct{}) + var releaseTriggerLockOnce sync.Once + defer func() { + releaseTriggerLockOnce.Do(func() { close(releaseTriggerLock) }) + wg.Wait() + }() + // Concurrently wait for gpbackup to block when it requests an AccessShareLock on public.foo. Once // that happens, acquire an AccessExclusiveLock on pg_catalog.pg_trigger to block gpbackup during its // trigger metadata dump. Then release the initial AccessExclusiveLock on public.foo (from the // beginning of the test) to unblock gpbackup and let gpbackup move forward to the trigger metadata dump. - anotherConn := testutils.SetupTestDbConn("testdb") - defer anotherConn.Close() + wg.Add(1) go func() { + defer wg.Done() + triggerLockConn := testutils.SetupTestDbConn("testdb") + defer triggerLockConn.Close() + // Query to see if gpbackup's AccessShareLock request on public.foo is blocked checkLockQuery := `SELECT count(*) FROM pg_locks l, pg_class c, pg_namespace n WHERE l.relation = c.oid AND n.oid = c.relnamespace AND n.nspname = 'public' AND c.relname = 'foo' AND l.granted = 'f' AND l.mode = 'AccessShareLock'` @@ -180,7 +230,7 @@ var _ = Describe("Deadlock handling", func() { var gpbackupBlockedLockCount int iterations := 100 for iterations > 0 { - _ = anotherConn.Get(&gpbackupBlockedLockCount, checkLockQuery) + _ = triggerLockConn.Get(&gpbackupBlockedLockCount, checkLockQuery) if gpbackupBlockedLockCount < 1 { time.Sleep(100 * time.Millisecond) iterations-- @@ -193,8 +243,12 @@ var _ = Describe("Deadlock handling", func() { // during the trigger metadata dump so that the test can queue a bunch of // AccessExclusiveLock requests against the test tables. Afterwards, release the // AccessExclusiveLock on public.foo to let gpbackup go to the trigger metadata dump. - anotherConn.MustExec(`BEGIN; LOCK TABLE pg_catalog.pg_trigger IN ACCESS EXCLUSIVE MODE`) - backupConn.MustExec("COMMIT") + triggerLockConn.MustBegin() + defer func() { _ = triggerLockConn.Rollback() }() + triggerLockConn.MustExec(`LOCK TABLE pg_catalog.pg_trigger IN ACCESS EXCLUSIVE MODE`) + initialLockConn.MustCommit() + <-releaseTriggerLock + triggerLockConn.MustCommit() }() // Concurrently wait for gpbackup to block on the trigger metadata dump section. Once we @@ -203,7 +257,9 @@ var _ = Describe("Deadlock handling", func() { dataTables := []string{`public."FOObar"`, "public.foo", "public.holds", "public.sales", "public.bigtable", "schema2.ao1", "schema2.ao2", "schema2.foo2", "schema2.foo3", "schema2.returns"} for _, dataTable := range dataTables { + wg.Add(1) go func(dataTable string) { + defer wg.Done() accessExclusiveLockConn := testutils.SetupTestDbConn("testdb") defer accessExclusiveLockConn.Close() @@ -225,23 +281,32 @@ var _ = Describe("Deadlock handling", func() { // Queue an AccessExclusiveLock request on a test table which will later // result in a detected deadlock during the gpbackup data dump section. - accessExclusiveLockConn.MustExec(fmt.Sprintf(`BEGIN; LOCK TABLE %s IN ACCESS EXCLUSIVE MODE; COMMIT`, dataTable)) + accessExclusiveLockConn.MustBegin() + defer func() { _ = accessExclusiveLockConn.Rollback() }() + accessExclusiveLockConn.MustExec(fmt.Sprintf(`LOCK TABLE %s IN ACCESS EXCLUSIVE MODE`, dataTable)) + accessExclusiveLockConn.MustCommit() }(dataTable) } // Concurrently wait for all AccessExclusiveLock requests on all 10 test tables to block. // Once that happens, release the AccessExclusiveLock on pg_catalog.pg_trigger to unblock // gpbackup and let gpbackup move forward to the data dump section. - var accessExclBlockedLockCount int + accessExclBlockedLockCountChan := make(chan int, 1) + wg.Add(1) go func() { + defer wg.Done() + accessExclBlockedLockConn := testutils.SetupTestDbConn("testdb") + defer accessExclBlockedLockConn.Close() + // Query to check for ungranted AccessExclusiveLock requests on our test tables checkLockQuery := `SELECT count(*) FROM pg_locks WHERE granted = 'f' AND mode = 'AccessExclusiveLock'` // Wait up to 10 seconds + var accessExclBlockedLockCount int iterations := 100 for iterations > 0 { - _ = backupConn.Get(&accessExclBlockedLockCount, checkLockQuery) - if accessExclBlockedLockCount < 10 { + _ = accessExclBlockedLockConn.Get(&accessExclBlockedLockCount, checkLockQuery) + if accessExclBlockedLockCount < len(dataTables) { time.Sleep(100 * time.Millisecond) iterations-- } else { @@ -250,15 +315,18 @@ var _ = Describe("Deadlock handling", func() { } // Unblock gpbackup by releasing AccessExclusiveLock on pg_catalog.pg_trigger - anotherConn.MustExec("COMMIT") + accessExclBlockedLockCountChan <- accessExclBlockedLockCount + releaseTriggerLockOnce.Do(func() { close(releaseTriggerLock) }) }() // gpbackup has finished - output, _ := cmd.CombinedOutput() + output, err := cmd.CombinedOutput() stdout := string(output) + accessExclBlockedLockCount := <-accessExclBlockedLockCountChan + Expect(err).ToNot(HaveOccurred(), "%s", stdout) // Check that 10 deadlock traps were placed during the test - Expect(accessExclBlockedLockCount).To(Equal(10)) + Expect(accessExclBlockedLockCount).To(Equal(len(dataTables))) // No non-main worker should have been able to run COPY due to deadlock detection for i := 1; i < 2; i++ { expectedLockString := fmt.Sprintf("[DEBUG]:-Worker %d: LOCK TABLE ", i) @@ -290,7 +358,11 @@ var _ = Describe("Deadlock handling", func() { } // Acquire AccessExclusiveLock on public.foo to block gpbackup when it attempts // to grab AccessShareLocks before its metadata dump section. - backupConn.MustExec("BEGIN; LOCK TABLE public.foo IN ACCESS EXCLUSIVE MODE") + initialLockConn := testutils.SetupTestDbConn("testdb") + defer initialLockConn.Close() + initialLockConn.MustBegin() + defer func() { _ = initialLockConn.Rollback() }() + initialLockConn.MustExec("LOCK TABLE public.foo IN ACCESS EXCLUSIVE MODE") args := []string{ "--dbname", "testdb", @@ -298,13 +370,25 @@ var _ = Describe("Deadlock handling", func() { "--jobs", "2", "--verbose"} cmd := exec.Command(gpbackupPath, args...) + + var wg sync.WaitGroup + releaseTriggerLock := make(chan struct{}) + var releaseTriggerLockOnce sync.Once + defer func() { + releaseTriggerLockOnce.Do(func() { close(releaseTriggerLock) }) + wg.Wait() + }() + // Concurrently wait for gpbackup to block when it requests an AccessShareLock on public.foo. Once // that happens, acquire an AccessExclusiveLock on pg_catalog.pg_trigger to block gpbackup during its // trigger metadata dump. Then release the initial AccessExclusiveLock on public.foo (from the // beginning of the test) to unblock gpbackup and let gpbackup move forward to the trigger metadata dump. - anotherConn := testutils.SetupTestDbConn("testdb") - defer anotherConn.Close() + wg.Add(1) go func() { + defer wg.Done() + triggerLockConn := testutils.SetupTestDbConn("testdb") + defer triggerLockConn.Close() + // Query to see if gpbackup's AccessShareLock request on public.foo is blocked checkLockQuery := `SELECT count(*) FROM pg_locks l, pg_class c, pg_namespace n WHERE l.relation = c.oid AND n.oid = c.relnamespace AND n.nspname = 'public' AND c.relname = 'foo' AND l.granted = 'f' AND l.mode = 'AccessShareLock'` @@ -312,7 +396,7 @@ var _ = Describe("Deadlock handling", func() { var gpbackupBlockedLockCount int iterations := 100 for iterations > 0 { - _ = anotherConn.Get(&gpbackupBlockedLockCount, checkLockQuery) + _ = triggerLockConn.Get(&gpbackupBlockedLockCount, checkLockQuery) if gpbackupBlockedLockCount < 1 { time.Sleep(100 * time.Millisecond) iterations-- @@ -325,8 +409,12 @@ var _ = Describe("Deadlock handling", func() { // during the trigger metadata dump so that the test can queue a bunch of // AccessExclusiveLock requests against the test tables. Afterwards, release the // AccessExclusiveLock on public.foo to let gpbackup go to the trigger metadata dump. - anotherConn.MustExec(`BEGIN; LOCK TABLE pg_catalog.pg_trigger IN ACCESS EXCLUSIVE MODE`) - backupConn.MustExec("COMMIT") + triggerLockConn.MustBegin() + defer func() { _ = triggerLockConn.Rollback() }() + triggerLockConn.MustExec(`LOCK TABLE pg_catalog.pg_trigger IN ACCESS EXCLUSIVE MODE`) + initialLockConn.MustCommit() + <-releaseTriggerLock + triggerLockConn.MustCommit() }() // Concurrently wait for gpbackup to block on the trigger metadata dump section. Once we @@ -336,7 +424,9 @@ var _ = Describe("Deadlock handling", func() { "schema2.ao1", "schema2.ao2", "schema2.foo2", "schema2.foo3", "schema2.returns"} lockedTables := []string{`public."FOObar"`, "public.foo"} for _, lockedTable := range lockedTables { + wg.Add(1) go func(lockedTable string) { + defer wg.Done() accessExclusiveLockConn := testutils.SetupTestDbConn("testdb") defer accessExclusiveLockConn.Close() @@ -357,23 +447,32 @@ var _ = Describe("Deadlock handling", func() { } // Queue an AccessExclusiveLock request on a test table which will later // result in a detected deadlock during the gpbackup data dump section. - accessExclusiveLockConn.MustExec(fmt.Sprintf(`BEGIN; LOCK TABLE %s IN ACCESS EXCLUSIVE MODE; COMMIT`, lockedTable)) + accessExclusiveLockConn.MustBegin() + defer func() { _ = accessExclusiveLockConn.Rollback() }() + accessExclusiveLockConn.MustExec(fmt.Sprintf(`LOCK TABLE %s IN ACCESS EXCLUSIVE MODE`, lockedTable)) + accessExclusiveLockConn.MustCommit() }(lockedTable) } - // Concurrently wait for all AccessExclusiveLock requests on all 10 test tables to block. + // Concurrently wait for all AccessExclusiveLock requests on all locked test tables to block. // Once that happens, release the AccessExclusiveLock on pg_catalog.pg_trigger to unblock // gpbackup and let gpbackup move forward to the data dump section. - var accessExclBlockedLockCount int + accessExclBlockedLockCountChan := make(chan int, 1) + wg.Add(1) go func() { + defer wg.Done() + accessExclBlockedLockConn := testutils.SetupTestDbConn("testdb") + defer accessExclBlockedLockConn.Close() + // Query to check for ungranted AccessExclusiveLock requests on our test tables checkLockQuery := `SELECT count(*) FROM pg_locks WHERE granted = 'f' AND mode = 'AccessExclusiveLock'` // Wait up to 10 seconds + var accessExclBlockedLockCount int iterations := 100 for iterations > 0 { - _ = backupConn.Get(&accessExclBlockedLockCount, checkLockQuery) - if accessExclBlockedLockCount < 9 { + _ = accessExclBlockedLockConn.Get(&accessExclBlockedLockCount, checkLockQuery) + if accessExclBlockedLockCount < len(lockedTables) { time.Sleep(100 * time.Millisecond) iterations-- } else { @@ -382,12 +481,15 @@ var _ = Describe("Deadlock handling", func() { } // Unblock gpbackup by releasing AccessExclusiveLock on pg_catalog.pg_trigger - anotherConn.MustExec("COMMIT") + accessExclBlockedLockCountChan <- accessExclBlockedLockCount + releaseTriggerLockOnce.Do(func() { close(releaseTriggerLock) }) }() // gpbackup has finished - output, _ := cmd.CombinedOutput() + output, err := cmd.CombinedOutput() stdout := string(output) + accessExclBlockedLockCount := <-accessExclBlockedLockCountChan + Expect(err).ToNot(HaveOccurred(), "%s", stdout) // Check that 2 deadlock traps were placed during the test Expect(accessExclBlockedLockCount).To(Equal(2)) diff --git a/end_to_end/signal_handler_test.go b/end_to_end/signal_handler_test.go index e10c399e..95c7a07e 100644 --- a/end_to_end/signal_handler_test.go +++ b/end_to_end/signal_handler_test.go @@ -2,15 +2,60 @@ package end_to_end_test import ( "math/rand" + "os" "os/exec" "time" + "github.com/apache/cloudberry-backup/testutils" "github.com/apache/cloudberry-go-libs/testhelper" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "golang.org/x/sys/unix" ) +func gpbackupWithBlockedFoo2LockSignal(cmd *exec.Cmd, sig os.Signal, checkLockQuery string) ([]byte, int, int) { + lockConn := testutils.SetupTestDbConn("testdb") + defer lockConn.Close() + lockConn.MustBegin() + defer func() { _ = lockConn.Rollback() }() + lockConn.MustExec("LOCK TABLE schema2.foo2 IN ACCESS EXCLUSIVE MODE") + + beforeLockCountChan := make(chan int, 1) + signalDone := make(chan struct{}) + go func() { + defer close(signalDone) + lockCheckConn := testutils.SetupTestDbConn("testdb") + defer lockCheckConn.Close() + + var beforeLockCount int + iterations := 50 + for iterations > 0 { + _ = lockCheckConn.Get(&beforeLockCount, checkLockQuery) + if beforeLockCount < 1 { + time.Sleep(100 * time.Millisecond) + iterations-- + } else { + break + } + } + beforeLockCountChan <- beforeLockCount + if cmd.Process != nil { + _ = cmd.Process.Signal(sig) + } + }() + + output, _ := cmd.CombinedOutput() + <-signalDone + beforeLockCount := <-beforeLockCountChan + + afterLockCountConn := testutils.SetupTestDbConn("testdb") + defer afterLockCountConn.Close() + var afterLockCount int + _ = afterLockCountConn.Get(&afterLockCount, checkLockQuery) + + return output, beforeLockCount, afterLockCount +} + var _ = Describe("Signal handler tests", func() { BeforeEach(func() { end_to_end_setup() @@ -90,8 +135,6 @@ var _ = Describe("Signal handler tests", func() { // Query to see if gpbackup lock acquire on schema2.foo2 is blocked checkLockQuery := `SELECT count(*) FROM pg_locks l, pg_class c, pg_namespace n WHERE l.relation = c.oid AND n.oid = c.relnamespace AND n.nspname = 'schema2' AND c.relname = 'foo2' AND l.granted = 'f'` - // Acquire AccessExclusiveLock on schema2.foo2 to prevent gpbackup from acquiring AccessShareLock - backupConn.MustExec("BEGIN; LOCK TABLE schema2.foo2 IN ACCESS EXCLUSIVE MODE") args := []string{ "--dbname", "testdb", "--backup-dir", backupDir, @@ -100,29 +143,12 @@ var _ = Describe("Signal handler tests", func() { // Wait up to 5 seconds for gpbackup to block on acquiring AccessShareLock. // Once blocked, we send a SIGINT to cancel gpbackup. - var beforeLockCount int - go func() { - iterations := 50 - for iterations > 0 { - _ = backupConn.Get(&beforeLockCount, checkLockQuery) - if beforeLockCount < 1 { - time.Sleep(100 * time.Millisecond) - iterations-- - } else { - break - } - } - _ = cmd.Process.Signal(unix.SIGINT) - }() - output, _ := cmd.CombinedOutput() + output, beforeLockCount, afterLockCount := gpbackupWithBlockedFoo2LockSignal(cmd, unix.SIGINT, checkLockQuery) Expect(beforeLockCount).To(Equal(1)) // After gpbackup has been canceled, we should no longer see a blocked SQL // session trying to acquire AccessShareLock on foo2. - var afterLockCount int - _ = backupConn.Get(&afterLockCount, checkLockQuery) Expect(afterLockCount).To(Equal(0)) - backupConn.MustExec("ROLLBACK") stdout := string(output) Expect(stdout).To(ContainSubstring("Received an interrupt signal, aborting backup process")) @@ -140,8 +166,6 @@ var _ = Describe("Signal handler tests", func() { // Query to see if gpbackup lock acquire on schema2.foo2 is blocked checkLockQuery := `SELECT count(*) FROM pg_locks l, pg_class c, pg_namespace n WHERE l.relation = c.oid AND n.oid = c.relnamespace AND n.nspname = 'schema2' AND c.relname = 'foo2' AND l.granted = 'f'` - // Acquire AccessExclusiveLock on schema2.foo2 to prevent gpbackup from acquiring AccessShareLock - backupConn.MustExec("BEGIN; LOCK TABLE schema2.foo2 IN ACCESS EXCLUSIVE MODE") args := []string{ "--dbname", "testdb", "--backup-dir", backupDir, @@ -151,29 +175,12 @@ var _ = Describe("Signal handler tests", func() { // Wait up to 5 seconds for gpbackup to block on acquiring AccessShareLock. // Once blocked, we send a SIGINT to cancel gpbackup. - var beforeLockCount int - go func() { - iterations := 50 - for iterations > 0 { - _ = backupConn.Get(&beforeLockCount, checkLockQuery) - if beforeLockCount < 1 { - time.Sleep(100 * time.Millisecond) - iterations-- - } else { - break - } - } - _ = cmd.Process.Signal(unix.SIGINT) - }() - output, _ := cmd.CombinedOutput() + output, beforeLockCount, afterLockCount := gpbackupWithBlockedFoo2LockSignal(cmd, unix.SIGINT, checkLockQuery) Expect(beforeLockCount).To(Equal(1)) // After gpbackup has been canceled, we should no longer see a blocked SQL // session trying to acquire AccessShareLock on foo2. - var afterLockCount int - _ = backupConn.Get(&afterLockCount, checkLockQuery) Expect(afterLockCount).To(Equal(0)) - backupConn.MustExec("ROLLBACK") stdout := string(output) Expect(stdout).To(ContainSubstring("Received an interrupt signal, aborting backup process")) @@ -319,8 +326,6 @@ var _ = Describe("Signal handler tests", func() { // Query to see if gpbackup lock acquire on schema2.foo2 is blocked checkLockQuery := `SELECT count(*) FROM pg_locks l, pg_class c, pg_namespace n WHERE l.relation = c.oid AND n.oid = c.relnamespace AND n.nspname = 'schema2' AND c.relname = 'foo2' AND l.granted = 'f'` - // Acquire AccessExclusiveLock on schema2.foo2 to prevent gpbackup from acquiring AccessShareLock - backupConn.MustExec("BEGIN; LOCK TABLE schema2.foo2 IN ACCESS EXCLUSIVE MODE") args := []string{ "--dbname", "testdb", "--backup-dir", backupDir, @@ -329,29 +334,12 @@ var _ = Describe("Signal handler tests", func() { // Wait up to 5 seconds for gpbackup to block on acquiring AccessShareLock. // Once blocked, we send a SIGTERM to cancel gpbackup. - var beforeLockCount int - go func() { - iterations := 50 - for iterations > 0 { - _ = backupConn.Get(&beforeLockCount, checkLockQuery) - if beforeLockCount < 1 { - time.Sleep(100 * time.Millisecond) - iterations-- - } else { - break - } - } - _ = cmd.Process.Signal(unix.SIGTERM) - }() - output, _ := cmd.CombinedOutput() + output, beforeLockCount, afterLockCount := gpbackupWithBlockedFoo2LockSignal(cmd, unix.SIGTERM, checkLockQuery) Expect(beforeLockCount).To(Equal(1)) // After gpbackup has been canceled, we should no longer see a blocked SQL // session trying to acquire AccessShareLock on foo2. - var afterLockCount int - _ = backupConn.Get(&afterLockCount, checkLockQuery) Expect(afterLockCount).To(Equal(0)) - backupConn.MustExec("ROLLBACK") stdout := string(output) Expect(stdout).To(ContainSubstring("Received a termination signal, aborting backup process")) @@ -369,8 +357,6 @@ var _ = Describe("Signal handler tests", func() { // Query to see if gpbackup lock acquire on schema2.foo2 is blocked checkLockQuery := `SELECT count(*) FROM pg_locks l, pg_class c, pg_namespace n WHERE l.relation = c.oid AND n.oid = c.relnamespace AND n.nspname = 'schema2' AND c.relname = 'foo2' AND l.granted = 'f'` - // Acquire AccessExclusiveLock on schema2.foo2 to prevent gpbackup from acquiring AccessShareLock - backupConn.MustExec("BEGIN; LOCK TABLE schema2.foo2 IN ACCESS EXCLUSIVE MODE") args := []string{ "--dbname", "testdb", "--backup-dir", backupDir, @@ -380,29 +366,12 @@ var _ = Describe("Signal handler tests", func() { // Wait up to 5 seconds for gpbackup to block on acquiring AccessShareLock. // Once blocked, we send a SIGTERM to cancel gpbackup. - var beforeLockCount int - go func() { - iterations := 50 - for iterations > 0 { - _ = backupConn.Get(&beforeLockCount, checkLockQuery) - if beforeLockCount < 1 { - time.Sleep(100 * time.Millisecond) - iterations-- - } else { - break - } - } - _ = cmd.Process.Signal(unix.SIGTERM) - }() - output, _ := cmd.CombinedOutput() + output, beforeLockCount, afterLockCount := gpbackupWithBlockedFoo2LockSignal(cmd, unix.SIGTERM, checkLockQuery) Expect(beforeLockCount).To(Equal(1)) // After gpbackup has been canceled, we should no longer see a blocked SQL // session trying to acquire AccessShareLock on foo2. - var afterLockCount int - _ = backupConn.Get(&afterLockCount, checkLockQuery) Expect(afterLockCount).To(Equal(0)) - backupConn.MustExec("ROLLBACK") stdout := string(output) Expect(stdout).To(ContainSubstring("Received a termination signal, aborting backup process")) diff --git a/go.mod b/go.mod index c39faca4..37ab24fc 100644 --- a/go.mod +++ b/go.mod @@ -1,16 +1,16 @@ module github.com/apache/cloudberry-backup -go 1.24.0 +go 1.25.0 require ( github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/alecthomas/kingpin/v2 v2.4.0 - github.com/apache/cloudberry-go-libs v1.0.12-0.20250910014224-fc376e8a1056 + github.com/apache/cloudberry-go-libs v1.0.12-0.20260624080114-3de23e29a87a github.com/aws/aws-sdk-go v1.44.257 - github.com/blang/semver v3.5.1+incompatible + github.com/blang/semver/v4 v4.0.0 github.com/blang/vfs v1.0.0 github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf - github.com/jackc/pgconn v1.14.3 + github.com/jackc/pgx/v5 v5.9.2 github.com/jmoiron/sqlx v1.3.5 github.com/klauspost/compress v1.18.0 github.com/lib/pq v1.10.7 @@ -47,13 +47,9 @@ require ( github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect - github.com/jackc/chunkreader/v2 v2.0.1 // indirect - github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgproto3/v2 v2.3.3 // indirect - github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/pgtype v1.14.0 // indirect - github.com/jackc/pgx/v4 v4.18.2 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect diff --git a/go.sum b/go.sum index 9f21e075..1de9bbcf 100644 --- a/go.sum +++ b/go.sum @@ -1,21 +1,18 @@ -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= -github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY= github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= -github.com/apache/cloudberry-go-libs v1.0.12-0.20250910014224-fc376e8a1056 h1:ycrFztmYATpidbSAU1rw60XuhuDxgBHtLD3Sueu947c= -github.com/apache/cloudberry-go-libs v1.0.12-0.20250910014224-fc376e8a1056/go.mod h1:lfHWkNYsno/lV+Nee0OoCmlOlBz5yvT6EW8WQEOUI5c= +github.com/apache/cloudberry-go-libs v1.0.12-0.20260624080114-3de23e29a87a h1:xDVS0fObqCupd0eBTdk1OQC5vQJ9YD7KCyAsnP8zZXw= +github.com/apache/cloudberry-go-libs v1.0.12-0.20260624080114-3de23e29a87a/go.mod h1:yaH60R8eMGETbTAAxrcBVXJ0T7WwWXPddm2LWLKB7kI= github.com/aws/aws-sdk-go v1.44.257 h1:HwelXYZZ8c34uFFhgVw3ybu2gB5fkk8KLj2idTvzZb8= github.com/aws/aws-sdk-go v1.44.257/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/blang/vfs v1.0.0 h1:AUZUgulCDzbaNjTRWEP45X7m/J10brAptZpSRKRZBZc= github.com/blang/vfs v1.0.0/go.mod h1:jjuNUc/IKcRNNWC9NUCvz4fR9PZLPIKxEygtPs/4tSI= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -23,32 +20,22 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= -github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo= github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= -github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= @@ -57,7 +44,6 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -65,53 +51,14 @@ github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7P github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s= github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf/go.mod h1:yrqSXGoD/4EKfF26AOGzscPOgTTJcyAwM2rpixWT+t4= -github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= -github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= -github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= -github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= -github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= -github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= -github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= -github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= -github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= -github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= -github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= -github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= -github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= -github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= -github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= -github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= -github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= -github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= -github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= -github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= -github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= -github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= -github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= -github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= -github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= -github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= -github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.18.2 h1:xVpYkNR5pk5bMCZGfClbO962UIqVABcAGt7ha1s/FeU= -github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= -github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= +github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -120,34 +67,22 @@ github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -173,7 +108,6 @@ github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4 github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -190,36 +124,22 @@ github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzM github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= -github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= -github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= -github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= -github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -234,44 +154,18 @@ github.com/urfave/cli v1.22.13/go.mod h1:VufqObjsMTF2BBwKawpx9R8eAneNEWhoO0yx8Vd github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= @@ -283,17 +177,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -304,15 +189,11 @@ golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54 h1:E2/AqCUMZGgd73TQkxUMcMla25GB9i/5HOdLr+uH7Vo= golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54/go.mod h1:hKdjCMrbv9skySur+Nek8Hd0uJ0GuxJIoIX2payrIdQ= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= @@ -320,33 +201,19 @@ golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= -golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -354,4 +221,3 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/integration/data_backup_test.go b/integration/data_backup_test.go index 7e8ba574..1df7a916 100644 --- a/integration/data_backup_test.go +++ b/integration/data_backup_test.go @@ -11,7 +11,7 @@ import ( "github.com/apache/cloudberry-backup/utils" "github.com/apache/cloudberry-go-libs/gplog" "github.com/apache/cloudberry-go-libs/testhelper" - "github.com/blang/semver" + "github.com/blang/semver/v4" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" diff --git a/integration/integration_suite_test.go b/integration/integration_suite_test.go index da5627b5..ba87342d 100644 --- a/integration/integration_suite_test.go +++ b/integration/integration_suite_test.go @@ -16,7 +16,7 @@ import ( "github.com/apache/cloudberry-go-libs/dbconn" "github.com/apache/cloudberry-go-libs/gplog" "github.com/apache/cloudberry-go-libs/testhelper" - "github.com/blang/semver" + "github.com/blang/semver/v4" "github.com/spf13/pflag" . "github.com/onsi/ginkgo/v2" diff --git a/report/report.go b/report/report.go index d50ad9e5..8278775e 100644 --- a/report/report.go +++ b/report/report.go @@ -16,7 +16,7 @@ import ( "github.com/apache/cloudberry-go-libs/gplog" "github.com/apache/cloudberry-go-libs/iohelper" "github.com/apache/cloudberry-go-libs/operating" - "github.com/blang/semver" + "github.com/blang/semver/v4" "github.com/pkg/errors" "gopkg.in/yaml.v2" ) diff --git a/report/report_test.go b/report/report_test.go index e53cc2de..f01066eb 100644 --- a/report/report_test.go +++ b/report/report_test.go @@ -20,7 +20,7 @@ import ( "github.com/apache/cloudberry-go-libs/operating" "github.com/apache/cloudberry-go-libs/structmatcher" "github.com/apache/cloudberry-go-libs/testhelper" - "github.com/blang/semver" + "github.com/blang/semver/v4" "github.com/pkg/errors" "github.com/spf13/pflag" "gopkg.in/yaml.v2" diff --git a/restore/data.go b/restore/data.go index 44d21120..13ec4218 100644 --- a/restore/data.go +++ b/restore/data.go @@ -16,7 +16,7 @@ import ( "github.com/apache/cloudberry-go-libs/cluster" "github.com/apache/cloudberry-go-libs/dbconn" "github.com/apache/cloudberry-go-libs/gplog" - "github.com/jackc/pgconn" + "github.com/jackc/pgx/v5/pgconn" "github.com/pkg/errors" "gopkg.in/cheggaaa/pb.v1" ) @@ -60,7 +60,8 @@ func CopyTableIn(connectionPool *dbconn.DBConn, tableName string, tableAttribute errStr := fmt.Sprintf("Error loading data into table %s", tableName) // The COPY ON SEGMENT error might contain useful CONTEXT output - if pgErr, ok := err.(*pgconn.PgError); ok && pgErr.Where != "" { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Where != "" { errStr = fmt.Sprintf("%s: %s", errStr, pgErr.Where) } diff --git a/restore/data_test.go b/restore/data_test.go index 01a03b05..2e1c9da4 100644 --- a/restore/data_test.go +++ b/restore/data_test.go @@ -10,7 +10,7 @@ import ( "github.com/apache/cloudberry-backup/restore" "github.com/apache/cloudberry-backup/utils" "github.com/apache/cloudberry-go-libs/cluster" - "github.com/jackc/pgconn" + "github.com/jackc/pgx/v5/pgconn" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" diff --git a/testutils/functions.go b/testutils/functions.go index 19d786aa..da86eb0d 100644 --- a/testutils/functions.go +++ b/testutils/functions.go @@ -118,7 +118,7 @@ func SetupTestDBConnSegment(dbname string, port int, host string, gpVersion dbco gpRoleGuc = "gp_role" } - connStr := fmt.Sprintf("postgres://%s@%s:%d/%s?sslmode=disable&statement_cache_capacity=0&%s=utility", conn.User, conn.Host, conn.Port, conn.DBName, gpRoleGuc) + connStr := fmt.Sprintf("postgres://%s@%s:%d/%s?sslmode=disable&default_query_exec_mode=exec&%s=utility", conn.User, conn.Host, conn.Port, conn.DBName, gpRoleGuc) segConn, err := conn.Driver.Connect("pgx", connStr) if err != nil { diff --git a/utils/plugin.go b/utils/plugin.go index e1e9b1a2..93d09924 100644 --- a/utils/plugin.go +++ b/utils/plugin.go @@ -14,7 +14,7 @@ import ( "github.com/apache/cloudberry-go-libs/gplog" "github.com/apache/cloudberry-go-libs/iohelper" "github.com/apache/cloudberry-go-libs/operating" - "github.com/blang/semver" + "github.com/blang/semver/v4" "github.com/pkg/errors" "gopkg.in/yaml.v2" ) diff --git a/utils/plugin_test.go b/utils/plugin_test.go index 2fc22cf9..99989b1d 100644 --- a/utils/plugin_test.go +++ b/utils/plugin_test.go @@ -13,7 +13,7 @@ import ( "github.com/apache/cloudberry-go-libs/iohelper" "github.com/apache/cloudberry-go-libs/operating" "github.com/apache/cloudberry-go-libs/testhelper" - "github.com/blang/semver" + "github.com/blang/semver/v4" "github.com/pkg/errors" . "github.com/onsi/ginkgo/v2" From 7944c8751d09bbecfa97aad3bac8657578b07625 Mon Sep 17 00:00:00 2001 From: woblerr Date: Wed, 15 Jul 2026 18:36:45 +0300 Subject: [PATCH 38/39] Bump dependencies to fix Dependabot alerts. Bump golang.org/x/crypto to v0.52.0. Bump golang.org/x/net to v0.55.0. Bump other dependencies. --- go.mod | 16 ++++++++-------- go.sum | 32 ++++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 37ab24fc..48996fcd 100644 --- a/go.mod +++ b/go.mod @@ -27,8 +27,8 @@ require ( github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 github.com/urfave/cli v1.22.13 - golang.org/x/sys v0.39.0 - golang.org/x/tools v0.39.0 + golang.org/x/sys v0.45.0 + golang.org/x/tools v0.44.0 gopkg.in/cheggaaa/pb.v1 v1.0.28 gopkg.in/yaml.v2 v2.4.0 ) @@ -64,13 +64,13 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xhit/go-str2duration/v2 v2.1.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect - golang.org/x/crypto v0.46.0 // indirect - golang.org/x/mod v0.30.0 // indirect - golang.org/x/net v0.48.0 // indirect + golang.org/x/crypto v0.52.0 // indirect + golang.org/x/mod v0.35.0 // indirect + golang.org/x/net v0.55.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect - golang.org/x/sync v0.19.0 // indirect - golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54 // indirect - golang.org/x/text v0.32.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa // indirect + golang.org/x/text v0.37.0 // indirect golang.org/x/time v0.14.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 1de9bbcf..5ebb3b2b 100644 --- a/go.sum +++ b/go.sum @@ -160,23 +160,23 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= -golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= -golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -185,10 +185,10 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54 h1:E2/AqCUMZGgd73TQkxUMcMla25GB9i/5HOdLr+uH7Vo= -golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54/go.mod h1:hKdjCMrbv9skySur+Nek8Hd0uJ0GuxJIoIX2payrIdQ= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa h1:efT73AJZfAAUV7SOip6pWGkwJDzIGiKBZGVzHYa+ve4= +golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa/go.mod h1:kHjTxDEnAu6/Nl9lDkzjWpR+bmKfxeiRuSDlsMb70gE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -196,15 +196,15 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= -golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= From 8a28b53d536459954b9ca1916fa1e9d082f9de20 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Fri, 24 Jul 2026 14:03:52 +0800 Subject: [PATCH 39/39] Add convenience package build workflow for cloudberry-backup (#108) Introduce a manually-triggered GitHub Actions workflow that builds portable tar.gz packages for Apache cloudberry-backup from its official source release tarball, and tests them against Apache Cloudberry built from its own official source release. Key design decisions: - cloudberry-backup binaries are built on Rocky 8 (glibc 2.28) to maximise compatibility across Linux distributions. - Cloudberry is built per-platform from its source release tarball (~8 min), avoiding dependence on pre-built Docker images or external DEB/RPM repositories. - Build-and-test are fused into one job per platform: after Cloudberry compiles, gpdemo spins up directly in the same container, eliminating artifact packaging, transfer, and a separate test job. - Only .sha512 checksums are generated; .asc signatures remain a release manager local step. --- .../package-convenience-binaries.yml | 924 ++++++++++++++++++ 1 file changed, 924 insertions(+) create mode 100644 .github/workflows/package-convenience-binaries.yml diff --git a/.github/workflows/package-convenience-binaries.yml b/.github/workflows/package-convenience-binaries.yml new file mode 100644 index 00000000..40c50ef4 --- /dev/null +++ b/.github/workflows/package-convenience-binaries.yml @@ -0,0 +1,924 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# GitHub Actions Workflow: Apache Cloudberry-backup Convenience Package Build +# -------------------------------------------------------------------- +# Description: +# +# This workflow manually builds convenience portable tarball packages +# from an ASF-approved Apache Cloudberry-backup source release tarball, +# and tests them against Apache Cloudberry built from its official +# source release tarball. +# +# Workflow Overview: +# +# 1. verify-cloudberry-backup-source +# Validates inputs, downloads cloudberry-backup source tarball +# + .asc + .sha512, verifies GPG signature and checksum, uploads +# verified source as a workflow artifact. +# +# 2. verify-cloudberry-source +# Same as above, but for the Cloudberry source release tarball. +# +# 3. build-backup-packages (matrix: amd64 / arm64) +# Downloads the verified cloudberry-backup source artifact, runs +# `make package` on Rocky 8 (glibc 2.28) for maximum run-time +# compatibility, generates .sha512 checksums, uploads per-arch +# portable tar.gz packages. +# +# 4. build-cloudberry (matrix: 5 platforms × amd64 / arm64 = 10) +# Extracts Cloudberry source, runs configure + build inside the +# official Cloudberry build container (~8 min per platform). +# After the build, Cloudberry is already installed at +# /usr/local/cloudberry-db. The job then downloads the matching +# cloudberry-backup package, installs it, creates a gpdemo demo +# cluster, and runs a functional backup + restore smoke test — +# all inside the same container, no separate test job needed. +# Platforms: rocky8, rocky9, rocky10, ubuntu22.04, ubuntu24.04. +# +# Scope: +# - Intended for official Apache Cloudberry-backup source releases +# managed by the release manager. It's designed for 2.2+ release. +# - Produces convenience binaries only; detached .asc signatures remain a +# release manager local step. +# -------------------------------------------------------------------- + +name: Apache Cloudberry-backup Convenience Package Build + +on: + workflow_dispatch: + inputs: + # ================================================================ + # cloudberry-backup source release inputs + # ================================================================ + version: + description: '[cloudberry-backup] Release version, e.g. 2.2.0-incubating' + required: true + type: string + source_url: + description: '[cloudberry-backup] Apache source tarball URL from downloads.apache.org' + required: true + type: string + source_asc_url: + description: '[cloudberry-backup] Detached GPG signature URL for the source tarball (.asc)' + required: true + type: string + source_sha512_url: + description: '[cloudberry-backup] SHA-512 checksum URL for the source tarball (.sha512)' + required: true + type: string + + # ================================================================ + # Cloudberry source release inputs (for the test environment) + # ================================================================ + cloudberry_version: + description: '[Cloudberry] Release version, e.g. 2.2.0-incubating' + required: true + type: string + cloudberry_source_url: + description: '[Cloudberry] Apache source tarball URL from downloads.apache.org' + required: true + type: string + cloudberry_source_asc_url: + description: '[Cloudberry] Detached GPG signature URL for the source tarball (.asc)' + required: true + type: string + cloudberry_source_sha512_url: + description: '[Cloudberry] SHA-512 checksum URL for the source tarball (.sha512)' + required: true + type: string + +permissions: + contents: read + +concurrency: + group: backup-package-build-${{ github.ref }}-${{ inputs.version }} + cancel-in-progress: true + +env: + LOG_RETENTION_DAYS: 14 + KEYS_URL: https://downloads.apache.org/incubator/cloudberry/KEYS + +jobs: + # ==================================================================== + # Job 1: Verify the Apache Cloudberry-backup source release + # ==================================================================== + verify-cloudberry-backup-source: + name: Verify cloudberry-backup source + runs-on: ubuntu-24.04 + timeout-minutes: 15 + outputs: + source_tarball_name: ${{ steps.validate.outputs.source_tarball_name }} + artifact_name: ${{ steps.validate.outputs.artifact_name }} + packaging_version: ${{ steps.validate.outputs.packaging_version }} + steps: + - name: Validate manual inputs + id: validate + shell: bash + env: + VERSION: ${{ github.event.inputs.version }} + SOURCE_URL: ${{ github.event.inputs.source_url }} + SOURCE_ASC_URL: ${{ github.event.inputs.source_asc_url }} + SOURCE_SHA512_URL: ${{ github.event.inputs.source_sha512_url }} + run: | + set -euo pipefail + + if [[ -z "${VERSION}" ]]; then + echo "::error::version must not be empty" + exit 1 + fi + + source_tarball_name="apache-cloudberry-backup-${VERSION}-src.tar.gz" + artifact_name="verified-source-release-cbbackup-${VERSION}" + packaging_version="${VERSION%-incubating}" + + if [[ -z "${packaging_version}" ]]; then + echo "::error::Unable to derive packaging version from version=${VERSION}" + exit 1 + fi + + validate_apache_url() { + local value="$1" + local label="$2" + local prefix="https://downloads.apache.org/incubator/cloudberry/" + if [[ -z "${value}" ]]; then + echo "::error::${label} must not be empty" + exit 1 + fi + if [[ "${value}" != "${prefix}"* ]]; then + echo "::error::${label} must use downloads.apache.org (got: ${value})" + exit 1 + fi + } + + validate_apache_url "${SOURCE_URL}" "source_url" + validate_apache_url "${SOURCE_ASC_URL}" "source_asc_url" + validate_apache_url "${SOURCE_SHA512_URL}" "source_sha512_url" + + if [[ "${SOURCE_URL}" != */"${source_tarball_name}" ]]; then + echo "::error::source_url must end with /${source_tarball_name}" + exit 1 + fi + if [[ "${SOURCE_ASC_URL}" != */"${source_tarball_name}.asc" ]]; then + echo "::error::source_asc_url must end with /${source_tarball_name}.asc" + exit 1 + fi + if [[ "${SOURCE_SHA512_URL}" != */"${source_tarball_name}.sha512" ]]; then + echo "::error::source_sha512_url must end with /${source_tarball_name}.sha512" + exit 1 + fi + + echo "source_tarball_name=${source_tarball_name}" >> "${GITHUB_OUTPUT}" + echo "artifact_name=${artifact_name}" >> "${GITHUB_OUTPUT}" + echo "packaging_version=${packaging_version}" >> "${GITHUB_OUTPUT}" + + - name: Download source release and verification files + shell: bash + env: + SOURCE_URL: ${{ github.event.inputs.source_url }} + SOURCE_ASC_URL: ${{ github.event.inputs.source_asc_url }} + SOURCE_SHA512_URL: ${{ github.event.inputs.source_sha512_url }} + SOURCE_TARBALL_NAME: ${{ steps.validate.outputs.source_tarball_name }} + run: | + set -euo pipefail + mkdir -p verified-source + + echo "=== Downloading source tarball... ===" + curl --fail --location --silent --show-error \ + --output "verified-source/${SOURCE_TARBALL_NAME}" \ + "${SOURCE_URL}" + + echo "=== Downloading signature... ===" + curl --fail --location --silent --show-error \ + --output "verified-source/${SOURCE_TARBALL_NAME}.asc" \ + "${SOURCE_ASC_URL}" + + echo "=== Downloading checksum... ===" + curl --fail --location --silent --show-error \ + --output "verified-source/${SOURCE_TARBALL_NAME}.sha512" \ + "${SOURCE_SHA512_URL}" + + echo "=== Downloading KEYS... ===" + curl --fail --location --silent --show-error \ + --output "verified-source/KEYS" \ + "${KEYS_URL}" + + echo "=== All files downloaded successfully. ===" + ls -la verified-source/ + + - name: Verify source release signature and checksum + shell: bash + env: + SOURCE_TARBALL_NAME: ${{ steps.validate.outputs.source_tarball_name }} + run: | + set -euo pipefail + + export GNUPGHOME="${RUNNER_TEMP}/gnupg" + mkdir -p "${GNUPGHOME}" + chmod 700 "${GNUPGHOME}" + + echo "=== Importing project KEYS... ===" + gpg --import verified-source/KEYS + + echo "=== Verifying GPG signature... ===" + gpg --verify \ + "verified-source/${SOURCE_TARBALL_NAME}.asc" \ + "verified-source/${SOURCE_TARBALL_NAME}" + + echo "=== Verifying SHA-512 checksum... ===" + ( + cd verified-source + sha512sum -c "${SOURCE_TARBALL_NAME}.sha512" + ) + + echo "=== Verifying tarball integrity (listing contents)... ===" + tar -tzf "verified-source/${SOURCE_TARBALL_NAME}" >/dev/null + echo "=== cloudberry-backup source release verification passed. ===" + + - name: Summarize verified source release + shell: bash + env: + VERSION: ${{ github.event.inputs.version }} + SOURCE_URL: ${{ github.event.inputs.source_url }} + SOURCE_ASC_URL: ${{ github.event.inputs.source_asc_url }} + SOURCE_SHA512_URL: ${{ github.event.inputs.source_sha512_url }} + run: | + { + echo "# Verified cloudberry-backup source release" + echo "- Version: ${VERSION}" + echo "- Packaging version: ${{ steps.validate.outputs.packaging_version }}" + echo "- Source URL: ${SOURCE_URL}" + echo "- Signature URL: ${SOURCE_ASC_URL}" + echo "- Checksum URL: ${SOURCE_SHA512_URL}" + echo "- KEYS URL: ${KEYS_URL}" + echo "- GPG verification: PASS" + echo "- SHA-512 verification: PASS" + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Upload verified source release + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.validate.outputs.artifact_name }} + retention-days: ${{ env.LOG_RETENTION_DAYS }} + if-no-files-found: error + path: verified-source/* + + # ==================================================================== + # Job 2: Verify the Apache Cloudberry source release + # ==================================================================== + verify-cloudberry-source: + name: Verify Cloudberry source + runs-on: ubuntu-24.04 + timeout-minutes: 15 + outputs: + source_tarball_name: ${{ steps.validate.outputs.source_tarball_name }} + artifact_name: ${{ steps.validate.outputs.artifact_name }} + steps: + - name: Validate manual inputs + id: validate + shell: bash + env: + VERSION: ${{ github.event.inputs.cloudberry_version }} + SOURCE_URL: ${{ github.event.inputs.cloudberry_source_url }} + SOURCE_ASC_URL: ${{ github.event.inputs.cloudberry_source_asc_url }} + SOURCE_SHA512_URL: ${{ github.event.inputs.cloudberry_source_sha512_url }} + run: | + set -euo pipefail + + if [[ -z "${VERSION}" ]]; then + echo "::error::cloudberry_version must not be empty" + exit 1 + fi + + source_tarball_name="apache-cloudberry-${VERSION}-src.tar.gz" + artifact_name="verified-source-release-cloudberry-${VERSION}" + + validate_apache_url() { + local value="$1" + local label="$2" + local prefix="https://downloads.apache.org/incubator/cloudberry/" + if [[ -z "${value}" ]]; then + echo "::error::${label} must not be empty" + exit 1 + fi + if [[ "${value}" != "${prefix}"* ]]; then + echo "::error::${label} must use downloads.apache.org (got: ${value})" + exit 1 + fi + } + + validate_apache_url "${SOURCE_URL}" "cloudberry_source_url" + validate_apache_url "${SOURCE_ASC_URL}" "cloudberry_source_asc_url" + validate_apache_url "${SOURCE_SHA512_URL}" "cloudberry_source_sha512_url" + + if [[ "${SOURCE_URL}" != */"${source_tarball_name}" ]]; then + echo "::error::cloudberry_source_url must end with /${source_tarball_name}" + exit 1 + fi + if [[ "${SOURCE_ASC_URL}" != */"${source_tarball_name}.asc" ]]; then + echo "::error::cloudberry_source_asc_url must end with /${source_tarball_name}.asc" + exit 1 + fi + if [[ "${SOURCE_SHA512_URL}" != */"${source_tarball_name}.sha512" ]]; then + echo "::error::cloudberry_source_sha512_url must end with /${source_tarball_name}.sha512" + exit 1 + fi + + echo "source_tarball_name=${source_tarball_name}" >> "${GITHUB_OUTPUT}" + echo "artifact_name=${artifact_name}" >> "${GITHUB_OUTPUT}" + + - name: Download source release and verification files + shell: bash + env: + SOURCE_URL: ${{ github.event.inputs.cloudberry_source_url }} + SOURCE_ASC_URL: ${{ github.event.inputs.cloudberry_source_asc_url }} + SOURCE_SHA512_URL: ${{ github.event.inputs.cloudberry_source_sha512_url }} + SOURCE_TARBALL_NAME: ${{ steps.validate.outputs.source_tarball_name }} + run: | + set -euo pipefail + mkdir -p verified-source + + echo "=== Downloading Cloudberry source tarball... ===" + curl --fail --location --silent --show-error \ + --output "verified-source/${SOURCE_TARBALL_NAME}" \ + "${SOURCE_URL}" + + echo "=== Downloading signature... ===" + curl --fail --location --silent --show-error \ + --output "verified-source/${SOURCE_TARBALL_NAME}.asc" \ + "${SOURCE_ASC_URL}" + + echo "=== Downloading checksum... ===" + curl --fail --location --silent --show-error \ + --output "verified-source/${SOURCE_TARBALL_NAME}.sha512" \ + "${SOURCE_SHA512_URL}" + + echo "=== Downloading KEYS... ===" + curl --fail --location --silent --show-error \ + --output "verified-source/KEYS" \ + "${KEYS_URL}" + + ls -la verified-source/ + + - name: Verify source release signature and checksum + shell: bash + env: + SOURCE_TARBALL_NAME: ${{ steps.validate.outputs.source_tarball_name }} + run: | + set -euo pipefail + + export GNUPGHOME="${RUNNER_TEMP}/gnupg" + mkdir -p "${GNUPGHOME}" + chmod 700 "${GNUPGHOME}" + + echo "=== Importing project KEYS... ===" + gpg --import verified-source/KEYS + + echo "=== Verifying GPG signature... ===" + gpg --verify \ + "verified-source/${SOURCE_TARBALL_NAME}.asc" \ + "verified-source/${SOURCE_TARBALL_NAME}" + + echo "=== Verifying SHA-512 checksum... ===" + ( + cd verified-source + sha512sum -c "${SOURCE_TARBALL_NAME}.sha512" + ) + + tar -tzf "verified-source/${SOURCE_TARBALL_NAME}" >/dev/null + echo "=== Cloudberry source release verification passed. ===" + + - name: Summarize verified source release + shell: bash + env: + VERSION: ${{ github.event.inputs.cloudberry_version }} + SOURCE_URL: ${{ github.event.inputs.cloudberry_source_url }} + SOURCE_ASC_URL: ${{ github.event.inputs.cloudberry_source_asc_url }} + SOURCE_SHA512_URL: ${{ github.event.inputs.cloudberry_source_sha512_url }} + run: | + { + echo "# Verified Cloudberry source release" + echo "- Version: ${VERSION}" + echo "- Source URL: ${SOURCE_URL}" + echo "- Signature URL: ${SOURCE_ASC_URL}" + echo "- Checksum URL: ${SOURCE_SHA512_URL}" + echo "- KEYS URL: ${KEYS_URL}" + echo "- GPG verification: PASS" + echo "- SHA-512 verification: PASS" + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Upload verified source release + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.validate.outputs.artifact_name }} + retention-days: ${{ env.LOG_RETENTION_DAYS }} + if-no-files-found: error + path: verified-source/* + + # ==================================================================== + # Job 3: Build cloudberry-backup portable tarball packages. + # + # Runs BEFORE build-cloudberry so the backup package is ready when + # Cloudberry finishes building. Built on Rocky 8 (glibc 2.28) for + # maximum run-time compatibility across Linux distributions. + # ==================================================================== + build-backup-packages: + name: Build backup ${{ matrix.arch }} + needs: verify-cloudberry-backup-source + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + container: + image: ${{ matrix.build_container_image }} + options: >- + --user root + --hostname cdw + strategy: + fail-fast: false + matrix: + include: + - arch: linux-amd64 + runner: ubuntu-24.04 + goarch: amd64 + build_container_image: apache/incubator-cloudberry:cbdb-build-rocky8-latest + - arch: linux-arm64 + runner: ubuntu-24.04-arm + goarch: arm64 + build_container_image: apache/incubator-cloudberry:cbdb-build-rocky8-latest + + steps: + - name: Initialize build container + shell: bash + run: | + set -euo pipefail + su - gpadmin -c "/tmp/init_system.sh" + + - name: Verify build environment + shell: bash + run: | + set -euo pipefail + + echo "Build host: $(uname -m)" + su - gpadmin -c " + echo \"Go version: \$(go version)\" + echo \"GCC version: \$(gcc --version | head -1 || true)\" + echo \"glibc version: \$(ldd --version 2>&1 | head -1 || true)\" + " + + - name: Download verified cloudberry-backup source + uses: actions/download-artifact@v4 + with: + name: ${{ needs.verify-cloudberry-backup-source.outputs.artifact_name }} + path: ${{ github.workspace }}/verified-source + + - name: Extract source release + id: extract + shell: bash + env: + SOURCE_TARBALL_NAME: ${{ needs.verify-cloudberry-backup-source.outputs.source_tarball_name }} + run: | + set -euo pipefail + tarball_path="${GITHUB_WORKSPACE}/verified-source/${SOURCE_TARBALL_NAME}" + source_root_name="$(tar -tzf "${tarball_path}" | head -1 | cut -d/ -f1 || true)" + + echo "Extracting source: ${source_root_name}" + tar -xzf "${tarball_path}" -C "${GITHUB_WORKSPACE}" + source_dir="${GITHUB_WORKSPACE}/${source_root_name}" + echo "source_dir=${source_dir}" >> "${GITHUB_OUTPUT}" + echo "Source extracted to: ${source_dir}" + ls -la "${source_dir}" + + - name: Build convenience package + id: build + shell: bash + env: + SOURCE_DIR: ${{ steps.extract.outputs.source_dir }} + VERSION: ${{ github.event.inputs.version }} + run: | + set -euo pipefail + + # Give gpadmin ownership so make package can write build/ artifacts + chown -R gpadmin:gpadmin "${SOURCE_DIR}" + + echo "Building package natively on $(uname -m)..." + su - gpadmin -c " + set -euo pipefail + export GOPATH=\$HOME/go + export PATH=\$PATH:/usr/local/go/bin:\$GOPATH/bin + cd '${SOURCE_DIR}' + make package 2>&1 + " + + build_dir="${SOURCE_DIR}/build" + package_file=$(ls -1 "${build_dir}"/*.tar.gz 2>/dev/null | head -1 || true) + if [[ -z "${package_file}" || ! -f "${package_file}" ]]; then + echo "::error::Package file not found in ${build_dir}" + ls -la "${build_dir}" || true + exit 1 + fi + echo "Package built: ${package_file}" + echo "package_file=${package_file}" >> "${GITHUB_OUTPUT}" + echo "Package contents:" + tar -tzf "${package_file}" + + - name: Generate SHA512 checksum + shell: bash + env: + BUILD_DIR: ${{ steps.extract.outputs.source_dir }}/build + run: | + set -euo pipefail + artifact_dir="${GITHUB_WORKSPACE}/package-artifacts" + mkdir -p "${artifact_dir}" + cp "${BUILD_DIR}"/*.tar.gz "${artifact_dir}/" + ( + cd "${artifact_dir}" + for pkg in *.tar.gz; do + sha512sum "${pkg}" > "${pkg}.sha512" + echo "Generated: ${pkg}.sha512" + cat "${pkg}.sha512" + done + ) + + - name: Summarize build + shell: bash + env: + VERSION: ${{ github.event.inputs.version }} + ARCH: ${{ matrix.arch }} + run: | + artifact_dir="${GITHUB_WORKSPACE}/package-artifacts" + { + echo "# Build: ${ARCH}" + echo "- Release version: ${VERSION}" + echo "- Architecture: ${ARCH}" + echo "- Runner: ${{ matrix.runner }}" + echo "- Build container: ${{ matrix.build_container_image }}" + echo "- glibc baseline: Rocky 8 (glibc 2.28) for max compatibility" + echo "- Packages and checksums:" + ( + cd "${artifact_dir}" + for pkg in *.tar.gz; do + echo " - ${pkg}" + echo " - ${pkg}.sha512" + done + ) + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Upload package artifacts + uses: actions/upload-artifact@v4 + with: + name: packages-${{ matrix.arch }} + retention-days: ${{ env.LOG_RETENTION_DAYS }} + if-no-files-found: error + path: package-artifacts/ + + # ==================================================================== + # Job 4: Build Apache Cloudberry from verified source (~8 min), + # then test the cloudberry-backup package against it. + # + # Cloudberry is built and installed inside the build container at + # /usr/local/cloudberry-db. After the build, we download the + # cloudberry-backup package artifact, install it, spin up gpdemo, + # and run a functional backup + restore smoke test — all in one job. + # ==================================================================== + build-cloudberry: + name: Build & test ${{ matrix.target_os }}-${{ matrix.arch }} + needs: + - verify-cloudberry-source + - build-backup-packages + runs-on: ${{ matrix.runner }} + timeout-minutes: 75 + container: + image: ${{ matrix.build_container_image }} + options: >- + --user root + --hostname cdw + --shm-size=2gb + -v /usr/share:/host_usr_share + -v /usr/local:/host_usr_local + -v /opt:/host_opt + strategy: + fail-fast: false + matrix: + include: + # Rocky 8 + - {target_os: rocky8, arch: amd64, backup_arch: linux-amd64, runner: ubuntu-24.04, build_container_image: apache/incubator-cloudberry:cbdb-build-rocky8-latest} + - {target_os: rocky8, arch: arm64, backup_arch: linux-arm64, runner: ubuntu-24.04-arm, build_container_image: apache/incubator-cloudberry:cbdb-build-rocky8-latest} + # Rocky 9 + - {target_os: rocky9, arch: amd64, backup_arch: linux-amd64, runner: ubuntu-24.04, build_container_image: apache/incubator-cloudberry:cbdb-build-rocky9-latest} + - {target_os: rocky9, arch: arm64, backup_arch: linux-arm64, runner: ubuntu-24.04-arm, build_container_image: apache/incubator-cloudberry:cbdb-build-rocky9-latest} + # Rocky 10 + - {target_os: rocky10, arch: amd64, backup_arch: linux-amd64, runner: ubuntu-24.04, build_container_image: apache/incubator-cloudberry:cbdb-build-rocky10-latest} + - {target_os: rocky10, arch: arm64, backup_arch: linux-arm64, runner: ubuntu-24.04-arm, build_container_image: apache/incubator-cloudberry:cbdb-build-rocky10-latest} + # Ubuntu 22.04 + - {target_os: ubuntu22.04, arch: amd64, backup_arch: linux-amd64, runner: ubuntu-24.04, build_container_image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest} + - {target_os: ubuntu22.04, arch: arm64, backup_arch: linux-arm64, runner: ubuntu-24.04-arm, build_container_image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest} + # Ubuntu 24.04 + - {target_os: ubuntu24.04, arch: amd64, backup_arch: linux-amd64, runner: ubuntu-24.04, build_container_image: apache/incubator-cloudberry:cbdb-build-ubuntu24.04-latest} + - {target_os: ubuntu24.04, arch: arm64, backup_arch: linux-arm64, runner: ubuntu-24.04-arm, build_container_image: apache/incubator-cloudberry:cbdb-build-ubuntu24.04-latest} + + steps: + # ---------------------------------------------------------------- + # Phase A: Build Cloudberry from verified source + # ---------------------------------------------------------------- + + - name: Free disk space + shell: bash + run: | + set -euo pipefail + rm -rf /host_opt/hostedtoolcache || true + rm -rf /host_usr_local/lib/android || true + rm -rf /host_usr_share/dotnet || true + rm -rf /host_opt/ghc || true + rm -rf /host_usr_local/.ghcup || true + rm -rf /host_usr_share/swift || true + rm -rf /host_usr_local/share/powershell || true + rm -rf /host_usr_local/share/chromium || true + rm -rf /host_usr_share/miniconda || true + rm -rf /host_opt/az || true + rm -rf /host_usr_share/sbt || true + df -h / + + - name: Initialize build container + shell: bash + run: | + set -euo pipefail + su - gpadmin -c "/tmp/init_system.sh" + + - name: Download verified Cloudberry source + uses: actions/download-artifact@v4 + with: + name: ${{ needs.verify-cloudberry-source.outputs.artifact_name }} + path: ${{ github.workspace }}/verified-source + + - name: Extract Cloudberry source + id: extract + shell: bash + env: + SOURCE_TARBALL_NAME: ${{ needs.verify-cloudberry-source.outputs.source_tarball_name }} + run: | + set -euo pipefail + tarball_path="${GITHUB_WORKSPACE}/verified-source/${SOURCE_TARBALL_NAME}" + source_root_name="$(tar -tzf "${tarball_path}" | head -1 | cut -d/ -f1 || true)" + + tar -xzf "${tarball_path}" -C "${GITHUB_WORKSPACE}" + mkdir -p "${GITHUB_WORKSPACE}/cloudberry" + mv "${GITHUB_WORKSPACE}/${source_root_name}"/* "${GITHUB_WORKSPACE}/cloudberry/" + mv "${GITHUB_WORKSPACE}/${source_root_name}"/.[!.]* "${GITHUB_WORKSPACE}/cloudberry/" 2>/dev/null || true + rmdir "${GITHUB_WORKSPACE}/${source_root_name}" + + source_dir="${GITHUB_WORKSPACE}/cloudberry" + chown -R gpadmin:gpadmin "${GITHUB_WORKSPACE}" + echo "source_dir=${source_dir}" >> "${GITHUB_OUTPUT}" + echo "Cloudberry source extracted to: ${source_dir}" + + - name: Configure Cloudberry + shell: bash + env: + SOURCE_DIR: ${{ steps.extract.outputs.source_dir }} + run: | + set -euo pipefail + export BUILD_DESTINATION="/usr/local/cloudberry-db" + + mkdir -p "${SOURCE_DIR}/build-logs" + chown -R gpadmin:gpadmin "${SOURCE_DIR}/build-logs" + chmod +x "${SOURCE_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh + + echo "=== Configuring Cloudberry... ===" + if ! su - gpadmin -c "cd ${SOURCE_DIR} && SRC_DIR=${SOURCE_DIR} BUILD_USER=github-actions BUILD_DESTINATION=${BUILD_DESTINATION} ${SOURCE_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + echo "::error::Cloudberry configure script failed" + exit 1 + fi + echo "=== Cloudberry configuration complete. ===" + + - name: Build Cloudberry + shell: bash + env: + SOURCE_DIR: ${{ steps.extract.outputs.source_dir }} + run: | + set -euo pipefail + export BUILD_DESTINATION="/usr/local/cloudberry-db" + + chmod +x "${SOURCE_DIR}"/devops/build/automation/cloudberry/scripts/build-cloudberry.sh + + echo "=== Building Cloudberry... ===" + if ! su - gpadmin -c "cd ${SOURCE_DIR} && SRC_DIR=${SOURCE_DIR} BUILD_DESTINATION=${BUILD_DESTINATION} ${SOURCE_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"; then + echo "::error::Cloudberry build script failed" + exit 1 + fi + echo "=== Cloudberry build complete. ===" + + - name: Verify Cloudberry installation + shell: bash + run: | + set -euo pipefail + echo "Verifying Cloudberry installation..." + if [[ ! -f /usr/local/cloudberry-db/cloudberry-env.sh ]]; then + echo "::error::Cloudberry installation not found at /usr/local/cloudberry-db" + exit 1 + fi + ls -la /usr/local/cloudberry-db/bin/ + echo "Cloudberry installation verified." + + # ---------------------------------------------------------------- + # Phase B: Download & test the cloudberry-backup package + # ---------------------------------------------------------------- + + - name: Download backup package + uses: actions/download-artifact@v4 + with: + name: packages-${{ matrix.backup_arch }} + path: ${{ github.workspace }}/package-artifacts + + - name: Verify backup package checksum + id: verify-package + shell: bash + run: | + set -euo pipefail + artifact_dir="${GITHUB_WORKSPACE}/package-artifacts" + + package_file=$(ls "${artifact_dir}"/*.tar.gz 2>/dev/null | head -1 || true) + if [[ -z "${package_file}" || ! -f "${package_file}" ]]; then + echo "::error::No tar.gz package found in ${artifact_dir}" + ls -la "${artifact_dir}" || true + exit 1 + fi + + checksum_file="${package_file}.sha512" + if [[ ! -f "${checksum_file}" ]]; then + echo "::error::Checksum file not found: ${checksum_file}" + exit 1 + fi + + ( cd "${artifact_dir}" && sha512sum -c "$(basename "${checksum_file}")" ) + echo "package_file=${package_file}" >> "${GITHUB_OUTPUT}" + echo "Checksum verification passed." + + - name: Extract and install package + shell: bash + env: + PACKAGE_FILE: ${{ steps.verify-package.outputs.package_file }} + run: | + set -euo pipefail + + work_dir="${GITHUB_WORKSPACE}/pkg-test" + mkdir -p "${work_dir}" + + echo "=== Extracting package ===" + tar -xzf "${PACKAGE_FILE}" -C "${work_dir}" + + extracted_dir=$(ls -1d "${work_dir}"/apache-cloudberry-backup-incubating-*/ 2>/dev/null | head -1 || true) + if [[ -z "${extracted_dir}" ]]; then + echo "::error::Extracted package directory not found" + ls -la "${work_dir}" + exit 1 + fi + + echo "Package extracted to: ${extracted_dir}" + echo "Extracted contents:" + ls -lhR "${extracted_dir}" + + echo "=== Installing via install.sh ===" + chmod +x "${extracted_dir}/install.sh" + sed -i 's/sudo //g' "${extracted_dir}/install.sh" + export GPHOME="/usr/local/cloudberry-db" + "${extracted_dir}/install.sh" 2>&1 + echo "Installation complete." + + - name: Verify installed commands + shell: bash + run: | + set -euo pipefail + GPHOME="/usr/local/cloudberry-db" + expected="gpbackup gprestore gpbackup_helper gpbackup_s3_plugin gpbackman gpbackup_exporter" + + echo "=== Verifying installed binaries ===" + for binary in ${expected}; do + bin_path="${GPHOME}/bin/${binary}" + if [[ ! -f "${bin_path}" ]]; then + echo "::warning::${binary} not found — may be absent in older releases" + continue + fi + version_output=$("${bin_path}" --version 2>&1) || { + echo "::warning::${binary} --version failed" + continue + } + echo " ${binary}: ${version_output}" + done + echo "Command verification complete." + + - name: Run backup/restore functional test + shell: bash + env: + TARGET_OS: ${{ matrix.target_os }} + ARCH: ${{ matrix.arch }} + run: | + set -euo pipefail + + work_dir="${GITHUB_WORKSPACE}/functional-test" + mkdir -p "${work_dir}" + chown -R gpadmin:gpadmin "${GITHUB_WORKSPACE}" + + cat <<'SCRIPT' > /tmp/run_backup_restore_test.sh + #!/bin/bash + set -euo pipefail + + GPHOME="/usr/local/cloudberry-db" + + source "${GPHOME}/cloudberry-env.sh" + + echo "=== Creating demo cluster ===" + cd "${HOME}" + gpdemo + source "${HOME}/gpdemo-env.sh" + + echo "=== Verifying cluster is running ===" + psql -d postgres -c "SELECT 1 AS cluster_ok;" + psql -d postgres -c "SELECT version();" + + echo "=== Preparing test database ===" + test_db="package_test_db" + psql -d postgres -c "DROP DATABASE IF EXISTS ${test_db}" + createdb "${test_db}" + psql -d "${test_db}" -c "CREATE TABLE test_tbl (id int, name text) DISTRIBUTED RANDOMLY;" + psql -d "${test_db}" -c "INSERT INTO test_tbl VALUES (1, 'hello'), (2, 'world');" + psql -d "${test_db}" -c "SELECT count(*) AS row_count FROM test_tbl;" + + echo "=== Running gpbackup ===" + backup_dir="/tmp/backup_restore_test" + rm -rf "${backup_dir}" + mkdir -p "${backup_dir}" + + backup_log="${backup_dir}/gpbackup_output.log" + gpbackup --dbname "${test_db}" --backup-dir "${backup_dir}" 2>&1 | tee "${backup_log}" + + timestamp=$(grep -E "Backup Timestamp[[:space:]]*=" "${backup_log}" | grep -Eo "[[:digit:]]{14}" | head -1 || true) + + if [[ -z "${timestamp}" ]]; then + latest_gpbackup_log=$(ls -1t "${HOME}/gpAdminLogs"/gpbackup_*.log 2>/dev/null | head -1 || true) + if [[ -n "${latest_gpbackup_log}" ]]; then + timestamp=$(grep -E "Backup Timestamp[[:space:]]*=" "${latest_gpbackup_log}" | grep -Eo "[[:digit:]]{14}" | head -1 || true) + fi + fi + + if [[ -z "${timestamp}" ]]; then + echo "ERROR: Could not parse backup timestamp from gpbackup logs" + echo "=== Backup log ===" + cat "${backup_log}" || true + echo "=== Backup directory contents ===" + find "${backup_dir}" -type f | sort + exit 1 + fi + + echo "Backup timestamp: ${timestamp}" + + echo "=== Dropping test database ===" + dropdb "${test_db}" + + echo "=== Running gprestore ===" + gprestore --timestamp "${timestamp}" --backup-dir "${backup_dir}" --create-db --on-error-continue 2>&1 + + echo "=== Verification: connecting to restored database ===" + row_count=$(psql -d "${test_db}" -t -c "SELECT count(*) FROM test_tbl" | xargs) + if [[ "${row_count}" != "2" ]]; then + echo "ERROR: Expected 2 rows in restored test_tbl, got ${row_count}" + exit 1 + fi + echo "Restore verified: ${row_count} rows in test_tbl" + + echo "=== All functional tests passed ===" + SCRIPT + + chmod +x /tmp/run_backup_restore_test.sh + + set +e + su - gpadmin -c "/tmp/run_backup_restore_test.sh" + test_status=$? + set -e + + { + echo "## Cloudberry-backup package functional test" + echo "- Target: ${{ matrix.target_os }}/${{ matrix.arch }}" + echo "- Package: ${{ steps.verify-package.outputs.package_file }}" + if [[ ${test_status} -eq 0 ]]; then + echo "- Result: PASS" + else + echo "- Result: FAIL" + fi + } >> "${GITHUB_STEP_SUMMARY}" + + exit ${test_status}