From 631bb657eb59d0b952cc58aa52b4c61c9e07a7e9 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Tue, 24 Sep 2024 02:06:13 -0700 Subject: [PATCH] moving tests to endtoend/backup/xtrabackup Signed-off-by: Renan Rangel --- .../backup/vtctlbackup/backup_test.go | 137 ------------------ .../backup/vtctlbackup/backup_utils.go | 129 +++++++++++++++++ .../backup/xtrabackup/select_engine_test.go | 17 +++ 3 files changed, 146 insertions(+), 137 deletions(-) create mode 100644 go/test/endtoend/backup/xtrabackup/select_engine_test.go diff --git a/go/test/endtoend/backup/vtctlbackup/backup_test.go b/go/test/endtoend/backup/vtctlbackup/backup_test.go index d31c5c4258e..be9e5bbf490 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_test.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_test.go @@ -17,14 +17,8 @@ limitations under the License. package vtctlbackup import ( - "fmt" - "os" - "path" "testing" - "github.com/stretchr/testify/require" - - "vitess.io/vitess/go/test/endtoend/cluster" "vitess.io/vitess/go/vt/mysqlctl" ) @@ -79,134 +73,3 @@ func setDefaultCompressionFlag() { mysqlctl.ExternalDecompressorCmd = "" mysqlctl.ManifestExternalDecompressorCmd = "" } - -func setDefaultCommonArgs() { commonTabletArg = getDefaultCommonArgs() } - -func TestBackupEngineSelector(t *testing.T) { - defer setDefaultCompressionFlag() - defer setDefaultCommonArgs() - defer cluster.PanicHandler(t) - - // launch the custer with xtrabackup as the default engine - code, err := LaunchCluster(XtraBackup, "xbstream", 0, &CompressionDetails{CompressorEngineName: "pgzip"}) - require.Nilf(t, err, "setup failed with status code %d", code) - - defer TearDownCluster() - - localCluster.DisableVTOrcRecoveries(t) - defer func() { - localCluster.EnableVTOrcRecoveries(t) - }() - verifyInitialReplication(t) - - // first try to backup with an alternative engine (builtin) - err = localCluster.VtctldClientProcess.ExecuteCommand("Backup", "--allow-primary", "--backup-engine=builtin", primary.Alias) - require.NoError(t, err) - engineUsed := getBackupEngineOfLastBackup(t) - require.Equal(t, "builtin", engineUsed) - - // then try to backup specifying the xtrabackup engine - err = localCluster.VtctldClientProcess.ExecuteCommand("Backup", "--allow-primary", "--backup-engine=xtrabackup", primary.Alias) - require.NoError(t, err) - engineUsed = getBackupEngineOfLastBackup(t) - require.Equal(t, "xtrabackup", engineUsed) - - // check that by default we still use the xtrabackup engine if not specified - err = localCluster.VtctldClientProcess.ExecuteCommand("Backup", "--allow-primary", primary.Alias) - require.NoError(t, err) - engineUsed = getBackupEngineOfLastBackup(t) - require.Equal(t, "xtrabackup", engineUsed) -} - -// fetch the backup engine used on the last backup triggered by the end-to-end tests. -func getBackupEngineOfLastBackup(t *testing.T) string { - lastBackup := getLastBackup(t) - - manifest := readManifestFile(t, path.Join(localCluster.CurrentVTDATAROOT, "backups", keyspaceName, shardName, lastBackup)) - - return manifest.BackupMethod -} - -func getLastBackup(t *testing.T) string { - backups, err := localCluster.ListBackups(shardKsName) - require.NoError(t, err) - - return backups[len(backups)-1] -} - -func TestRestoreIgnoreBackups(t *testing.T) { - defer setDefaultCompressionFlag() - defer setDefaultCommonArgs() - defer cluster.PanicHandler(t) - - backupMsg := "right after xtrabackup backup" - - cDetails := &CompressionDetails{CompressorEngineName: "pgzip"} - - // launch the custer with xtrabackup as the default engine - code, err := LaunchCluster(XtraBackup, "xbstream", 0, cDetails) - require.Nilf(t, err, "setup failed with status code %d", code) - - defer TearDownCluster() - - localCluster.DisableVTOrcRecoveries(t) - defer func() { - localCluster.EnableVTOrcRecoveries(t) - }() - verifyInitialReplication(t) - - t.Run("generate backups", func(t *testing.T) { - // lets take two backups, each using a different backup engine - err = localCluster.VtctldClientProcess.ExecuteCommand("Backup", "--allow-primary", "--backup-engine=builtin", primary.Alias) - require.NoError(t, err) - - err = localCluster.VtctldClientProcess.ExecuteCommand("Backup", "--allow-primary", "--backup-engine=xtrabackup", primary.Alias) - require.NoError(t, err) - }) - - // insert more data on the primary - _, err = primary.VttabletProcess.QueryTablet(fmt.Sprintf("insert into vt_insert_test (msg) values ('%s')", backupMsg), keyspaceName, true) - require.NoError(t, err) - - t.Run("restore replica and verify data", func(t *testing.T) { - // now bring up another replica, letting it restore from backup. - restoreWaitForBackup(t, "replica", cDetails, true) - err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, timeout) - require.NoError(t, err) - - // check the new replica has the data - cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) - result, err := replica2.VttabletProcess.QueryTablet( - fmt.Sprintf("select msg from vt_insert_test where msg='%s'", backupMsg), replica2.VttabletProcess.Keyspace, true) - require.NoError(t, err) - require.Equal(t, backupMsg, result.Named().Row().AsString("msg", "")) - }) - - t.Run("test broken restore", func(t *testing.T) { - // now lets break the last backup in the shard - err = os.Remove(path.Join(localCluster.CurrentVTDATAROOT, - "backups", keyspaceName, shardName, - getLastBackup(t), "backup.xbstream.gz")) - require.NoError(t, err) - - // and try to restore from it - err = localCluster.VtctldClientProcess.ExecuteCommand("RestoreFromBackup", replica2.Alias) - require.Error(t, err) // this should fail - }) - - t.Run("test older working backup", func(t *testing.T) { - // now we retry but trying the earlier backup - err = localCluster.VtctldClientProcess.ExecuteCommand("RestoreFromBackup", "--allowed-backup-engines=builtin", replica2.Alias) - require.NoError(t, err) // this should succeed - - // make sure we are replicating after the restore is done - err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, timeout) - require.NoError(t, err) - cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) - - result, err := replica2.VttabletProcess.QueryTablet( - fmt.Sprintf("select msg from vt_insert_test where msg='%s'", backupMsg), replica2.VttabletProcess.Keyspace, true) - require.NoError(t, err) - require.Equal(t, backupMsg, result.Named().Row().AsString("msg", "")) - }) -} diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index 2ffe6e776a5..98c8dc5d24b 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -1449,3 +1449,132 @@ func getDefaultCommonArgs() []string { "--serving_state_grace_period", "1s", } } + +func setDefaultCommonArgs() { commonTabletArg = getDefaultCommonArgs() } + +// fetch the backup engine used on the last backup triggered by the end-to-end tests. +func getBackupEngineOfLastBackup(t *testing.T) string { + lastBackup := getLastBackup(t) + + manifest := readManifestFile(t, path.Join(localCluster.CurrentVTDATAROOT, "backups", keyspaceName, shardName, lastBackup)) + + return manifest.BackupMethod +} + +func getLastBackup(t *testing.T) string { + backups, err := localCluster.ListBackups(shardKsName) + require.NoError(t, err) + + return backups[len(backups)-1] +} + +func TestBackupEngineSelector(t *testing.T) { + defer setDefaultCommonArgs() + defer cluster.PanicHandler(t) + + // launch the custer with xtrabackup as the default engine + code, err := LaunchCluster(XtraBackup, "xbstream", 0, &CompressionDetails{CompressorEngineName: "pgzip"}) + require.Nilf(t, err, "setup failed with status code %d", code) + + defer TearDownCluster() + + localCluster.DisableVTOrcRecoveries(t) + defer func() { + localCluster.EnableVTOrcRecoveries(t) + }() + verifyInitialReplication(t) + + // first try to backup with an alternative engine (builtin) + err = localCluster.VtctldClientProcess.ExecuteCommand("Backup", "--allow-primary", "--backup-engine=builtin", primary.Alias) + require.NoError(t, err) + engineUsed := getBackupEngineOfLastBackup(t) + require.Equal(t, "builtin", engineUsed) + + // then try to backup specifying the xtrabackup engine + err = localCluster.VtctldClientProcess.ExecuteCommand("Backup", "--allow-primary", "--backup-engine=xtrabackup", primary.Alias) + require.NoError(t, err) + engineUsed = getBackupEngineOfLastBackup(t) + require.Equal(t, "xtrabackup", engineUsed) + + // check that by default we still use the xtrabackup engine if not specified + err = localCluster.VtctldClientProcess.ExecuteCommand("Backup", "--allow-primary", primary.Alias) + require.NoError(t, err) + engineUsed = getBackupEngineOfLastBackup(t) + require.Equal(t, "xtrabackup", engineUsed) +} + +func TestRestoreAllowedBackupEngines(t *testing.T) { + defer setDefaultCommonArgs() + defer cluster.PanicHandler(t) + + backupMsg := "right after xtrabackup backup" + + cDetails := &CompressionDetails{CompressorEngineName: "pgzip"} + + // launch the custer with xtrabackup as the default engine + code, err := LaunchCluster(XtraBackup, "xbstream", 0, cDetails) + require.Nilf(t, err, "setup failed with status code %d", code) + + defer TearDownCluster() + + localCluster.DisableVTOrcRecoveries(t) + defer func() { + localCluster.EnableVTOrcRecoveries(t) + }() + verifyInitialReplication(t) + + t.Run("generate backups", func(t *testing.T) { + // lets take two backups, each using a different backup engine + err = localCluster.VtctldClientProcess.ExecuteCommand("Backup", "--allow-primary", "--backup-engine=builtin", primary.Alias) + require.NoError(t, err) + + err = localCluster.VtctldClientProcess.ExecuteCommand("Backup", "--allow-primary", "--backup-engine=xtrabackup", primary.Alias) + require.NoError(t, err) + }) + + // insert more data on the primary + _, err = primary.VttabletProcess.QueryTablet(fmt.Sprintf("insert into vt_insert_test (msg) values ('%s')", backupMsg), keyspaceName, true) + require.NoError(t, err) + + t.Run("restore replica and verify data", func(t *testing.T) { + // now bring up another replica, letting it restore from backup. + restoreWaitForBackup(t, "replica", cDetails, true) + err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, timeout) + require.NoError(t, err) + + // check the new replica has the data + cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) + result, err := replica2.VttabletProcess.QueryTablet( + fmt.Sprintf("select msg from vt_insert_test where msg='%s'", backupMsg), replica2.VttabletProcess.Keyspace, true) + require.NoError(t, err) + require.Equal(t, backupMsg, result.Named().Row().AsString("msg", "")) + }) + + t.Run("test broken restore", func(t *testing.T) { + // now lets break the last backup in the shard + err = os.Remove(path.Join(localCluster.CurrentVTDATAROOT, + "backups", keyspaceName, shardName, + getLastBackup(t), "backup.xbstream.gz")) + require.NoError(t, err) + + // and try to restore from it + err = localCluster.VtctldClientProcess.ExecuteCommand("RestoreFromBackup", replica2.Alias) + require.Error(t, err) // this should fail + }) + + t.Run("test older working backup", func(t *testing.T) { + // now we retry but with the first backup + err = localCluster.VtctldClientProcess.ExecuteCommand("RestoreFromBackup", "--allowed-backup-engines=builtin", replica2.Alias) + require.NoError(t, err) // this should succeed + + // make sure we are replicating after the restore is done + err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, timeout) + require.NoError(t, err) + cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) + + result, err := replica2.VttabletProcess.QueryTablet( + fmt.Sprintf("select msg from vt_insert_test where msg='%s'", backupMsg), replica2.VttabletProcess.Keyspace, true) + require.NoError(t, err) + require.Equal(t, backupMsg, result.Named().Row().AsString("msg", "")) + }) +} diff --git a/go/test/endtoend/backup/xtrabackup/select_engine_test.go b/go/test/endtoend/backup/xtrabackup/select_engine_test.go new file mode 100644 index 00000000000..582cb58d874 --- /dev/null +++ b/go/test/endtoend/backup/xtrabackup/select_engine_test.go @@ -0,0 +1,17 @@ +package vtctlbackup + +import ( + "testing" + + backup "vitess.io/vitess/go/test/endtoend/backup/vtctlbackup" +) + +func TestBackupEngineSelector(t *testing.T) { + defer setDefaultCompressionFlag() + backup.TestBackupEngineSelector(t) +} + +func TestRestoreAllowedBackupEngines(t *testing.T) { + defer setDefaultCompressionFlag() + backup.TestRestoreAllowedBackupEngines(t) +}