diff --git a/cmd/db.go b/cmd/db.go index eed26c222..b5dbe1f93 100644 --- a/cmd/db.go +++ b/cmd/db.go @@ -121,6 +121,7 @@ var ( } dryRun bool + includeAll bool includeRoles bool includeSeed bool @@ -133,7 +134,7 @@ var ( return err } ctx, _ := signal.NotifyContext(cmd.Context(), os.Interrupt) - return push.Run(ctx, dryRun, includeRoles, includeSeed, dbConfig, fsys) + return push.Run(ctx, dryRun, includeAll, includeRoles, includeSeed, dbConfig, fsys) }, } @@ -263,6 +264,7 @@ func init() { dbCmd.AddCommand(dbDumpCmd) // Build push command pushFlags := dbPushCmd.Flags() + pushFlags.BoolVar(&includeAll, "include-all", false, "Include all migrations not found on remote history table.") pushFlags.BoolVar(&includeRoles, "include-roles", false, "Include custom roles from "+utils.CustomRolesPath+".") pushFlags.BoolVar(&includeSeed, "include-seed", false, "Include seed data from "+utils.SeedDataPath+".") pushFlags.BoolVar(&dryRun, "dry-run", false, "Print the migrations that would be applied, but don't actually apply them.") diff --git a/cmd/migration.go b/cmd/migration.go index 52f880ccb..353c33555 100644 --- a/cmd/migration.go +++ b/cmd/migration.go @@ -95,7 +95,7 @@ var ( Short: "Apply pending migrations to local database", RunE: func(cmd *cobra.Command, args []string) error { ctx, _ := signal.NotifyContext(cmd.Context(), os.Interrupt) - return up.Run(ctx, afero.NewOsFs()) + return up.Run(ctx, includeAll, afero.NewOsFs()) }, PostRun: func(cmd *cobra.Command, args []string) { fmt.Println("Local database is up to date.") @@ -131,6 +131,8 @@ func init() { migrationSquashCmd.MarkFlagsMutuallyExclusive("db-url", "linked") migrationCmd.AddCommand(migrationSquashCmd) // Build up command + upFlags := migrationUpCmd.Flags() + upFlags.BoolVar(&includeAll, "include-all", false, "Include all migrations not found on remote history table.") migrationCmd.AddCommand(migrationUpCmd) // Build new command migrationCmd.AddCommand(migrationNewCmd) diff --git a/internal/db/push/push.go b/internal/db/push/push.go index 216c10d0d..879fbec6d 100644 --- a/internal/db/push/push.go +++ b/internal/db/push/push.go @@ -16,7 +16,7 @@ import ( "github.com/supabase/cli/internal/utils" ) -func Run(ctx context.Context, dryRun, includeRoles, includeSeed bool, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { +func Run(ctx context.Context, dryRun, ignoreVersionMismatch bool, includeRoles, includeSeed bool, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { if dryRun { fmt.Fprintln(os.Stderr, "DRY RUN: migrations will *not* be pushed to the database.") } @@ -31,7 +31,7 @@ func Run(ctx context.Context, dryRun, includeRoles, includeSeed bool, config pgc return err } } - pending, err := up.GetPendingMigrations(ctx, conn, fsys) + pending, err := up.GetPendingMigrations(ctx, ignoreVersionMismatch, conn, fsys) if err != nil { return err } diff --git a/internal/db/push/push_test.go b/internal/db/push/push_test.go index e2d92044e..71e854f03 100644 --- a/internal/db/push/push_test.go +++ b/internal/db/push/push_test.go @@ -36,7 +36,7 @@ func TestMigrationPush(t *testing.T) { conn.Query(list.LIST_MIGRATION_VERSION). Reply("SELECT 0") // Run test - err := Run(context.Background(), true, false, false, dbConfig, fsys, conn.Intercept) + err := Run(context.Background(), true, false, false, false, dbConfig, fsys, conn.Intercept) // Check error assert.NoError(t, err) }) @@ -50,7 +50,7 @@ func TestMigrationPush(t *testing.T) { conn.Query(list.LIST_MIGRATION_VERSION). Reply("SELECT 0") // Run test - err := Run(context.Background(), false, false, false, dbConfig, fsys, conn.Intercept) + err := Run(context.Background(), false, false, false, false, dbConfig, fsys, conn.Intercept) // Check error assert.NoError(t, err) }) @@ -59,7 +59,7 @@ func TestMigrationPush(t *testing.T) { // Setup in-memory fs fsys := afero.NewMemMapFs() // Run test - err := Run(context.Background(), false, false, false, pgconn.Config{}, fsys) + err := Run(context.Background(), false, false, false, false, pgconn.Config{}, fsys) // Check error assert.ErrorContains(t, err, "invalid port (outside range)") }) @@ -73,7 +73,7 @@ func TestMigrationPush(t *testing.T) { conn.Query(list.LIST_MIGRATION_VERSION). ReplyError(pgerrcode.InvalidCatalogName, `database "target" does not exist`) // Run test - err := Run(context.Background(), false, false, false, dbConfig, fsys, conn.Intercept) + err := Run(context.Background(), false, false, false, false, dbConfig, fsys, conn.Intercept) // Check error assert.ErrorContains(t, err, `ERROR: database "target" does not exist (SQLSTATE 3D000)`) }) @@ -95,7 +95,7 @@ func TestMigrationPush(t *testing.T) { Query(repair.ADD_STATEMENTS_COLUMN). Query(repair.ADD_NAME_COLUMN) // Run test - err := Run(context.Background(), false, false, false, dbConfig, fsys, conn.Intercept) + err := Run(context.Background(), false, false, false, false, dbConfig, fsys, conn.Intercept) // Check error assert.ErrorContains(t, err, `ERROR: permission denied for relation supabase_migrations (SQLSTATE 42501)`) }) @@ -121,7 +121,7 @@ func TestMigrationPush(t *testing.T) { Query(repair.INSERT_MIGRATION_VERSION, "0", "test", "{}"). ReplyError(pgerrcode.NotNullViolation, `null value in column "version" of relation "schema_migrations"`) // Run test - err := Run(context.Background(), false, false, false, dbConfig, fsys, conn.Intercept) + err := Run(context.Background(), false, false, false, false, dbConfig, fsys, conn.Intercept) // Check error assert.ErrorContains(t, err, `ERROR: null value in column "version" of relation "schema_migrations" (SQLSTATE 23502)`) assert.ErrorContains(t, err, "At statement 0: "+repair.INSERT_MIGRATION_VERSION) diff --git a/internal/migration/up/up.go b/internal/migration/up/up.go index f2fadadad..334b1c6eb 100644 --- a/internal/migration/up/up.go +++ b/internal/migration/up/up.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "path/filepath" "github.com/jackc/pgconn" "github.com/jackc/pgx/v4" @@ -13,9 +14,12 @@ import ( "github.com/supabase/cli/internal/utils" ) -var errConflict = errors.New("supabase_migrations.schema_migrations table conflicts with the contents of " + utils.Bold(utils.MigrationsDir) + ".") +var ( + errMissingRemote = errors.New("Found local migration files to be inserted before the last migration on remote database.") + errMissingLocal = errors.New("Remote migration versions not found in " + utils.MigrationsDir + " directory.") +) -func Run(ctx context.Context, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { +func Run(ctx context.Context, includeAll bool, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error { if err := utils.LoadConfigFS(fsys); err != nil { return err } @@ -24,14 +28,14 @@ func Run(ctx context.Context, fsys afero.Fs, options ...func(*pgx.ConnConfig)) e return err } defer conn.Close(context.Background()) - pending, err := GetPendingMigrations(ctx, conn, fsys) + pending, err := GetPendingMigrations(ctx, includeAll, conn, fsys) if err != nil { return err } return apply.MigrateUp(ctx, conn, pending, fsys) } -func GetPendingMigrations(ctx context.Context, conn *pgx.Conn, fsys afero.Fs) ([]string, error) { +func GetPendingMigrations(ctx context.Context, includeAll bool, conn *pgx.Conn, fsys afero.Fs) ([]string, error) { remoteMigrations, err := list.LoadRemoteMigrations(ctx, conn) if err != nil { return nil, err @@ -40,17 +44,47 @@ func GetPendingMigrations(ctx context.Context, conn *pgx.Conn, fsys afero.Fs) ([ if err != nil { return nil, err } - // Check remote is in-sync or behind local - if len(remoteMigrations) > len(localMigrations) { - return nil, fmt.Errorf("%w; Found %d versions and %d migrations.", errConflict, len(remoteMigrations), len(localMigrations)) - } + // Find local migrations older than the last migration on remote + var unapplied []string for i, remote := range remoteMigrations { - filename := localMigrations[i] - // LoadLocalMigrations guarantees we always have a match - local := utils.MigrateFilePattern.FindStringSubmatch(filename)[1] - if remote != local { - return nil, fmt.Errorf("%w; Expected version %s but found migration %s at index %d.", errConflict, remote, filename, i) + for _, filename := range localMigrations[i+len(unapplied):] { + // Check if migration has been applied before, LoadLocalMigrations guarantees a match + local := utils.MigrateFilePattern.FindStringSubmatch(filename)[1] + if remote == local { + break + } + // Include out-of-order local migrations + unapplied = append(unapplied, filename) + } + // Check if all remote versions exist in local + if i+len(unapplied) >= len(localMigrations) { + utils.CmdSuggestion = suggestRevertHistory(remoteMigrations[i:]) + return nil, errMissingLocal } } - return localMigrations[len(remoteMigrations):], nil + // Enforce migrations are applied in chronological order by default + if !includeAll && len(unapplied) > 0 { + utils.CmdSuggestion = suggestIgnoreFlag(unapplied) + return nil, errMissingRemote + } + pending := localMigrations[len(remoteMigrations)+len(unapplied):] + return append(unapplied, pending...), nil +} + +func suggestRevertHistory(versions []string) string { + result := fmt.Sprintln("\nMake sure your local git repo is up-to-date. If the error persists, try repairing the migration history table:") + for _, ver := range versions { + result += fmt.Sprintln(utils.Bold("supabase migration repair --status reverted " + ver)) + } + result += fmt.Sprintln("\nAnd update local migrations to match remote database:") + result += fmt.Sprintln(utils.Bold("supabase db remote commit")) + return result +} + +func suggestIgnoreFlag(filenames []string) string { + result := "\nRerun the command with --include-all flag to apply these migrations:\n" + for _, name := range filenames { + result += fmt.Sprintln(utils.Bold(filepath.Join(utils.MigrationsDir, name))) + } + return result } diff --git a/internal/migration/up/up_test.go b/internal/migration/up/up_test.go index 1d9761b53..37cd429b3 100644 --- a/internal/migration/up/up_test.go +++ b/internal/migration/up/up_test.go @@ -39,7 +39,7 @@ func TestPendingMigrations(t *testing.T) { require.NoError(t, err) defer mock.Close(ctx) // Run test - pending, err := GetPendingMigrations(ctx, mock, fsys) + pending, err := GetPendingMigrations(ctx, false, mock, fsys) // Check error assert.NoError(t, err) assert.ElementsMatch(t, files[2:], pending) @@ -59,12 +59,12 @@ func TestPendingMigrations(t *testing.T) { require.NoError(t, err) defer mock.Close(ctx) // Run test - _, err = GetPendingMigrations(ctx, mock, fsys) + _, err = GetPendingMigrations(ctx, false, mock, fsys) // Check error assert.ErrorContains(t, err, "operation not permitted") }) - t.Run("throws error on missing migration", func(t *testing.T) { + t.Run("throws error on missing local migration", func(t *testing.T) { // Setup in-memory fs fsys := afero.NewMemMapFs() // Setup mock postgres @@ -78,29 +78,85 @@ func TestPendingMigrations(t *testing.T) { require.NoError(t, err) defer mock.Close(ctx) // Run test - _, err = GetPendingMigrations(ctx, mock, fsys) + _, err = GetPendingMigrations(ctx, false, mock, fsys) // Check error - assert.ErrorContains(t, err, "Found 1 versions and 0 migrations.") + assert.ErrorIs(t, err, errMissingLocal) }) - t.Run("throws error on version mismatch", func(t *testing.T) { + t.Run("throws error on missing remote version", func(t *testing.T) { // Setup in-memory fs fsys := afero.NewMemMapFs() - path := filepath.Join(utils.MigrationsDir, "1_test.sql") + files := []string{"0_test.sql", "1_test.sql"} + for _, name := range files { + path := filepath.Join(utils.MigrationsDir, name) + require.NoError(t, afero.WriteFile(fsys, path, []byte(""), 0644)) + } + // Setup mock postgres + conn := pgtest.NewConn() + defer conn.Close(t) + conn.Query(list.LIST_MIGRATION_VERSION). + Reply("SELECT 1", []interface{}{"1"}) + // Connect to mock + ctx := context.Background() + mock, err := utils.ConnectLocalPostgres(ctx, pgconn.Config{Port: 5432}, conn.Intercept) + require.NoError(t, err) + defer mock.Close(ctx) + // Run test + _, err = GetPendingMigrations(ctx, false, mock, fsys) + // Check error + assert.ErrorIs(t, err, errMissingRemote) + }) +} + +func TestIgnoreVersionMismatch(t *testing.T) { + t.Run("applies out-of-order local migrations", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + files := []string{ + "20221201000000_test.sql", + "20221201000001_test.sql", + "20221201000002_test.sql", + "20221201000003_test.sql", + } + for _, name := range files { + path := filepath.Join(utils.MigrationsDir, name) + require.NoError(t, afero.WriteFile(fsys, path, []byte(""), 0644)) + } + // Setup mock postgres + conn := pgtest.NewConn() + defer conn.Close(t) + conn.Query(list.LIST_MIGRATION_VERSION). + Reply("SELECT 2", []interface{}{"20221201000000"}, []interface{}{"20221201000002"}) + // Connect to mock + ctx := context.Background() + mock, err := utils.ConnectLocalPostgres(ctx, pgconn.Config{Port: 5432}, conn.Intercept) + require.NoError(t, err) + defer mock.Close(ctx) + // Run test + pending, err := GetPendingMigrations(ctx, true, mock, fsys) + // Check error + assert.NoError(t, err) + assert.ElementsMatch(t, []string{files[1], files[3]}, pending) + }) + + t.Run("throws error on missing local migration", func(t *testing.T) { + // Setup in-memory fs + fsys := afero.NewMemMapFs() + path := filepath.Join(utils.MigrationsDir, "20221201000000_test.sql") require.NoError(t, afero.WriteFile(fsys, path, []byte(""), 0644)) // Setup mock postgres conn := pgtest.NewConn() defer conn.Close(t) conn.Query(list.LIST_MIGRATION_VERSION). - Reply("SELECT 1", []interface{}{"0"}) + Reply("SELECT 1", []interface{}{"20221201000001"}) // Connect to mock ctx := context.Background() mock, err := utils.ConnectLocalPostgres(ctx, pgconn.Config{Port: 5432}, conn.Intercept) require.NoError(t, err) defer mock.Close(ctx) // Run test - _, err = GetPendingMigrations(ctx, mock, fsys) + _, err = GetPendingMigrations(ctx, true, mock, fsys) // Check error - assert.ErrorContains(t, err, "Expected version 0 but found migration 1_test.sql at index 0.") + assert.ErrorIs(t, err, errMissingLocal) }) }