feat: add ability to ignore version mismatch check

cmd/db.go

@@ -121,6 +121,7 @@ var (
 	}
 
 	dryRun       bool
+	includeAll   bool
 	includeRoles bool
 	includeSeed  bool
 
@@ -133,7 +134,7 @@ var (
 				return err
 			}
 			ctx, _ := signal.NotifyContext(cmd.Context(), os.Interrupt)
-			return push.Run(ctx, dryRun, includeRoles, includeSeed, dbConfig, fsys)
+			return push.Run(ctx, dryRun, includeAll, includeRoles, includeSeed, dbConfig, fsys)
 		},
 	}
 
@@ -263,6 +264,7 @@ func init() {
 	dbCmd.AddCommand(dbDumpCmd)
 	// Build push command
 	pushFlags := dbPushCmd.Flags()
+	pushFlags.BoolVar(&includeAll, "include-all", false, "Include all migrations not found on remote history table.")
 	pushFlags.BoolVar(&includeRoles, "include-roles", false, "Include custom roles from "+utils.CustomRolesPath+".")
 	pushFlags.BoolVar(&includeSeed, "include-seed", false, "Include seed data from "+utils.SeedDataPath+".")
 	pushFlags.BoolVar(&dryRun, "dry-run", false, "Print the migrations that would be applied, but don't actually apply them.")

cmd/migration.go

@@ -95,7 +95,7 @@ var (
 		Short: "Apply pending migrations to local database",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx, _ := signal.NotifyContext(cmd.Context(), os.Interrupt)
-			return up.Run(ctx, afero.NewOsFs())
+			return up.Run(ctx, includeAll, afero.NewOsFs())
 		},
 		PostRun: func(cmd *cobra.Command, args []string) {
 			fmt.Println("Local database is up to date.")
@@ -131,6 +131,8 @@ func init() {
 	migrationSquashCmd.MarkFlagsMutuallyExclusive("db-url", "linked")
 	migrationCmd.AddCommand(migrationSquashCmd)
 	// Build up command
+	upFlags := migrationUpCmd.Flags()
+	upFlags.BoolVar(&includeAll, "include-all", false, "Include all migrations not found on remote history table.")
 	migrationCmd.AddCommand(migrationUpCmd)
 	// Build new command
 	migrationCmd.AddCommand(migrationNewCmd)

internal/db/push/push.go

@@ -16,7 +16,7 @@ import (
 	"github.com/supabase/cli/internal/utils"
 )
 
-func Run(ctx context.Context, dryRun, includeRoles, includeSeed bool, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {
+func Run(ctx context.Context, dryRun, ignoreVersionMismatch bool, includeRoles, includeSeed bool, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {
 	if dryRun {
 		fmt.Fprintln(os.Stderr, "DRY RUN: migrations will *not* be pushed to the database.")
 	}
@@ -31,7 +31,7 @@ func Run(ctx context.Context, dryRun, includeRoles, includeSeed bool, config pgc
 			return err
 		}
 	}
-	pending, err := up.GetPendingMigrations(ctx, conn, fsys)
+	pending, err := up.GetPendingMigrations(ctx, ignoreVersionMismatch, conn, fsys)
 	if err != nil {
 		return err
 	}

internal/db/push/push_test.go

@@ -36,7 +36,7 @@ func TestMigrationPush(t *testing.T) {
 		conn.Query(list.LIST_MIGRATION_VERSION).
 			Reply("SELECT 0")
 		// Run test
-		err := Run(context.Background(), true, false, false, dbConfig, fsys, conn.Intercept)
+		err := Run(context.Background(), true, false, false, false, dbConfig, fsys, conn.Intercept)
 		// Check error
 		assert.NoError(t, err)
 	})
@@ -50,7 +50,7 @@ func TestMigrationPush(t *testing.T) {
 		conn.Query(list.LIST_MIGRATION_VERSION).
 			Reply("SELECT 0")
 		// Run test
-		err := Run(context.Background(), false, false, false, dbConfig, fsys, conn.Intercept)
+		err := Run(context.Background(), false, false, false, false, dbConfig, fsys, conn.Intercept)
 		// Check error
 		assert.NoError(t, err)
 	})
@@ -59,7 +59,7 @@ func TestMigrationPush(t *testing.T) {
 		// Setup in-memory fs
 		fsys := afero.NewMemMapFs()
 		// Run test
-		err := Run(context.Background(), false, false, false, pgconn.Config{}, fsys)
+		err := Run(context.Background(), false, false, false, false, pgconn.Config{}, fsys)
 		// Check error
 		assert.ErrorContains(t, err, "invalid port (outside range)")
 	})
@@ -73,7 +73,7 @@ func TestMigrationPush(t *testing.T) {
 		conn.Query(list.LIST_MIGRATION_VERSION).
 			ReplyError(pgerrcode.InvalidCatalogName, `database "target" does not exist`)
 		// Run test
-		err := Run(context.Background(), false, false, false, dbConfig, fsys, conn.Intercept)
+		err := Run(context.Background(), false, false, false, false, dbConfig, fsys, conn.Intercept)
 		// Check error
 		assert.ErrorContains(t, err, `ERROR: database "target" does not exist (SQLSTATE 3D000)`)
 	})
@@ -95,7 +95,7 @@ func TestMigrationPush(t *testing.T) {
 			Query(repair.ADD_STATEMENTS_COLUMN).
 			Query(repair.ADD_NAME_COLUMN)
 		// Run test
-		err := Run(context.Background(), false, false, false, dbConfig, fsys, conn.Intercept)
+		err := Run(context.Background(), false, false, false, false, dbConfig, fsys, conn.Intercept)
 		// Check error
 		assert.ErrorContains(t, err, `ERROR: permission denied for relation supabase_migrations (SQLSTATE 42501)`)
 	})
@@ -121,7 +121,7 @@ func TestMigrationPush(t *testing.T) {
 			Query(repair.INSERT_MIGRATION_VERSION, "0", "test", "{}").
 			ReplyError(pgerrcode.NotNullViolation, `null value in column "version" of relation "schema_migrations"`)
 		// Run test
-		err := Run(context.Background(), false, false, false, dbConfig, fsys, conn.Intercept)
+		err := Run(context.Background(), false, false, false, false, dbConfig, fsys, conn.Intercept)
 		// Check error
 		assert.ErrorContains(t, err, `ERROR: null value in column "version" of relation "schema_migrations" (SQLSTATE 23502)`)
 		assert.ErrorContains(t, err, "At statement 0: "+repair.INSERT_MIGRATION_VERSION)

internal/migration/up/up.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"path/filepath"
 
 	"github.com/jackc/pgconn"
 	"github.com/jackc/pgx/v4"
@@ -13,9 +14,12 @@ import (
 	"github.com/supabase/cli/internal/utils"
 )
 
-var errConflict = errors.New("supabase_migrations.schema_migrations table conflicts with the contents of " + utils.Bold(utils.MigrationsDir) + ".")
+var (
+	errMissingRemote = errors.New("Found local migration files to be inserted before the last migration on remote database.")
+	errMissingLocal  = errors.New("Remote migration versions not found in " + utils.MigrationsDir + " directory.")
+)
 
-func Run(ctx context.Context, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {
+func Run(ctx context.Context, includeAll bool, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {
 	if err := utils.LoadConfigFS(fsys); err != nil {
 		return err
 	}
@@ -24,14 +28,14 @@ func Run(ctx context.Context, fsys afero.Fs, options ...func(*pgx.ConnConfig)) e
 		return err
 	}
 	defer conn.Close(context.Background())
-	pending, err := GetPendingMigrations(ctx, conn, fsys)
+	pending, err := GetPendingMigrations(ctx, includeAll, conn, fsys)
 	if err != nil {
 		return err
 	}
 	return apply.MigrateUp(ctx, conn, pending, fsys)
 }
 
-func GetPendingMigrations(ctx context.Context, conn *pgx.Conn, fsys afero.Fs) ([]string, error) {
+func GetPendingMigrations(ctx context.Context, includeAll bool, conn *pgx.Conn, fsys afero.Fs) ([]string, error) {
 	remoteMigrations, err := list.LoadRemoteMigrations(ctx, conn)
 	if err != nil {
 		return nil, err
@@ -40,17 +44,47 @@ func GetPendingMigrations(ctx context.Context, conn *pgx.Conn, fsys afero.Fs) ([
 	if err != nil {
 		return nil, err
 	}
-	// Check remote is in-sync or behind local
-	if len(remoteMigrations) > len(localMigrations) {
-		return nil, fmt.Errorf("%w; Found %d versions and %d migrations.", errConflict, len(remoteMigrations), len(localMigrations))
-	}
+	// Find local migrations older than the last migration on remote
+	var unapplied []string
 	for i, remote := range remoteMigrations {
-		filename := localMigrations[i]
-		// LoadLocalMigrations guarantees we always have a match
-		local := utils.MigrateFilePattern.FindStringSubmatch(filename)[1]
-		if remote != local {
-			return nil, fmt.Errorf("%w; Expected version %s but found migration %s at index %d.", errConflict, remote, filename, i)
+		for _, filename := range localMigrations[i+len(unapplied):] {
+			// Check if migration has been applied before, LoadLocalMigrations guarantees a match
+			local := utils.MigrateFilePattern.FindStringSubmatch(filename)[1]
+			if remote == local {
+				break
+			}
+			// Include out-of-order local migrations
+			unapplied = append(unapplied, filename)
+		}
+		// Check if all remote versions exist in local
+		if i+len(unapplied) >= len(localMigrations) {
+			utils.CmdSuggestion = suggestRevertHistory(remoteMigrations[i:])
+			return nil, errMissingLocal
 		}
 	}
-	return localMigrations[len(remoteMigrations):], nil
+	// Enforce migrations are applied in chronological order by default
+	if !includeAll && len(unapplied) > 0 {
+		utils.CmdSuggestion = suggestIgnoreFlag(unapplied)
+		return nil, errMissingRemote
+	}
+	pending := localMigrations[len(remoteMigrations)+len(unapplied):]
+	return append(unapplied, pending...), nil
+}
+
+func suggestRevertHistory(versions []string) string {
+	result := fmt.Sprintln("\nMake sure your local git repo is up-to-date. If the error persists, try repairing the migration history table:")
+	for _, ver := range versions {
+		result += fmt.Sprintln(utils.Bold("supabase migration repair --status reverted " + ver))
+	}
+	result += fmt.Sprintln("\nAnd update local migrations to match remote database:")
+	result += fmt.Sprintln(utils.Bold("supabase db remote commit"))
+	return result
+}
+
+func suggestIgnoreFlag(filenames []string) string {
+	result := "\nRerun the command with --include-all flag to apply these migrations:\n"
+	for _, name := range filenames {
+		result += fmt.Sprintln(utils.Bold(filepath.Join(utils.MigrationsDir, name)))
+	}
+	return result
 }

internal/migration/up/up_test.go

@@ -39,7 +39,7 @@ func TestPendingMigrations(t *testing.T) {
 		require.NoError(t, err)
 		defer mock.Close(ctx)
 		// Run test
-		pending, err := GetPendingMigrations(ctx, mock, fsys)
+		pending, err := GetPendingMigrations(ctx, false, mock, fsys)
 		// Check error
 		assert.NoError(t, err)
 		assert.ElementsMatch(t, files[2:], pending)
@@ -59,12 +59,12 @@ func TestPendingMigrations(t *testing.T) {
 		require.NoError(t, err)
 		defer mock.Close(ctx)
 		// Run test
-		_, err = GetPendingMigrations(ctx, mock, fsys)
+		_, err = GetPendingMigrations(ctx, false, mock, fsys)
 		// Check error
 		assert.ErrorContains(t, err, "operation not permitted")
 	})
 
-	t.Run("throws error on missing migration", func(t *testing.T) {
+	t.Run("throws error on missing local migration", func(t *testing.T) {
 		// Setup in-memory fs
 		fsys := afero.NewMemMapFs()
 		// Setup mock postgres
@@ -78,29 +78,85 @@ func TestPendingMigrations(t *testing.T) {
 		require.NoError(t, err)
 		defer mock.Close(ctx)
 		// Run test
-		_, err = GetPendingMigrations(ctx, mock, fsys)
+		_, err = GetPendingMigrations(ctx, false, mock, fsys)
 		// Check error
-		assert.ErrorContains(t, err, "Found 1 versions and 0 migrations.")
+		assert.ErrorIs(t, err, errMissingLocal)
 	})
 
-	t.Run("throws error on version mismatch", func(t *testing.T) {
+	t.Run("throws error on missing remote version", func(t *testing.T) {
 		// Setup in-memory fs
 		fsys := afero.NewMemMapFs()
-		path := filepath.Join(utils.MigrationsDir, "1_test.sql")
+		files := []string{"0_test.sql", "1_test.sql"}
+		for _, name := range files {
+			path := filepath.Join(utils.MigrationsDir, name)
+			require.NoError(t, afero.WriteFile(fsys, path, []byte(""), 0644))
+		}
+		// Setup mock postgres
+		conn := pgtest.NewConn()
+		defer conn.Close(t)
+		conn.Query(list.LIST_MIGRATION_VERSION).
+			Reply("SELECT 1", []interface{}{"1"})
+		// Connect to mock
+		ctx := context.Background()
+		mock, err := utils.ConnectLocalPostgres(ctx, pgconn.Config{Port: 5432}, conn.Intercept)
+		require.NoError(t, err)
+		defer mock.Close(ctx)
+		// Run test
+		_, err = GetPendingMigrations(ctx, false, mock, fsys)
+		// Check error
+		assert.ErrorIs(t, err, errMissingRemote)
+	})
+}
+
+func TestIgnoreVersionMismatch(t *testing.T) {
+	t.Run("applies out-of-order local migrations", func(t *testing.T) {
+		// Setup in-memory fs
+		fsys := afero.NewMemMapFs()
+		files := []string{
+			"20221201000000_test.sql",
+			"20221201000001_test.sql",
+			"20221201000002_test.sql",
+			"20221201000003_test.sql",
+		}
+		for _, name := range files {
+			path := filepath.Join(utils.MigrationsDir, name)
+			require.NoError(t, afero.WriteFile(fsys, path, []byte(""), 0644))
+		}
+		// Setup mock postgres
+		conn := pgtest.NewConn()
+		defer conn.Close(t)
+		conn.Query(list.LIST_MIGRATION_VERSION).
+			Reply("SELECT 2", []interface{}{"20221201000000"}, []interface{}{"20221201000002"})
+		// Connect to mock
+		ctx := context.Background()
+		mock, err := utils.ConnectLocalPostgres(ctx, pgconn.Config{Port: 5432}, conn.Intercept)
+		require.NoError(t, err)
+		defer mock.Close(ctx)
+		// Run test
+		pending, err := GetPendingMigrations(ctx, true, mock, fsys)
+		// Check error
+		assert.NoError(t, err)
+		assert.ElementsMatch(t, []string{files[1], files[3]}, pending)
+	})
+
+	t.Run("throws error on missing local migration", func(t *testing.T) {
+		// Setup in-memory fs
+		fsys := afero.NewMemMapFs()
+		path := filepath.Join(utils.MigrationsDir, "20221201000000_test.sql")
 		require.NoError(t, afero.WriteFile(fsys, path, []byte(""), 0644))
 		// Setup mock postgres
 		conn := pgtest.NewConn()
 		defer conn.Close(t)
 		conn.Query(list.LIST_MIGRATION_VERSION).
-			Reply("SELECT 1", []interface{}{"0"})
+			Reply("SELECT 1", []interface{}{"20221201000001"})
 		// Connect to mock
 		ctx := context.Background()
 		mock, err := utils.ConnectLocalPostgres(ctx, pgconn.Config{Port: 5432}, conn.Intercept)
 		require.NoError(t, err)
 		defer mock.Close(ctx)
 		// Run test
-		_, err = GetPendingMigrations(ctx, mock, fsys)
+		_, err = GetPendingMigrations(ctx, true, mock, fsys)
 		// Check error
-		assert.ErrorContains(t, err, "Expected version 0 but found migration 1_test.sql at index 0.")
+		assert.ErrorIs(t, err, errMissingLocal)
 	})
 }