diff --git a/README.md b/README.md index efedf5ba0..e60d33c79 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,7 @@ Goose is a database migration tool. Manage your database schema by creating incr - goose pkg doesn't register any SQL drivers anymore, thus no driver `panic()` conflict within your codebase! - goose pkg doesn't have any vendor dependencies anymore -- We encourage using sequential versioning of migration files - (rather than timestamps-based versioning) to prevent version - mismatch and migration colissions +- We use timestamped migrations by default but recommend a hybrid approach of using timestamps in the development process and sequential versions in production. # Install @@ -51,7 +49,7 @@ Commands: redo Re-run the latest migration status Dump the migration status for the current DB version Print the current version of the database - create NAME [sql|go] Creates new migration file with next version + create NAME [sql|go] Creates new migration file with the current timestamp Options: -dir string @@ -74,14 +72,14 @@ Examples: Create a new SQL migration. $ goose create add_some_column sql - $ Created new file: 00001_add_some_column.sql + $ Created new file: 20170506082420_add_some_column.sql Edit the newly created file to define the behavior of your migration. You can also create a Go migration, if you then invoke it with [your own goose binary](#go-migrations): $ goose create fetch_user_data go - $ Created new file: 00002_fetch_user_data.go + $ Created new file: 20170506082421_fetch_user_data.go ## up @@ -241,6 +239,11 @@ func Down(tx *sql.Tx) error { } ``` +# Hybrid Versioning +We strongly recommend adopting a hybrid versioning approach, using both timestamps and sequential numbers. Migrations created during the development process are timestamped and sequential versions are ran on production. We believe this method will prevent the problem of conflicting versions when writing software in a team environment. + +To help you adopt this approach, `create` will use the current timestamp as the migration version. When you're ready to deploy your migrations in a production environment, we also provide a helpful `fix` command to convert your migrations into sequential order, while preserving the timestamp ordering. We recommend running `fix` in the CI pipeline, and only when the migrations are ready for production. + ## License Licensed under [MIT License](./LICENSE) diff --git a/cmd/goose/main.go b/cmd/goose/main.go index dcc59101b..15b4caf6f 100644 --- a/cmd/goose/main.go +++ b/cmd/goose/main.go @@ -33,6 +33,14 @@ func main() { return } + // TODO clean up arg/flag parsing flow + if args[0] == "fix" { + if err := goose.Run("fix", nil, *dir); err != nil { + log.Fatalf("goose run: %v", err) + } + return + } + if len(args) < 3 { flags.Usage() return @@ -117,6 +125,7 @@ Commands: reset Roll back all migrations status Dump the migration status for the current DB version Print the current version of the database - create NAME [sql|go] Creates new migration file with next version + create NAME [sql|go] Creates new migration file with the current timestamp + fix Apply sequential ordering to migrations ` ) diff --git a/create.go b/create.go index b86307652..6761dec32 100644 --- a/create.go +++ b/create.go @@ -6,22 +6,12 @@ import ( "os" "path/filepath" "text/template" + "time" ) // Create writes a new blank migration file. func CreateWithTemplate(db *sql.DB, dir string, migrationTemplate *template.Template, name, migrationType string) error { - migrations, err := CollectMigrations(dir, minVersion, maxVersion) - if err != nil { - return err - } - - // Initial version. - version := "00001" - - if last, err := migrations.Last(); err == nil { - version = fmt.Sprintf("%05v", last.Version+1) - } - + version := time.Now().Format(timestampFormat) filename := fmt.Sprintf("%v_%v.%v", version, name, migrationType) fpath := filepath.Join(dir, filename) diff --git a/dialect.go b/dialect.go index 488f5e040..1f21aa6a3 100644 --- a/dialect.go +++ b/dialect.go @@ -10,6 +10,7 @@ import ( type SQLDialect interface { createVersionTableSQL() string // sql string to create the db version table insertVersionSQL() string // sql string to insert the initial version table row + updateVersionSQL() string // sql string to update version dbVersionQuery(db *sql.DB) (*sql.Rows, error) } @@ -61,6 +62,10 @@ func (pg PostgresDialect) insertVersionSQL() string { return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES ($1, $2);", TableName()) } +func (pg PostgresDialect) updateVersionSQL() string { + return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName()) +} + func (pg PostgresDialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) { rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName())) if err != nil { @@ -91,6 +96,10 @@ func (m MySQLDialect) insertVersionSQL() string { return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES (?, ?);", TableName()) } +func (m MySQLDialect) updateVersionSQL() string { + return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName()) +} + func (m MySQLDialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) { rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName())) if err != nil { @@ -120,6 +129,10 @@ func (m Sqlite3Dialect) insertVersionSQL() string { return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES (?, ?);", TableName()) } +func (m Sqlite3Dialect) updateVersionSQL() string { + return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName()) +} + func (m Sqlite3Dialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) { rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName())) if err != nil { @@ -150,6 +163,10 @@ func (rs RedshiftDialect) insertVersionSQL() string { return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES ($1, $2);", TableName()) } +func (rs RedshiftDialect) updateVersionSQL() string { + return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName()) +} + func (rs RedshiftDialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) { rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName())) if err != nil { @@ -180,6 +197,10 @@ func (m TiDBDialect) insertVersionSQL() string { return fmt.Sprintf("INSERT INTO %s (version_id, is_applied) VALUES (?, ?);", TableName()) } +func (m TiDBDialect) updateVersionSQL() string { + return fmt.Sprintf("UPDATE %s SET version_id=? WHERE version_id=?;", TableName()) +} + func (m TiDBDialect) dbVersionQuery(db *sql.DB) (*sql.Rows, error) { rows, err := db.Query(fmt.Sprintf("SELECT version_id, is_applied from %s ORDER BY id DESC", TableName())) if err != nil { diff --git a/examples/go-migrations/main.go b/examples/go-migrations/main.go index 64226ea3a..bb323afd1 100644 --- a/examples/go-migrations/main.go +++ b/examples/go-migrations/main.go @@ -33,6 +33,14 @@ func main() { return } + // TODO clean up arg/flag parsing flow + if args[0] == "fix" { + if err := goose.Run("fix", nil, *dir); err != nil { + log.Fatalf("goose run: %v", err) + } + return + } + if len(args) < 3 { flags.Usage() return @@ -117,6 +125,7 @@ Commands: redo Re-run the latest migration status Dump the migration status for the current DB version Print the current version of the database - create NAME [sql|go] Creates new migration file with next version + create NAME [sql|go] Creates new migration file with the current timestamp + fix Apply sequential ordering to migrations ` ) diff --git a/fix.go b/fix.go new file mode 100644 index 000000000..dcd801000 --- /dev/null +++ b/fix.go @@ -0,0 +1,46 @@ +package goose + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +func Fix(dir string) error { + migrations, err := CollectMigrations(dir, minVersion, maxVersion) + if err != nil { + return err + } + + // split into timestamped and versioned migrations + tsMigrations, err := migrations.timestamped() + if err != nil { + return err + } + + vMigrations, err := migrations.versioned() + if err != nil { + return err + } + // Initial version. + version := int64(1) + if last, err := vMigrations.Last(); err == nil { + version = last.Version + 1 + } + + // fix filenames by replacing timestamps with sequential versions + for _, tsm := range tsMigrations { + oldPath := tsm.Source + newPath := strings.Replace(oldPath, fmt.Sprintf("%d", tsm.Version), fmt.Sprintf("%05v", version), 1) + + if err := os.Rename(oldPath, newPath); err != nil { + return err + } + + log.Printf("RENAMED %s => %s", filepath.Base(oldPath), filepath.Base(newPath)) + version++ + } + + return nil +} diff --git a/fix_test.go b/fix_test.go new file mode 100644 index 000000000..956070c91 --- /dev/null +++ b/fix_test.go @@ -0,0 +1,81 @@ +package goose + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +func TestFix(t *testing.T) { + dir, err := ioutil.TempDir("", "tmptest") + if err != nil { + t.Fatal(err) + } + + defer os.RemoveAll(dir) // clean up + defer os.Remove("goose") // clean up + + commands := []string{ + "go build -i -o goose ./cmd/goose", + fmt.Sprintf("./goose -dir=%s create create_table", dir), + fmt.Sprintf("./goose -dir=%s create add_users", dir), + fmt.Sprintf("./goose -dir=%s create add_indices", dir), + fmt.Sprintf("./goose -dir=%s create update_users", dir), + fmt.Sprintf("./goose -dir=%s fix", dir), + } + + for _, cmd := range commands { + args := strings.Split(cmd, " ") + time.Sleep(1 * time.Second) + out, err := exec.Command(args[0], args[1:]...).CombinedOutput() + if err != nil { + t.Fatalf("%s:\n%v\n\n%s", err, cmd, out) + } + } + + files, err := ioutil.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + // check that the files are in order + for i, f := range files { + expected := fmt.Sprintf("%05v", i+1) + if !strings.HasPrefix(f.Name(), expected) { + t.Errorf("failed to find %s prefix in %s", expected, f.Name()) + } + } + + // add more migrations and then fix it + commands = []string{ + fmt.Sprintf("./goose -dir=%s create remove_column", dir), + fmt.Sprintf("./goose -dir=%s create create_books_table", dir), + fmt.Sprintf("./goose -dir=%s fix", dir), + } + + for _, cmd := range commands { + args := strings.Split(cmd, " ") + time.Sleep(1 * time.Second) + out, err := exec.Command(args[0], args[1:]...).CombinedOutput() + if err != nil { + t.Fatalf("%s:\n%v\n\n%s", err, cmd, out) + } + } + + files, err = ioutil.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + // check that the files still in order + for i, f := range files { + expected := fmt.Sprintf("%05v", i+1) + if !strings.HasPrefix(f.Name(), expected) { + t.Errorf("failed to find %s prefix in %s", expected, f.Name()) + } + } +} diff --git a/goose.go b/goose.go index 51ca6ecf4..ff4b5e84c 100644 --- a/goose.go +++ b/goose.go @@ -11,6 +11,7 @@ var ( duplicateCheckOnce sync.Once minVersion = int64(0) maxVersion = int64((1 << 63) - 1) + timestampFormat = "20060102150405" ) // Run runs a goose command. @@ -64,6 +65,10 @@ func Run(command string, db *sql.DB, dir string, args ...string) error { if err := DownTo(db, dir, version); err != nil { return err } + case "fix": + if err := Fix(dir); err != nil { + return err + } case "redo": if err := Redo(db, dir); err != nil { return err diff --git a/migrate.go b/migrate.go index 4774af022..552944278 100644 --- a/migrate.go +++ b/migrate.go @@ -8,6 +8,7 @@ import ( "path/filepath" "runtime" "sort" + "time" ) var ( @@ -76,6 +77,43 @@ func (ms Migrations) Last() (*Migration, error) { return ms[len(ms)-1], nil } +// Versioned gets versioned migrations. +func (ms Migrations) versioned() (Migrations, error) { + var migrations Migrations + + // assume that the user will never have more than 19700101000000 migrations + for _, m := range ms { + // parse version as timestmap + versionTime, err := time.Parse(timestampFormat, fmt.Sprintf("%d", m.Version)) + + if versionTime.Before(time.Unix(0, 0)) || err != nil { + migrations = append(migrations, m) + } + } + + return migrations, nil +} + +// Timestamped gets the timestamped migrations. +func (ms Migrations) timestamped() (Migrations, error) { + var migrations Migrations + + // assume that the user will never have more than 19700101000000 migrations + for _, m := range ms { + // parse version as timestmap + versionTime, err := time.Parse(timestampFormat, fmt.Sprintf("%d", m.Version)) + if err != nil { + // probably not a timestamp + continue + } + + if versionTime.After(time.Unix(0, 0)) { + migrations = append(migrations, m) + } + } + return migrations, nil +} + func (ms Migrations) String() string { str := "" for _, m := range ms {