feat(database/sqlmigrate): add migration library with multi-backend support

Extract a database-agnostic migration interface (Migrator) with Collect,
Up, Down, and GetStatus orchestration. Each backend is its own Go module
to avoid dependency bloat:

- sqlmigrate: core interface, types, sentinel errors, file collection
- pgmigrate: PostgreSQL via pgx/v5
- mymigrate: MySQL/MariaDB via go-sql-driver/mysql
- litemigrate: SQLite via database/sql (caller imports driver)
- msmigrate: SQL Server via go-mssqldb
- shmigrate: shell script generation (file-path commands, not heredocs)

Integrates shmigrate into the existing cmd/sql-migrate CLI, replacing
its bespoke up/down/status logic with sqlmigrate orchestration.
This commit is contained in:
AJ ONeal 2026-04-05 23:58:44 -06:00
parent 075cc7b286
commit a86d901d64
No known key found for this signature in database
17 changed files with 950 additions and 138 deletions

View File

@ -1,3 +1,13 @@
module github.com/therootcompany/golib/cmd/sql-migrate/v2
go 1.25.3
go 1.26.1
require (
github.com/therootcompany/golib/database/sqlmigrate v0.0.0
github.com/therootcompany/golib/database/sqlmigrate/shmigrate v0.0.0
)
replace (
github.com/therootcompany/golib/database/sqlmigrate => ../../database/sqlmigrate
github.com/therootcompany/golib/database/sqlmigrate/shmigrate => ../../database/sqlmigrate/shmigrate
)

View File

@ -13,6 +13,7 @@ package main
import (
"bufio"
"context"
"crypto/rand"
"encoding/hex"
"flag"
@ -26,6 +27,9 @@ import (
"strconv"
"strings"
"time"
"github.com/therootcompany/golib/database/sqlmigrate"
"github.com/therootcompany/golib/database/sqlmigrate/shmigrate"
)
const (
@ -274,14 +278,21 @@ func main() {
os.Exit(1)
}
ctx := context.Background()
runner := &shmigrate.Migrator{
Writer: os.Stdout,
SqlCommand: state.SQLCommand,
MigrationsDir: state.MigrationsDir,
LogQueryPath: filepath.Join(state.MigrationsDir, LOG_QUERY_NAME),
LogPath: state.LogPath,
}
migrations := sqlmigrate.NamesOnly(ups)
switch subcmd {
case "init":
break
case "sync":
if err := syncLog(&state); err != nil {
log.Fatal(err)
}
return
syncLog(runner)
case "create":
if len(leafArgs) == 0 {
log.Fatal("create requires a description")
@ -297,16 +308,15 @@ func main() {
os.Exit(1)
}
case "status":
if len(leafArgs) > 0 && subcmd != "create" {
if len(leafArgs) > 0 {
fmt.Fprintf(os.Stderr, "Error: unexpected args: %s\n", strings.Join(leafArgs, " "))
os.Exit(1)
}
err = status(&state, ups)
if err != nil {
if err := cmdStatus(ctx, &state, runner, migrations); err != nil {
log.Fatal(err)
}
case "list":
if len(leafArgs) > 0 && subcmd != "create" {
if len(leafArgs) > 0 {
fmt.Fprintf(os.Stderr, "Error: unexpected args: %s\n", strings.Join(leafArgs, " "))
os.Exit(1)
}
@ -341,8 +351,7 @@ func main() {
os.Exit(1)
}
err = up(&state, ups, upN)
if err != nil {
if err := cmdUp(ctx, &state, runner, migrations, upN); err != nil {
log.Fatal(err)
}
case "down":
@ -361,8 +370,7 @@ func main() {
os.Exit(1)
}
err = down(&state, downN)
if err != nil {
if err := cmdDown(ctx, &state, runner, migrations, downN); err != nil {
log.Fatal(err)
}
default:
@ -851,190 +859,171 @@ func fixupMigration(dir string, basename string) (up, down bool, warn error, err
return up, down, nil, nil
}
func syncLog(state *State) error {
getMigsPath := filepath.Join(state.MigrationsDir, LOG_QUERY_NAME)
getMigsPath = filepathUnclean(getMigsPath)
getMigs := strings.Replace(state.SQLCommand, "%s", getMigsPath, 1)
logPath := filepathUnclean(state.LogPath)
func syncLog(runner *shmigrate.Migrator) {
syncCmd := strings.Replace(runner.SqlCommand, "%s", filepathUnclean(runner.LogQueryPath), 1)
logPath := filepathUnclean(runner.LogPath)
fmt.Printf(shHeader)
fmt.Printf(shmigrate.ShHeader)
fmt.Println("")
fmt.Println("# SYNC: reload migrations log from DB")
fmt.Printf("%s > %s || true\n", getMigs, logPath)
fmt.Printf("%s > %s || true\n", syncCmd, logPath)
fmt.Printf("cat %s\n", logPath)
return nil
}
func up(state *State, ups []string, n int) error {
var pending []string
for _, mig := range ups {
found := slices.Contains(state.Migrated, mig)
if !found {
pending = append(pending, mig)
}
func cmdUp(ctx context.Context, state *State, runner *shmigrate.Migrator, migrations []sqlmigrate.Migration, n int) error {
// fixup pending migrations before generating the script
fixedUp, fixedDown := fixupAll(state.MigrationsDir, state.Migrated, migrations)
status, err := sqlmigrate.GetStatus(ctx, runner, migrations)
if err != nil {
return err
}
getMigsPath := filepath.Join(state.MigrationsDir, LOG_QUERY_NAME)
getMigsPath = filepathUnclean(getMigsPath)
getMigs := strings.Replace(state.SQLCommand, "%s", getMigsPath, 1)
if len(pending) == 0 {
if len(status.Pending) == 0 {
syncCmd := strings.Replace(runner.SqlCommand, "%s", filepathUnclean(runner.LogQueryPath), 1)
fmt.Fprintf(os.Stderr, "# Already up-to-date\n")
fmt.Fprintf(os.Stderr, "#\n")
fmt.Fprintf(os.Stderr, "# To reload the migrations log:\n")
fmt.Fprintf(os.Stderr, "# %s > %s\n", getMigs, filepathUnclean(state.LogPath))
fmt.Fprintf(os.Stderr, "# %s > %s\n", syncCmd, filepathUnclean(runner.LogPath))
return nil
}
if n == 0 {
n = len(pending)
}
fixedUp := []string{}
fixedDown := []string{}
fmt.Printf(shHeader)
fmt.Printf(shmigrate.ShHeader)
fmt.Println("")
fmt.Println("# FORWARD / UP Migrations")
fmt.Println("")
for i, migration := range pending {
if i >= n {
break
}
path := filepath.Join(state.MigrationsDir, migration+".up.sql")
path = filepathUnclean(path)
{
up, down, warn, err := fixupMigration(state.MigrationsDir, migration)
if warn != nil {
fmt.Fprintf(os.Stderr, "Warn: %s\n", warn)
}
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
}
if up {
fixedUp = append(fixedUp, migration)
}
if down {
fixedDown = append(fixedDown, migration)
}
}
cmd := strings.Replace(state.SQLCommand, "%s", path, 1)
fmt.Printf("# +%d %s\n", i+1, migration)
fmt.Println(cmd)
fmt.Println(getMigs + " > " + filepathUnclean(state.LogPath))
fmt.Println("")
applied, err := sqlmigrate.Up(ctx, runner, migrations, n)
if err != nil {
return err
}
fmt.Println("cat", filepathUnclean(state.LogPath))
_ = applied
fmt.Println("cat", filepathUnclean(runner.LogPath))
showFixes(fixedUp, fixedDown)
return nil
}
func down(state *State, n int) error {
lines := make([]string, len(state.Lines))
copy(lines, state.Lines)
slices.Reverse(lines)
func cmdDown(ctx context.Context, state *State, runner *shmigrate.Migrator, migrations []sqlmigrate.Migration, n int) error {
// fixup applied migrations before generating the script
fixedUp, fixedDown := fixupAll(state.MigrationsDir, state.Migrated, migrations)
getMigsPath := filepath.Join(state.MigrationsDir, LOG_QUERY_NAME)
getMigsPath = filepathUnclean(getMigsPath)
getMigs := strings.Replace(state.SQLCommand, "%s", getMigsPath, 1)
status, err := sqlmigrate.GetStatus(ctx, runner, migrations)
if err != nil {
return err
}
if len(lines) == 0 {
if len(status.Applied) == 0 {
syncCmd := strings.Replace(runner.SqlCommand, "%s", filepathUnclean(runner.LogQueryPath), 1)
fmt.Fprintf(os.Stderr, "# No migration history\n")
fmt.Fprintf(os.Stderr, "#\n")
fmt.Fprintf(os.Stderr, "# To reload the migrations log:\n")
fmt.Fprintf(os.Stderr, "# %s > %s\n", getMigs, filepathUnclean(state.LogPath))
fmt.Fprintf(os.Stderr, "# %s > %s\n", syncCmd, filepathUnclean(runner.LogPath))
return nil
}
if n == 0 {
n = 1
}
fixedUp := []string{}
fixedDown := []string{}
var applied []string
for _, line := range lines {
migration := commentStartRe.ReplaceAllString(line, "")
migration = strings.TrimSpace(migration)
if migration == "" {
continue
}
applied = append(applied, migration)
}
fmt.Printf(shHeader)
fmt.Printf(shmigrate.ShHeader)
fmt.Println("")
fmt.Println("# ROLLBACK / DOWN Migration")
fmt.Println("")
for i, migration := range applied {
if i >= n {
break
}
downPath := filepath.Join(state.MigrationsDir, migration+".down.sql")
cmd := strings.Replace(state.SQLCommand, "%s", downPath, 1)
fmt.Printf("\n# -%d %s\n", i+1, migration)
// check for missing down files before generating script
reversed := make([]string, len(status.Applied))
copy(reversed, status.Applied)
slices.Reverse(reversed)
limit := n
if limit <= 0 {
limit = 1
}
if limit > len(reversed) {
limit = len(reversed)
}
for _, name := range reversed[:limit] {
downPath := filepath.Join(state.MigrationsDir, name+".down.sql")
if !fileExists(downPath) {
fmt.Fprintf(os.Stderr, "# Warn: missing %s\n", filepathUnclean(downPath))
fmt.Fprintf(os.Stderr, "# (the migration will fail to run)\n")
fmt.Printf("# ERROR: MISSING FILE\n")
} else {
up, down, warn, err := fixupMigration(state.MigrationsDir, migration)
if warn != nil {
fmt.Fprintf(os.Stderr, "Warn: %s\n", warn)
}
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
}
if up {
fixedUp = append(fixedUp, migration)
}
if down {
fixedDown = append(fixedDown, migration)
}
}
fmt.Println(cmd)
fmt.Println(getMigs + " > " + filepathUnclean(state.LogPath))
fmt.Println("")
}
fmt.Println("cat", filepathUnclean(state.LogPath))
rolled, err := sqlmigrate.Down(ctx, runner, migrations, n)
if err != nil {
return err
}
_ = rolled
fmt.Println("cat", filepathUnclean(runner.LogPath))
showFixes(fixedUp, fixedDown)
return nil
}
func status(state *State, ups []string) error {
previous := make([]string, len(state.Lines))
copy(previous, state.Lines)
slices.Reverse(previous)
func cmdStatus(ctx context.Context, state *State, runner *shmigrate.Migrator, migrations []sqlmigrate.Migration) error {
status, err := sqlmigrate.GetStatus(ctx, runner, migrations)
if err != nil {
return err
}
fmt.Fprintf(os.Stderr, "migrations_dir: %s\n", filepathUnclean(state.MigrationsDir))
fmt.Fprintf(os.Stderr, "migrations_log: %s\n", filepathUnclean(state.LogPath))
fmt.Fprintf(os.Stderr, "sql_command: %s\n", state.SQLCommand)
fmt.Fprintf(os.Stderr, "\n")
fmt.Printf("# previous: %d\n", len(previous))
for _, mig := range previous {
// show applied in reverse (most recent first)
applied := make([]string, len(status.Applied))
copy(applied, status.Applied)
slices.Reverse(applied)
fmt.Printf("# previous: %d\n", len(applied))
for _, mig := range applied {
fmt.Printf(" %s\n", mig)
}
if len(previous) == 0 {
if len(applied) == 0 {
fmt.Println(" # (no previous migrations)")
}
fmt.Println("")
var pending []string
for _, mig := range ups {
found := slices.Contains(state.Migrated, mig)
if !found {
pending = append(pending, mig)
}
}
fmt.Printf("# pending: %d\n", len(pending))
for _, mig := range pending {
fmt.Printf("# pending: %d\n", len(status.Pending))
for _, mig := range status.Pending {
fmt.Printf(" %s\n", mig)
}
if len(pending) == 0 {
if len(status.Pending) == 0 {
fmt.Println(" # (no pending migrations)")
}
return nil
}
// fixupAll runs fixupMigration on all known migrations (applied + pending).
func fixupAll(migrationsDir string, applied []string, migrations []sqlmigrate.Migration) (fixedUp, fixedDown []string) {
seen := map[string]bool{}
var all []string
for _, name := range applied {
if !seen[name] {
all = append(all, name)
seen[name] = true
}
}
for _, m := range migrations {
if !seen[m.Name] {
all = append(all, m.Name)
seen[m.Name] = true
}
}
for _, name := range all {
up, down, warn, err := fixupMigration(migrationsDir, name)
if warn != nil {
fmt.Fprintf(os.Stderr, "Warn: %s\n", warn)
}
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
}
if up {
fixedUp = append(fixedUp, name)
}
if down {
fixedDown = append(fixedDown, name)
}
}
return fixedUp, fixedDown
}

View File

@ -0,0 +1,3 @@
module github.com/therootcompany/golib/database/sqlmigrate
go 1.26.1

View File

@ -0,0 +1,7 @@
module github.com/therootcompany/golib/database/sqlmigrate/litemigrate
go 1.26.1
require github.com/therootcompany/golib/database/sqlmigrate v0.0.0
replace github.com/therootcompany/golib/database/sqlmigrate => ../

View File

@ -0,0 +1,84 @@
// Package litemigrate implements sqlmigrate.Migrator for SQLite
// using database/sql. The caller imports the driver:
//
// import _ "modernc.org/sqlite"
//
// db, err := sql.Open("sqlite", "file:app.db")
package litemigrate
import (
"context"
"database/sql"
"fmt"
"strings"
"github.com/therootcompany/golib/database/sqlmigrate"
)
// Migrator implements sqlmigrate.Migrator using a *sql.DB with SQLite.
type Migrator struct {
DB *sql.DB
}
// New creates a Migrator from the given database handle.
func New(db *sql.DB) *Migrator {
return &Migrator{DB: db}
}
var _ sqlmigrate.Migrator = (*Migrator)(nil)
// ExecUp runs the up migration SQL inside a transaction.
func (m *Migrator) ExecUp(ctx context.Context, mig sqlmigrate.Migration) error {
return m.execInTx(ctx, mig.Up)
}
// ExecDown runs the down migration SQL inside a transaction.
func (m *Migrator) ExecDown(ctx context.Context, mig sqlmigrate.Migration) error {
return m.execInTx(ctx, mig.Down)
}
func (m *Migrator) execInTx(ctx context.Context, sqlStr string) error {
tx, err := m.DB.BeginTx(ctx, nil)
if err != nil {
return fmt.Errorf("%w: begin: %w", sqlmigrate.ErrExecFailed, err)
}
defer tx.Rollback() //nolint:errcheck
if _, err := tx.ExecContext(ctx, sqlStr); err != nil {
return fmt.Errorf("%w: exec: %w", sqlmigrate.ErrExecFailed, err)
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("%w: commit: %w", sqlmigrate.ErrExecFailed, err)
}
return nil
}
// Applied returns the names of all applied migrations from the _migrations table.
// Returns an empty slice if the table does not exist.
func (m *Migrator) Applied(ctx context.Context) ([]string, error) {
rows, err := m.DB.QueryContext(ctx, "SELECT name FROM _migrations ORDER BY name")
if err != nil {
// SQLite reports "no such table: _migrations" — stable across versions
if strings.Contains(err.Error(), "no such table") {
return nil, nil
}
return nil, fmt.Errorf("%w: %w", sqlmigrate.ErrQueryApplied, err)
}
defer rows.Close()
var names []string
for rows.Next() {
var name string
if err := rows.Scan(&name); err != nil {
return nil, fmt.Errorf("%w: scanning row: %w", sqlmigrate.ErrQueryApplied, err)
}
names = append(names, name)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("%w: reading rows: %w", sqlmigrate.ErrQueryApplied, err)
}
return names, nil
}

View File

@ -0,0 +1,19 @@
module github.com/therootcompany/golib/database/sqlmigrate/msmigrate
go 1.26.1
require (
github.com/microsoft/go-mssqldb v1.9.8
github.com/therootcompany/golib/database/sqlmigrate v0.0.0
)
require (
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/shopspring/decimal v1.4.0 // indirect
golang.org/x/crypto v0.48.0 // indirect
golang.org/x/text v0.34.0 // indirect
)
replace github.com/therootcompany/golib/database/sqlmigrate => ../

View File

@ -0,0 +1,44 @@
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI=
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.4.0 h1:E4MgwLBGeVB5f2MdcIVD3ELVAWpr+WD6MUe1i+tM/PA=
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.4.0/go.mod h1:Y2b/1clN4zsAoUd/pgNAQHjLDnTis/6ROkUfyob6psM=
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4=
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA=
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/microsoft/go-mssqldb v1.9.8 h1:d4IFMvF/o+HdpXUqbBfzHvn/NlFA75YGcfHUUvDFJEM=
github.com/microsoft/go-mssqldb v1.9.8/go.mod h1:eGSRSGAW4hKMy5YcAenhCDjIRm2rhqIdmmwgciMzLus=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@ -0,0 +1,84 @@
// Package msmigrate implements sqlmigrate.Migrator for Microsoft SQL Server
// using database/sql with github.com/microsoft/go-mssqldb.
//
// db, err := sql.Open("sqlserver", "sqlserver://user:pass@host:1433?database=mydb")
package msmigrate
import (
"context"
"database/sql"
"errors"
"fmt"
mssql "github.com/microsoft/go-mssqldb"
"github.com/therootcompany/golib/database/sqlmigrate"
)
// Migrator implements sqlmigrate.Migrator using a *sql.DB with SQL Server.
type Migrator struct {
DB *sql.DB
}
// New creates a Migrator from the given database handle.
func New(db *sql.DB) *Migrator {
return &Migrator{DB: db}
}
var _ sqlmigrate.Migrator = (*Migrator)(nil)
// ExecUp runs the up migration SQL inside a transaction.
func (m *Migrator) ExecUp(ctx context.Context, mig sqlmigrate.Migration) error {
return m.execInTx(ctx, mig.Up)
}
// ExecDown runs the down migration SQL inside a transaction.
func (m *Migrator) ExecDown(ctx context.Context, mig sqlmigrate.Migration) error {
return m.execInTx(ctx, mig.Down)
}
func (m *Migrator) execInTx(ctx context.Context, sqlStr string) error {
tx, err := m.DB.BeginTx(ctx, nil)
if err != nil {
return fmt.Errorf("%w: begin: %w", sqlmigrate.ErrExecFailed, err)
}
defer tx.Rollback() //nolint:errcheck
if _, err := tx.ExecContext(ctx, sqlStr); err != nil {
return fmt.Errorf("%w: exec: %w", sqlmigrate.ErrExecFailed, err)
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("%w: commit: %w", sqlmigrate.ErrExecFailed, err)
}
return nil
}
// Applied returns the names of all applied migrations from the _migrations table.
// Returns an empty slice if the table does not exist (SQL Server error 208).
func (m *Migrator) Applied(ctx context.Context) ([]string, error) {
rows, err := m.DB.QueryContext(ctx, "SELECT name FROM _migrations ORDER BY name")
if err != nil {
// SQL Server error 208: "Invalid object name '_migrations'"
if msErr, ok := errors.AsType[mssql.Error](err); ok && msErr.Number == 208 {
return nil, nil
}
return nil, fmt.Errorf("%w: %w", sqlmigrate.ErrQueryApplied, err)
}
defer rows.Close()
var names []string
for rows.Next() {
var name string
if err := rows.Scan(&name); err != nil {
return nil, fmt.Errorf("%w: scanning row: %w", sqlmigrate.ErrQueryApplied, err)
}
names = append(names, name)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("%w: reading rows: %w", sqlmigrate.ErrQueryApplied, err)
}
return names, nil
}

View File

@ -0,0 +1,12 @@
module github.com/therootcompany/golib/database/sqlmigrate/mymigrate
go 1.26.1
require (
github.com/go-sql-driver/mysql v1.9.3
github.com/therootcompany/golib/database/sqlmigrate v0.0.0
)
require filippo.io/edwards25519 v1.1.0 // indirect
replace github.com/therootcompany/golib/database/sqlmigrate => ../

View File

@ -0,0 +1,4 @@
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=

View File

@ -0,0 +1,86 @@
// Package mymigrate implements sqlmigrate.Migrator for MySQL and MariaDB
// using database/sql with github.com/go-sql-driver/mysql.
//
// The caller must open the *sql.DB with multiStatements=true in the DSN
// for multi-statement migration files to work:
//
// db, err := sql.Open("mysql", "user:pass@tcp(host:3306)/dbname?multiStatements=true")
package mymigrate
import (
"context"
"database/sql"
"errors"
"fmt"
"github.com/go-sql-driver/mysql"
"github.com/therootcompany/golib/database/sqlmigrate"
)
// Migrator implements sqlmigrate.Migrator using a *sql.DB with MySQL/MariaDB.
type Migrator struct {
DB *sql.DB
}
// New creates a Migrator from the given database handle.
func New(db *sql.DB) *Migrator {
return &Migrator{DB: db}
}
var _ sqlmigrate.Migrator = (*Migrator)(nil)
// ExecUp runs the up migration SQL inside a transaction.
func (m *Migrator) ExecUp(ctx context.Context, mig sqlmigrate.Migration) error {
return m.execInTx(ctx, mig.Up)
}
// ExecDown runs the down migration SQL inside a transaction.
func (m *Migrator) ExecDown(ctx context.Context, mig sqlmigrate.Migration) error {
return m.execInTx(ctx, mig.Down)
}
func (m *Migrator) execInTx(ctx context.Context, sqlStr string) error {
tx, err := m.DB.BeginTx(ctx, nil)
if err != nil {
return fmt.Errorf("%w: begin: %w", sqlmigrate.ErrExecFailed, err)
}
defer tx.Rollback() //nolint:errcheck
if _, err := tx.ExecContext(ctx, sqlStr); err != nil {
return fmt.Errorf("%w: exec: %w", sqlmigrate.ErrExecFailed, err)
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("%w: commit: %w", sqlmigrate.ErrExecFailed, err)
}
return nil
}
// Applied returns the names of all applied migrations from the _migrations table.
// Returns an empty slice if the table does not exist (MySQL error 1146).
func (m *Migrator) Applied(ctx context.Context) ([]string, error) {
rows, err := m.DB.QueryContext(ctx, "SELECT name FROM _migrations ORDER BY name")
if err != nil {
if mysqlErr, ok := errors.AsType[*mysql.MySQLError](err); ok && mysqlErr.Number == 1146 {
return nil, nil
}
return nil, fmt.Errorf("%w: %w", sqlmigrate.ErrQueryApplied, err)
}
defer rows.Close()
var names []string
for rows.Next() {
var name string
if err := rows.Scan(&name); err != nil {
return nil, fmt.Errorf("%w: scanning row: %w", sqlmigrate.ErrQueryApplied, err)
}
names = append(names, name)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("%w: reading rows: %w", sqlmigrate.ErrQueryApplied, err)
}
return names, nil
}

View File

@ -0,0 +1,18 @@
module github.com/therootcompany/golib/database/sqlmigrate/pgmigrate
go 1.26.1
require (
github.com/jackc/pgx/v5 v5.9.1
github.com/therootcompany/golib/database/sqlmigrate v0.0.0
)
replace github.com/therootcompany/golib/database/sqlmigrate => ../
require (
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
golang.org/x/sync v0.17.0 // indirect
golang.org/x/text v0.29.0 // indirect
)

View File

@ -0,0 +1,26 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc=
github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@ -0,0 +1,75 @@
// Package pgmigrate implements sqlmigrate.Migrator for PostgreSQL via pgx/v5.
package pgmigrate
import (
"context"
"errors"
"fmt"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/therootcompany/golib/database/sqlmigrate"
)
// Migrator implements sqlmigrate.Migrator using a pgxpool.Pool.
type Migrator struct {
Pool *pgxpool.Pool
}
// New creates a Migrator from the given pool.
func New(pool *pgxpool.Pool) *Migrator {
return &Migrator{Pool: pool}
}
// verify interface compliance at compile time
var _ sqlmigrate.Migrator = (*Migrator)(nil)
// ExecUp runs the up migration SQL inside a PostgreSQL transaction.
func (r *Migrator) ExecUp(ctx context.Context, m sqlmigrate.Migration) error {
return r.execInTx(ctx, m.Up)
}
// ExecDown runs the down migration SQL inside a PostgreSQL transaction.
func (r *Migrator) ExecDown(ctx context.Context, m sqlmigrate.Migration) error {
return r.execInTx(ctx, m.Down)
}
func (r *Migrator) execInTx(ctx context.Context, sql string) error {
tx, err := r.Pool.Begin(ctx)
if err != nil {
return fmt.Errorf("%w: begin: %w", sqlmigrate.ErrExecFailed, err)
}
defer tx.Rollback(ctx) //nolint:errcheck
if _, err := tx.Exec(ctx, sql); err != nil {
return fmt.Errorf("%w: exec: %w", sqlmigrate.ErrExecFailed, err)
}
if err := tx.Commit(ctx); err != nil {
return fmt.Errorf("%w: commit: %w", sqlmigrate.ErrExecFailed, err)
}
return nil
}
// Applied returns the names of all applied migrations from the _migrations table.
// Returns an empty slice if the table does not exist (PG error 42P01).
func (r *Migrator) Applied(ctx context.Context) ([]string, error) {
rows, err := r.Pool.Query(ctx, "SELECT name FROM _migrations ORDER BY name")
if err != nil {
if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok && pgErr.Code == "42P01" {
return nil, nil
}
return nil, fmt.Errorf("%w: %w", sqlmigrate.ErrQueryApplied, err)
}
defer rows.Close()
names, err := pgx.CollectRows(rows, pgx.RowTo[string])
if err != nil {
return nil, fmt.Errorf("%w: reading rows: %w", sqlmigrate.ErrQueryApplied, err)
}
return names, nil
}

View File

@ -0,0 +1,7 @@
module github.com/therootcompany/golib/database/sqlmigrate/shmigrate
go 1.26.1
require github.com/therootcompany/golib/database/sqlmigrate v0.0.0
replace github.com/therootcompany/golib/database/sqlmigrate => ../

View File

@ -0,0 +1,125 @@
// Package shmigrate implements sqlmigrate.Runner by generating POSIX
// shell commands that reference migration files on disk. It is used by
// the sql-migrate CLI to produce scripts that can be piped to sh.
package shmigrate
import (
"bufio"
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/therootcompany/golib/database/sqlmigrate"
)
// ShHeader is the standard header for generated shell scripts.
const ShHeader = `#!/bin/sh
set -e
set -u
if test -s ./.env; then
. ./.env
fi
`
// Migrator generates shell scripts for migration execution.
// It implements sqlmigrate.Migrator so it can be used with
// sqlmigrate.Up, sqlmigrate.Down, and sqlmigrate.GetStatus.
type Migrator struct {
// Writer receives the generated shell script.
Writer io.Writer
// SqlCommand is the shell command template with %s for the file path.
// Example: `psql "$PG_URL" -v ON_ERROR_STOP=on -A -t --file %s`
SqlCommand string
// MigrationsDir is the path to the migrations directory on disk.
MigrationsDir string
// LogQueryPath is the path to the _migrations.sql query file.
// Used to sync the migrations log after each migration.
LogQueryPath string
// LogPath is the path to the migrations.log file.
LogPath string
counter int
}
// verify interface compliance at compile time
var _ sqlmigrate.Migrator = (*Migrator)(nil)
// ExecUp outputs a shell command to run the .up.sql migration file.
func (r *Migrator) ExecUp(ctx context.Context, m sqlmigrate.Migration) error {
r.counter++
return r.exec(m.Name, ".up.sql", fmt.Sprintf("+%d", r.counter))
}
// ExecDown outputs a shell command to run the .down.sql migration file.
func (r *Migrator) ExecDown(ctx context.Context, m sqlmigrate.Migration) error {
r.counter++
return r.exec(m.Name, ".down.sql", fmt.Sprintf("-%d", r.counter))
}
func (r *Migrator) exec(name, suffix, label string) error {
path := unclean(filepath.Join(r.MigrationsDir, name+suffix))
cmd := strings.Replace(r.SqlCommand, "%s", path, 1)
syncCmd := strings.Replace(r.SqlCommand, "%s", unclean(r.LogQueryPath), 1)
logPath := unclean(r.LogPath)
fmt.Fprintf(r.Writer, "# %s %s\n", label, name)
fmt.Fprintln(r.Writer, cmd)
fmt.Fprintf(r.Writer, "%s > %s || true\n", syncCmd, logPath)
fmt.Fprintln(r.Writer)
return nil
}
// Applied reads the migrations log file and returns applied migration names.
// Returns an empty slice if the file does not exist.
func (r *Migrator) Applied(ctx context.Context) ([]string, error) {
data, err := os.ReadFile(r.LogPath)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, fmt.Errorf("reading migrations log: %w", err)
}
var names []string
scanner := bufio.NewScanner(strings.NewReader(string(data)))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
// strip inline comments
if idx := strings.Index(line, "#"); idx >= 0 {
line = strings.TrimSpace(line[:idx])
}
if line != "" {
names = append(names, line)
}
}
return names, nil
}
// Reset resets the migration counter. Call between Up and Down
// if generating both in the same script.
func (r *Migrator) Reset() {
r.counter = 0
}
// unclean ensures a relative path starts with ./ or ../ so it
// is not interpreted as a command name in shell scripts.
func unclean(path string) string {
if strings.HasPrefix(path, "/") {
return path
}
if strings.HasPrefix(path, "./") || strings.HasPrefix(path, "../") {
return path
}
return "./" + path
}

View File

@ -0,0 +1,219 @@
// Package sqlmigrate provides types and utilities for SQL migration files.
// It is database-agnostic — see pgmigrate for PostgreSQL execution,
// shmigrate for shell script generation.
package sqlmigrate
import (
"context"
"errors"
"fmt"
"io/fs"
"slices"
"strings"
)
// Sentinel errors for migration operations.
var (
ErrMissingUp = errors.New("missing up migration")
ErrMissingDown = errors.New("missing down migration")
ErrWalkFailed = errors.New("walking migrations")
ErrExecFailed = errors.New("migration exec failed")
ErrQueryApplied = errors.New("querying applied migrations")
)
// Migration represents a paired up/down migration.
type Migration struct {
Name string // e.g. "2026-04-05-001000_create-todos"
Up string // SQL content of the .up.sql file
Down string // SQL content of the .down.sql file
}
// Status represents the current migration state.
type Status struct {
Applied []string
Pending []string
}
// Migrator executes migrations. Implementations handle the
// database-specific or output-specific details.
type Migrator interface {
// ExecUp runs the up migration. For database migrators this executes
// m.Up in a transaction. For shell migrators this outputs a command
// referencing the .up.sql file.
ExecUp(ctx context.Context, m Migration) error
// ExecDown runs the down migration.
ExecDown(ctx context.Context, m Migration) error
// Applied returns the names of all applied migrations, sorted
// lexicographically. Returns an empty slice (not an error) if the
// migrations table or log does not exist yet.
Applied(ctx context.Context) ([]string, error)
}
// Collect reads .up.sql and .down.sql files from fsys, pairs them by
// basename, and returns them sorted lexicographically by name.
func Collect(fsys fs.FS) ([]Migration, error) {
ups := map[string]string{}
downs := map[string]string{}
err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
name := d.Name()
if base, ok := strings.CutSuffix(name, ".up.sql"); ok {
b, readErr := fs.ReadFile(fsys, path)
if readErr != nil {
return readErr
}
ups[base] = string(b)
return nil
}
if base, ok := strings.CutSuffix(name, ".down.sql"); ok {
b, readErr := fs.ReadFile(fsys, path)
if readErr != nil {
return readErr
}
downs[base] = string(b)
return nil
}
return nil
})
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrWalkFailed, err)
}
var migrations []Migration
for name, upSQL := range ups {
downSQL, ok := downs[name]
if !ok {
return nil, fmt.Errorf("%w: %s", ErrMissingDown, name)
}
migrations = append(migrations, Migration{
Name: name,
Up: upSQL,
Down: downSQL,
})
}
for name := range downs {
if _, ok := ups[name]; !ok {
return nil, fmt.Errorf("%w: %s", ErrMissingUp, name)
}
}
slices.SortFunc(migrations, func(a, b Migration) int {
return strings.Compare(a.Name, b.Name)
})
return migrations, nil
}
// NamesOnly builds a Migration slice from a list of names, with empty
// Up/Down content. Useful for shell-based runners that reference files
// on disk rather than executing SQL directly.
func NamesOnly(names []string) []Migration {
migrations := make([]Migration, len(names))
for i, name := range names {
migrations[i] = Migration{Name: name}
}
return migrations
}
// Up applies up to n pending migrations using the given Runner.
// If n <= 0, applies all pending. Returns the names of applied migrations.
func Up(ctx context.Context, r Migrator, migrations []Migration, n int) ([]string, error) {
appliedNames, err := r.Applied(ctx)
if err != nil {
return nil, err
}
var pending []Migration
for _, m := range migrations {
if !slices.Contains(appliedNames, m.Name) {
pending = append(pending, m)
}
}
if n <= 0 {
n = len(pending)
}
if n > len(pending) {
n = len(pending)
}
var ran []string
for _, m := range pending[:n] {
if err := r.ExecUp(ctx, m); err != nil {
return ran, fmt.Errorf("%s (up): %w", m.Name, err)
}
ran = append(ran, m.Name)
}
return ran, nil
}
// Down rolls back up to n applied migrations, most recent first.
// If n <= 0, rolls back one. Returns the names of rolled-back migrations.
func Down(ctx context.Context, r Migrator, migrations []Migration, n int) ([]string, error) {
appliedNames, err := r.Applied(ctx)
if err != nil {
return nil, err
}
if n <= 0 {
n = 1
}
byName := map[string]Migration{}
for _, m := range migrations {
byName[m.Name] = m
}
reversed := make([]string, len(appliedNames))
copy(reversed, appliedNames)
slices.Reverse(reversed)
if n > len(reversed) {
n = len(reversed)
}
var ran []string
for _, name := range reversed[:n] {
m, ok := byName[name]
if !ok {
return ran, fmt.Errorf("%w: %s", ErrMissingDown, name)
}
if err := r.ExecDown(ctx, m); err != nil {
return ran, fmt.Errorf("%s (down): %w", name, err)
}
ran = append(ran, name)
}
return ran, nil
}
// GetStatus returns applied and pending migration lists.
func GetStatus(ctx context.Context, r Migrator, migrations []Migration) (*Status, error) {
appliedNames, err := r.Applied(ctx)
if err != nil {
return nil, err
}
var pending []string
for _, m := range migrations {
if !slices.Contains(appliedNames, m.Name) {
pending = append(pending, m.Name)
}
}
return &Status{
Applied: appliedNames,
Pending: pending,
}, nil
}