feat: Add support for converting auto-increment to sequence (#832)

* feat: uuid changes * changes * linting * test cases * fixes tests tests tests test fix * fixes * changes * changes * changes * changes to backend * changes to backend * ui * changes * changes * merge changes * changes * changes * changes * changes * changes * changes * changes * changes * tests * create sequence backend * fixing css * cahnegs * comment fixes * change * tests * changes * fixes * changes * changes * comment changes * changes * changes * tests * changes * changes * changes * changes * change * changes * test * changes * Update infoschema.go * changes
GoogleCloudPlatform · Jun 24, 2024 · 74e76a4 · 74e76a4
1 parent 1b18c01
commit 74e76a4
Show file tree

Hide file tree

Showing 27 changed files with 314 additions and 25 deletions.
diff --git a/common/constants/constants.go b/common/constants/constants.go
@@ -104,6 +104,7 @@ const (
 	// Auto Generated Keys
 	UUID     string = "UUID"
 	SEQUENCE string = "Sequence"
+	AUTO_INCREMENT string ="Auto Increment"
 	// Default gcs path of the Dataflow template.
 	DEFAULT_TEMPLATE_PATH string = "gs://dataflow-templates/latest/flex/Cloud_Datastream_to_Spanner"
 )
diff --git a/internal/convert.go b/internal/convert.go
@@ -123,6 +123,7 @@ const (
 	ShardIdColumnAdded
 	ShardIdColumnPrimaryKey
 	ArrayTypeNotSupported
+	SequenceCreated
 )
 
 const (

diff --git a/internal/reports/report_helpers.go b/internal/reports/report_helpers.go
@@ -204,6 +204,12 @@ func buildTableReportBody(conv *internal.Conv, tableId string, issues map[string
 						Description: fmt.Sprintf("Column '%s' is an autoincrement column in table '%s'. %s", spColName, conv.SpSchema[tableId].Name, IssueDB[i].Brief),
 					}
 					l = append(l, toAppend)
+				case internal.SequenceCreated:
+					toAppend := Issue{
+						Category:    IssueDB[i].Category,
+						Description: fmt.Sprintf("Auto-Increment has been converted to Sequence '%s' for column '%s' in table '%s'. Set Skipped Range or Start with Counter to avoid duplicate value errors.", conv.SpSchema[tableId].ColDefs[colId].AutoGen.Name, spColName, conv.SpSchema[tableId].Name),
+					}
+					l = append(l, toAppend)
 				case internal.Timestamp:
 					// Avoid the confusing "timestamp is mapped to timestamp" message.
 					toAppend := Issue{
@@ -524,6 +530,7 @@ var IssueDB = map[internal.SchemaIssue]struct {
 	internal.UniqueIndexPrimaryKey: {Category: "UNIQUE_INDEX_PRIMARY_KEY",
 		CategoryDescription: "Primary Key is missing, unique column(s) used as primary key"},
 	internal.ArrayTypeNotSupported: {Brief: "Array datatype migration is not fully supported. Please validate data after data migration", severity: warning, Category: "ARRAY_TYPE_NOT_SUPPORTED"},
+	internal.SequenceCreated: {Brief: "Auto Increment has been converted to Sequence, set Skipped Range or Start with Counter to avoid duplicate value errors", severity: warning, Category: "SEQUENCE_CREATED"},
 }
 
 type severity int

diff --git a/schema/schema.go b/schema/schema.go
@@ -29,6 +29,8 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/spanner/ddl"
 )
 
 // Table represents a database table.
@@ -52,6 +54,7 @@ type Column struct {
 	NotNull 	bool
 	Ignored 	Ignored
 	Id      	string
+	AutoGen		ddl.AutoGenCol
 }
 
 // ForeignKey represents a foreign key.
@@ -119,12 +122,6 @@ type Ignored struct {
 	AutoIncrement bool
 }
 
-type AutoGenCol struct {
-	Name string
-	// Type of autogenerated column, example, pre-defined(uuid) or user-defined(sequence) 
-	GenerationType string
-}
-
 // Print converts ty to a string suitable for printing.
 func (ty Type) Print() string {
 	s := ty.Name

diff --git a/sources/common/infoschema.go b/sources/common/infoschema.go
@@ -48,6 +48,7 @@ type InfoSchema interface {
 type SchemaAndName struct {
 	Schema string
 	Name   string
+	Id     string
 }
 
 // FkConstraint contains foreign key constraints
@@ -191,6 +192,7 @@ func (is *InfoSchemaImpl) processTable(conv *internal.Conv, table SchemaAndName,
 		return t, fmt.Errorf("couldn't get foreign key constraints for table %s.%s: %s", table.Schema, table.Name, err)
 	}
 
+	table.Id = tblId
 	colDefs, colIds, err := infoSchema.GetColumns(conv, table, constraints, primaryKeys)
 	if err != nil {
 		return t, fmt.Errorf("couldn't get schema for table %s.%s: %s", table.Schema, table.Name, err)

diff --git a/sources/common/toddl.go b/sources/common/toddl.go
@@ -47,19 +47,25 @@ import (
 // supported, an error is to be returned by the corresponding method.
 type ToDdl interface {
 	ToSpannerType(conv *internal.Conv, spType string, srcType schema.Type) (ddl.Type, []internal.SchemaIssue)
+	GetColumnAutoGen(conv *internal.Conv, autoGenCol ddl.AutoGenCol, colId string, tableId string) (*ddl.AutoGenCol, error)
 }
 
 type SchemaToSpannerInterface interface {
 	SchemaToSpannerDDL(conv *internal.Conv, toddl ToDdl) error
 	SchemaToSpannerDDLHelper(conv *internal.Conv, toddl ToDdl, srcTable schema.Table, isRestore bool) error
+	SchemaToSpannerSequenceHelper(conv *internal.Conv, srcSequence ddl.Sequence) error
 }
 
-type SchemaToSpannerImpl struct {}
+type SchemaToSpannerImpl struct{}
 
 // SchemaToSpannerDDL performs schema conversion from the source DB schema to
 // Spanner. It uses the source schema in conv.SrcSchema, and writes
 // the Spanner schema to conv.SpSchema.
 func (ss *SchemaToSpannerImpl) SchemaToSpannerDDL(conv *internal.Conv, toddl ToDdl) error {
+	srcSequences := conv.SrcSequences
+	for _, srcSequence := range srcSequences {
+		ss.SchemaToSpannerSequenceHelper(conv, srcSequence)
+	}
 	tableIds := GetSortedTableIdsBySrcName(conv.SrcSchema)
 	for _, tableId := range tableIds {
 		srcTable := conv.SrcSchema[tableId]
@@ -121,6 +127,20 @@ func (ss *SchemaToSpannerImpl) SchemaToSpannerDDLHelper(conv *internal.Conv, tod
 			issues = append(issues, internal.ArrayTypeNotSupported)
 			isNotNull = false
 		}
+		// Set auto generation for column
+		srcAutoGen := srcCol.AutoGen
+		var autoGenCol *ddl.AutoGenCol = &ddl.AutoGenCol{}
+		if srcAutoGen.Name != "" {
+			autoGenCol, err = toddl.GetColumnAutoGen(conv, srcAutoGen, srcColId, srcTable.Id)
+			if autoGenCol != nil {
+				if err != nil {
+					srcCol.Ignored.AutoIncrement = true
+					issues = append(issues, internal.AutoIncrement)
+				} else {
+					issues = append(issues, internal.SequenceCreated)
+				}
+			}
+		}
 		if len(issues) > 0 {
 			columnLevelIssues[srcColId] = issues
 		}
@@ -131,10 +151,7 @@ func (ss *SchemaToSpannerImpl) SchemaToSpannerDDLHelper(conv *internal.Conv, tod
 			NotNull: isNotNull,
 			Comment: "From: " + quoteIfNeeded(srcCol.Name) + " " + srcCol.Type.Print(),
 			Id:      srcColId,
-			AutoGen: ddl.AutoGenCol{
-				Name: "",
-				GenerationType: "",
-			},
+			AutoGen: *autoGenCol,
 		}
 		if !checkIfColumnIsPartOfPK(srcColId, srcTable.PrimaryKeys) {
 			totalNonKeyColumnSize += getColumnSize(ty.Name, ty.Len)
@@ -160,6 +177,32 @@ func (ss *SchemaToSpannerImpl) SchemaToSpannerDDLHelper(conv *internal.Conv, tod
 	return nil
 }
 
+func (ss *SchemaToSpannerImpl) SchemaToSpannerSequenceHelper(conv *internal.Conv, srcSequence ddl.Sequence) error {
+	switch srcSequence.SequenceKind {
+	case constants.AUTO_INCREMENT:
+		spSequence := ddl.Sequence{
+			Name:             srcSequence.Name,
+			Id:               srcSequence.Id,
+			SequenceKind:     "BIT REVERSED POSITIVE",
+			SkipRangeMin:     srcSequence.SkipRangeMin,
+			SkipRangeMax:     srcSequence.SkipRangeMax,
+			StartWithCounter: srcSequence.StartWithCounter,
+		}
+		conv.SpSequences[srcSequence.Id] = spSequence
+	default:
+		spSequence := ddl.Sequence{
+			Name:             srcSequence.Name,
+			Id:               srcSequence.Id,
+			SequenceKind:     "BIT REVERSED POSITIVE",
+			SkipRangeMin:     srcSequence.SkipRangeMin,
+			SkipRangeMax:     srcSequence.SkipRangeMax,
+			StartWithCounter: srcSequence.StartWithCounter,
+		}
+		conv.SpSequences[srcSequence.Id] = spSequence
+	}
+	return nil
+}
+
 func quoteIfNeeded(s string) string {
 	for _, r := range s {
 		if unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.IsPunct(r) {

diff --git a/sources/common/toddl_test.go b/sources/common/toddl_test.go
@@ -18,9 +18,11 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/internal"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/schema"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/spanner/ddl"
+	"github.com/stretchr/testify/assert"
 )
 
 func Test_quoteIfNeeded(t *testing.T) {
@@ -376,3 +378,38 @@ func Test_cvtForeignKeysForAReferenceTable(t *testing.T) {
 		})
 	}
 }
+
+func Test_SchemaToSpannerSequenceHelper(t *testing.T) {
+	expectedConv := internal.MakeConv()
+	expectedConv.SpSequences["s1"] = ddl.Sequence{
+		Name:             "Sequence1",
+		Id:               "s1",
+		SequenceKind:     "BIT REVERSED POSITIVE",
+		SkipRangeMin:     "1",
+		SkipRangeMax:     "2",
+		StartWithCounter: "3",
+	}
+	tc := []struct {
+		expectedConv *internal.Conv
+		srcSequence  ddl.Sequence
+	}{
+		{
+			expectedConv: expectedConv,
+			srcSequence: ddl.Sequence{
+				Name:             "Sequence1",
+				Id:               "s1",
+				SequenceKind:     constants.AUTO_INCREMENT,
+				SkipRangeMin:     "1",
+				SkipRangeMax:     "2",
+				StartWithCounter: "3",
+			},
+		},
+	}
+
+	for _, tt := range tc {
+		conv := internal.MakeConv()
+		ss := SchemaToSpannerImpl{}
+		ss.SchemaToSpannerSequenceHelper(conv, tt.srcSequence)
+		assert.Equal(t, expectedConv, conv)
+	}
+}
diff --git a/sources/dynamodb/schema_test.go b/sources/dynamodb/schema_test.go
@@ -656,7 +656,7 @@ func TestInfoSchemaImpl_GetTables(t *testing.T) {
 	isi := InfoSchemaImpl{client, nil, 10}
 	tables, err := isi.GetTables()
 	assert.Nil(t, err)
-	assert.Equal(t, []common.SchemaAndName{{"", "table-a"}, {"", "table-b"}}, tables)
+	assert.Equal(t, []common.SchemaAndName{{Schema: "", Name: "table-a", Id: ""}, {"", "table-b", ""}}, tables)
 }
 
 func TestInfoSchemaImpl_GetTableName(t *testing.T) {
@@ -705,7 +705,7 @@ func TestInfoSchemaImpl_GetColumns(t *testing.T) {
 	client := &mockDynamoClient{
 		scanOutputs: scanOutputs,
 	}
-	dySchema := common.SchemaAndName{Name: "test"}
+	dySchema := common.SchemaAndName{Name: "test", Id:  "t1"}
 
 	isi := InfoSchemaImpl{client, nil, 10}
 

diff --git a/sources/dynamodb/toddl.go b/sources/dynamodb/toddl.go
@@ -40,6 +40,10 @@ func (tdi ToDdlImpl) ToSpannerType(conv *internal.Conv, spType string, srcType s
 	return ty, issues
 }
 
+func (tdi ToDdlImpl) GetColumnAutoGen(conv *internal.Conv, autoGenCol ddl.AutoGenCol, colId string, tableId string) (*ddl.AutoGenCol, error) {
+	return nil, nil
+}
+
 func toSpannerTypeInternal(conv *internal.Conv, srcType schema.Type) (ddl.Type, []internal.SchemaIssue) {
 	switch srcType.Name {
 	case typeNumber:

diff --git a/sources/mysql/infoschema.go b/sources/mysql/infoschema.go
@@ -25,6 +25,7 @@ import (
 	_ "github.com/go-sql-driver/mysql" // The driver should be used via the database/sql package.
 	_ "github.com/lib/pq"
 
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/internal"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/profiles"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/schema"
@@ -185,6 +186,7 @@ func (isi InfoSchemaImpl) GetColumns(conv *internal.Conv, table common.SchemaAnd
 	var colName, dataType, isNullable, columnType string
 	var colDefault, colExtra sql.NullString
 	var charMaxLen, numericPrecision, numericScale sql.NullInt64
+	var colAutoGen ddl.AutoGenCol
 	for cols.Next() {
 		err := cols.Scan(&colName, &dataType, &columnType, &isNullable, &colDefault, &charMaxLen, &numericPrecision, &numericScale, &colExtra)
 		if err != nil {
@@ -203,16 +205,27 @@ func (isi InfoSchemaImpl) GetColumns(conv *internal.Conv, table common.SchemaAnd
 			}
 		}
 		ignored.Default = colDefault.Valid
+		colId := internal.GenerateColumnId()
 		if colExtra.String == "auto_increment" {
-			ignored.AutoIncrement = true
+			sequence := createSequence(conv)
+			colAutoGen = ddl.AutoGenCol{
+				Name:           sequence.Name,
+				GenerationType: constants.AUTO_INCREMENT,
+			}
+			sequence.ColumnsUsingSeq = map[string][]string{
+				table.Id: {colId},
+			}
+			conv.SrcSequences[sequence.Id] = sequence
+		} else {
+			colAutoGen = ddl.AutoGenCol{}
 		}
-		colId := internal.GenerateColumnId()
 		c := schema.Column{
 			Id:      colId,
 			Name:    colName,
 			Type:    toType(dataType, columnType, charMaxLen, numericPrecision, numericScale),
 			NotNull: common.ToNotNull(conv, isNullable),
 			Ignored: ignored,
+			AutoGen: colAutoGen,
 		}
 		colDefs[colId] = c
 		colIds = append(colIds, colId)
@@ -452,3 +465,18 @@ func valsToStrings(vals []sql.RawBytes) []string {
 	}
 	return s
 }
+
+func createSequence(conv *internal.Conv) ddl.Sequence {
+	id := internal.GenerateSequenceId()
+	sequenceName := "Sequence" + id[1:]
+	sequence := ddl.Sequence{
+		Id:           id,
+		Name:         sequenceName,
+		SequenceKind: "BIT REVERSED SEQUENCE",
+	}
+	conv.ConvLock.Lock()
+	defer conv.ConvLock.Unlock()
+	srcSequences := conv.SrcSequences
+	srcSequences[id] = sequence
+	return sequence
+}
diff --git a/sources/mysql/infoschema_test.go b/sources/mysql/infoschema_test.go
@@ -22,6 +22,7 @@ import (
 	"github.com/DATA-DOG/go-sqlmock"
 	"github.com/stretchr/testify/assert"
 
+	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/internal"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/profiles"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/schema"
@@ -237,7 +238,7 @@ func TestProcessSchemaMYSQL(t *testing.T) {
 			"f4":  schema.Column{Name: "f4", Type: schema.Type{Name: "float", Mods: []int64{24}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
 			"f8":  schema.Column{Name: "f8", Type: schema.Type{Name: "double", Mods: []int64{53}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
 			"i2":  schema.Column{Name: "i2", Type: schema.Type{Name: "smallint", Mods: []int64{16}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
-			"i4":  schema.Column{Name: "i4", Type: schema.Type{Name: "integer", Mods: []int64{32}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: true}, Id: ""},
+			"i4":  schema.Column{Name: "i4", Type: schema.Type{Name: "integer", Mods: []int64{32}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: "", AutoGen: ddl.AutoGenCol{Name: "Sequence34", GenerationType: constants.AUTO_INCREMENT}},
 			"i8":  schema.Column{Name: "i8", Type: schema.Type{Name: "bigint", Mods: []int64{64}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
 			"id":  schema.Column{Name: "id", Type: schema.Type{Name: "bigint", Mods: []int64{64}, ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
 			"s":   schema.Column{Name: "s", Type: schema.Type{Name: "set", Mods: []int64(nil), ArrayBounds: []int64{-1}}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""},
@@ -385,7 +386,7 @@ func TestProcessData_MultiCol(t *testing.T) {
 	}
 	internal.AssertSpSchema(conv, t, expectedSchema, stripSchemaComments(conv.SpSchema))
 	columnLevelIssues := map[string][]internal.SchemaIssue{
-		"c48": []internal.SchemaIssue{
+		"c49": []internal.SchemaIssue{
 			2,
 		},
 	}

diff --git a/sources/mysql/toddl.go b/sources/mysql/toddl.go
@@ -16,6 +16,8 @@
 package mysql
 
 import (
+	"fmt"
+
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/internal"
 	"github.com/GoogleCloudPlatform/spanner-migration-tool/schema"
@@ -49,6 +51,32 @@ func (tdi ToDdlImpl) ToSpannerType(conv *internal.Conv, spType string, srcType s
 	return ty, issues
 }
 
+func (tdi ToDdlImpl) GetColumnAutoGen(conv *internal.Conv, autoGenCol ddl.AutoGenCol, colId string, tableId string) (*ddl.AutoGenCol, error) {
+	switch autoGenCol.GenerationType {
+	case constants.AUTO_INCREMENT:
+		sequenceId := ""
+		srcSequences := conv.SrcSequences
+		for seqId, seq := range srcSequences {
+			if seq.Name == autoGenCol.Name {
+				sequenceId = seqId
+			}
+		}
+		if sequenceId == "" {
+			return &ddl.AutoGenCol{}, fmt.Errorf("sequence corresponding to column auto generation not found")
+		}
+		spSequences := conv.SpSequences
+		sequence := spSequences[sequenceId]
+		sequence.ColumnsUsingSeq = map[string][]string{
+			tableId: {colId},
+		}
+		spSequences[sequenceId] = sequence
+		conv.SpSequences = spSequences
+		return &ddl.AutoGenCol{Name: conv.SpSequences[sequenceId].Name, GenerationType: constants.SEQUENCE}, nil
+	default:
+		return &ddl.AutoGenCol{}, fmt.Errorf("auto generation not supported")
+	}
+}
+
 func toSpannerTypeInternal(srcType schema.Type, spType string) (ddl.Type, []internal.SchemaIssue) {
 	switch srcType.Name {
 	case "bool", "boolean":