From 3f43400e3ab5fd58eb94f64dc433e670f1a5c3ef Mon Sep 17 00:00:00 2001 From: wangzihuacool Date: Thu, 20 Oct 2022 13:34:31 +0000 Subject: [PATCH 01/11] add-rocksdb-as-transactional-engine --- go/base/context.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/go/base/context.go b/go/base/context.go index f3fe712d6..bdfe15625 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -427,6 +427,10 @@ func (this *MigrationContext) IsTransactionalTable() bool { { return true } + case "rocksdb": + { + return true + } } return false } From f053ccd9a6002cabd18105ea2c207f9f62960642 Mon Sep 17 00:00:00 2001 From: lukelewang Date: Wed, 23 Nov 2022 17:32:53 +0800 Subject: [PATCH 02/11] support rocksdb as transactional engine --- doc/command-line-flags.md | 3 +++ go/base/context.go | 17 +++++++++++++++-- go/cmd/gh-ost/main.go | 5 +++++ go/mysql/connection.go | 7 ++++--- go/mysql/connection_test.go | 15 ++++++++++----- 5 files changed, 37 insertions(+), 10 deletions(-) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 56bc6421a..5a6163189 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -246,6 +246,9 @@ Allows `gh-ost` to connect to the MySQL servers using encrypted connections, but `--ssl-key=/path/to/ssl-key.key`: SSL private key file (in PEM format). +### storage-engine +default is `innodb`. When set to `rocksdb`, some necessary changes (e.g. sets isolation level to READ_COMMITTED) is made to support rocksdb as transactional engine. + ### test-on-replica Issue the migration on a replica; do not modify data on master. Useful for validating, testing and benchmarking. See [`testing-on-replica`](testing-on-replica.md) diff --git a/go/base/context.go b/go/base/context.go index 7089874f2..cc5844ea4 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -270,8 +270,6 @@ func NewMigrationContext() *MigrationContext { Uuid: uuid.NewV4().String(), defaultNumRetries: 60, ChunkSize: 1000, - InspectorConnectionConfig: mysql.NewConnectionConfig(), - ApplierConnectionConfig: mysql.NewConnectionConfig(), MaxLagMillisecondsThrottleThreshold: 1500, CutOverLockTimeoutSeconds: 3, DMLBatchSize: 10, @@ -290,6 +288,21 @@ func NewMigrationContext() *MigrationContext { } } +func (this *MigrationContext) SetConnectionConfig(storageEngine string) error { + transactionIsolation := "REPEATABLE-READ" + switch storageEngine { + case "rocksdb": + transactionIsolation = "READ-COMMITTED" + case "innodb": + transactionIsolation = "REPEATABLE-READ" + default: + transactionIsolation = "REPEATABLE-READ" + } + this.InspectorConnectionConfig = mysql.NewConnectionConfig(transactionIsolation) + this.ApplierConnectionConfig = mysql.NewConnectionConfig(transactionIsolation) + return nil +} + func getSafeTableName(baseName string, suffix string) string { name := fmt.Sprintf("_%s_%s", baseName, suffix) if len(name) <= mysql.MaxTableNameLength { diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index c00b206f7..54083c390 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -68,6 +68,7 @@ func main() { flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)") flag.StringVar(&migrationContext.AlterStatement, "alter", "", "alter statement (mandatory)") flag.BoolVar(&migrationContext.AttemptInstantDDL, "attempt-instant-ddl", false, "Attempt to use instant DDL for this migration first") + storageEngine := flag.String("storage-engine", "innodb", "Specify table storage engine (default: 'innodb'). When 'rocksdb': change session transaction isolation level to READ_COMMITTED.") flag.BoolVar(&migrationContext.CountTableRows, "exact-rowcount", false, "actually count table rows as opposed to estimate them (results in more accurate progress estimation)") flag.BoolVar(&migrationContext.ConcurrentCountTableRows, "concurrent-rowcount", true, "(with --exact-rowcount), when true (default): count rows after row-copy begins, concurrently, and adjust row estimate later on; when false: first count rows, then start row copy") @@ -248,6 +249,10 @@ func main() { migrationContext.Log.Warning("--replication-lag-query is deprecated") } + if err := migrationContext.SetConnectionConfig(*storageEngine); err != nil { + migrationContext.Log.Fatale(err) + } + switch *cutOver { case "atomic", "default", "": migrationContext.CutOverType = base.CutOverAtomic diff --git a/go/mysql/connection.go b/go/mysql/connection.go index 6a5c890d8..dcf33ad29 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -18,7 +18,6 @@ import ( ) const ( - transactionIsolation = "REPEATABLE-READ" TLS_CONFIG_KEY = "ghost" ) @@ -30,11 +29,13 @@ type ConnectionConfig struct { ImpliedKey *InstanceKey tlsConfig *tls.Config Timeout float64 + transactionIsolation string } -func NewConnectionConfig() *ConnectionConfig { +func NewConnectionConfig(transactionIsolation string) *ConnectionConfig { config := &ConnectionConfig{ Key: InstanceKey{}, + transactionIsolation: transactionIsolation, } config.ImpliedKey = &config.Key return config @@ -126,7 +127,7 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string { "charset=utf8mb4,utf8,latin1", "interpolateParams=true", fmt.Sprintf("tls=%s", tlsOption), - fmt.Sprintf("transaction_isolation=%q", transactionIsolation), + fmt.Sprintf("transaction_isolation=%q", this.transactionIsolation), fmt.Sprintf("timeout=%fs", this.Timeout), fmt.Sprintf("readTimeout=%fs", this.Timeout), fmt.Sprintf("writeTimeout=%fs", this.Timeout), diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index 390774cd4..2f39c4f58 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -13,12 +13,17 @@ import ( test "github.com/openark/golib/tests" ) +const ( + transactionIsolation = "REPEATABLE-READ" +) + + func init() { log.SetLevel(log.ERROR) } func TestNewConnectionConfig(t *testing.T) { - c := NewConnectionConfig() + c := NewConnectionConfig(transactionIsolation) test.S(t).ExpectEquals(c.Key.Hostname, "") test.S(t).ExpectEquals(c.Key.Port, 0) test.S(t).ExpectEquals(c.ImpliedKey.Hostname, "") @@ -28,7 +33,7 @@ func TestNewConnectionConfig(t *testing.T) { } func TestDuplicateCredentials(t *testing.T) { - c := NewConnectionConfig() + c := NewConnectionConfig(transactionIsolation) c.Key = InstanceKey{Hostname: "myhost", Port: 3306} c.User = "gromit" c.Password = "penguin" @@ -48,7 +53,7 @@ func TestDuplicateCredentials(t *testing.T) { } func TestDuplicate(t *testing.T) { - c := NewConnectionConfig() + c := NewConnectionConfig(transactionIsolation) c.Key = InstanceKey{Hostname: "myhost", Port: 3306} c.User = "gromit" c.Password = "penguin" @@ -63,7 +68,7 @@ func TestDuplicate(t *testing.T) { } func TestGetDBUri(t *testing.T) { - c := NewConnectionConfig() + c := NewConnectionConfig(transactionIsolation) c.Key = InstanceKey{Hostname: "myhost", Port: 3306} c.User = "gromit" c.Password = "penguin" @@ -74,7 +79,7 @@ func TestGetDBUri(t *testing.T) { } func TestGetDBUriWithTLSSetup(t *testing.T) { - c := NewConnectionConfig() + c := NewConnectionConfig(transactionIsolation) c.Key = InstanceKey{Hostname: "myhost", Port: 3306} c.User = "gromit" c.Password = "penguin" From 3ee667a40e3495ce9842e879d6b46e0e76ad437e Mon Sep 17 00:00:00 2001 From: lukelewang Date: Fri, 25 Nov 2022 17:18:32 +0800 Subject: [PATCH 03/11] Modify tests to support rocksdb tests --- localtests/discard-fk/ignore_versions | 1 + localtests/fail-fk-parent/ignore_versions | 1 + localtests/fail-fk/ignore_versions | 1 + .../generated-columns-add/ignore_versions | 1 + .../generated-columns-rename/ignore_versions | 1 + .../generated-columns-unique/ignore_versions | 1 + localtests/generated-columns/ignore_versions | 1 + localtests/geometry/ignore_versions | 1 + localtests/spatial/ignore_versions | 1 + localtests/test.sh | 4 ++ script/cibuild-gh-ost-replica-tests | 40 +++++++++++++++++++ 11 files changed, 53 insertions(+) create mode 100644 localtests/discard-fk/ignore_versions create mode 100644 localtests/fail-fk-parent/ignore_versions create mode 100644 localtests/fail-fk/ignore_versions create mode 100644 localtests/generated-columns-add/ignore_versions create mode 100644 localtests/generated-columns-rename/ignore_versions create mode 100644 localtests/generated-columns-unique/ignore_versions create mode 100644 localtests/generated-columns/ignore_versions create mode 100644 localtests/geometry/ignore_versions create mode 100644 localtests/spatial/ignore_versions diff --git a/localtests/discard-fk/ignore_versions b/localtests/discard-fk/ignore_versions new file mode 100644 index 000000000..cf02abe24 --- /dev/null +++ b/localtests/discard-fk/ignore_versions @@ -0,0 +1 @@ +Percona \ No newline at end of file diff --git a/localtests/fail-fk-parent/ignore_versions b/localtests/fail-fk-parent/ignore_versions new file mode 100644 index 000000000..cf02abe24 --- /dev/null +++ b/localtests/fail-fk-parent/ignore_versions @@ -0,0 +1 @@ +Percona \ No newline at end of file diff --git a/localtests/fail-fk/ignore_versions b/localtests/fail-fk/ignore_versions new file mode 100644 index 000000000..cf02abe24 --- /dev/null +++ b/localtests/fail-fk/ignore_versions @@ -0,0 +1 @@ +Percona \ No newline at end of file diff --git a/localtests/generated-columns-add/ignore_versions b/localtests/generated-columns-add/ignore_versions new file mode 100644 index 000000000..cf02abe24 --- /dev/null +++ b/localtests/generated-columns-add/ignore_versions @@ -0,0 +1 @@ +Percona \ No newline at end of file diff --git a/localtests/generated-columns-rename/ignore_versions b/localtests/generated-columns-rename/ignore_versions new file mode 100644 index 000000000..cf02abe24 --- /dev/null +++ b/localtests/generated-columns-rename/ignore_versions @@ -0,0 +1 @@ +Percona \ No newline at end of file diff --git a/localtests/generated-columns-unique/ignore_versions b/localtests/generated-columns-unique/ignore_versions new file mode 100644 index 000000000..cf02abe24 --- /dev/null +++ b/localtests/generated-columns-unique/ignore_versions @@ -0,0 +1 @@ +Percona \ No newline at end of file diff --git a/localtests/generated-columns/ignore_versions b/localtests/generated-columns/ignore_versions new file mode 100644 index 000000000..cf02abe24 --- /dev/null +++ b/localtests/generated-columns/ignore_versions @@ -0,0 +1 @@ +Percona \ No newline at end of file diff --git a/localtests/geometry/ignore_versions b/localtests/geometry/ignore_versions new file mode 100644 index 000000000..cf02abe24 --- /dev/null +++ b/localtests/geometry/ignore_versions @@ -0,0 +1 @@ +Percona \ No newline at end of file diff --git a/localtests/spatial/ignore_versions b/localtests/spatial/ignore_versions new file mode 100644 index 000000000..cf02abe24 --- /dev/null +++ b/localtests/spatial/ignore_versions @@ -0,0 +1 @@ +Percona \ No newline at end of file diff --git a/localtests/test.sh b/localtests/test.sh index 5ca06ad41..d73dab273 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -99,9 +99,13 @@ test_single() { if [ -f $tests_path/$test_name/ignore_versions ] ; then ignore_versions=$(cat $tests_path/$test_name/ignore_versions) mysql_version=$(gh-ost-test-mysql-master -s -s -e "select @@version") + mysql_version_comment=$(gh-ost-test-mysql-master -s -s -e "select @@version_comment") if echo "$mysql_version" | egrep -q "^${ignore_versions}" ; then echo -n "Skipping: $test_name" return 0 + elif echo "$mysql_version_comment" | egrep -i -q "^${ignore_versions}" ; then + echo -n "Skipping: $test_name" + return 0 fi fi diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index c4dbfd292..bb60d7140 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -60,6 +60,46 @@ test_mysql_version() { gh-ost-test-mysql-master -uroot -e "create user 'gh-ost'@'%' identified by 'gh-ost'" gh-ost-test-mysql-master -uroot -e "grant all on *.* to 'gh-ost'@'%'" + local mysql_server=${mysql_version%-*} + if echo "$mysql_server" | egrep -i "percona" ; then + echo "### Preparing for rocksdb in PerconaServer" + gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_CFSTATS SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_DBSTATS SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_PERF_CONTEXT SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_PERF_CONTEXT_GLOBAL SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_CF_OPTIONS SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_GLOBAL_INFO SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_COMPACTION_HISTORY SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_COMPACTION_STATS SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_ACTIVE_COMPACTION_STATS SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_DDL SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_INDEX_FILE_MAP SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_LOCKS SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_TRX SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_DEADLOCK SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-master -uroot -e "set global default_storage_engine='ROCKSDB'" + gh-ost-test-mysql-master -uroot -e "set global transaction_isolation='READ-COMMITTED'" + + gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_CFSTATS SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_DBSTATS SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_PERF_CONTEXT SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_PERF_CONTEXT_GLOBAL SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_CF_OPTIONS SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_GLOBAL_INFO SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_COMPACTION_HISTORY SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_COMPACTION_STATS SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_ACTIVE_COMPACTION_STATS SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_DDL SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_INDEX_FILE_MAP SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_LOCKS SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_TRX SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_DEADLOCK SONAME 'ha_rocksdb.so'" + gh-ost-test-mysql-replica -uroot -e "set global default_storage_engine='ROCKSDB'" + gh-ost-test-mysql-replica -uroot -e "set global transaction_isolation='READ-COMMITTED'" + fi + echo "### Running gh-ost tests for $mysql_version" ./localtests/test.sh -b bin/gh-ost From ccf5b2e01df93ce1d0df9ca3b5bb199dc7c46fb2 Mon Sep 17 00:00:00 2001 From: wangzihuacool Date: Fri, 25 Nov 2022 09:39:57 +0000 Subject: [PATCH 04/11] gofmt --- go/mysql/connection.go | 16 ++++++++-------- go/mysql/connection_test.go | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/go/mysql/connection.go b/go/mysql/connection.go index dcf33ad29..15d7cb0a7 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -18,23 +18,23 @@ import ( ) const ( - TLS_CONFIG_KEY = "ghost" + TLS_CONFIG_KEY = "ghost" ) // ConnectionConfig is the minimal configuration required to connect to a MySQL server type ConnectionConfig struct { - Key InstanceKey - User string - Password string - ImpliedKey *InstanceKey - tlsConfig *tls.Config - Timeout float64 + Key InstanceKey + User string + Password string + ImpliedKey *InstanceKey + tlsConfig *tls.Config + Timeout float64 transactionIsolation string } func NewConnectionConfig(transactionIsolation string) *ConnectionConfig { config := &ConnectionConfig{ - Key: InstanceKey{}, + Key: InstanceKey{}, transactionIsolation: transactionIsolation, } config.ImpliedKey = &config.Key diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index 2f39c4f58..e8cdc2724 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -17,7 +17,6 @@ const ( transactionIsolation = "REPEATABLE-READ" ) - func init() { log.SetLevel(log.ERROR) } From 9c7857bd469b204399923a102c2ab120495d01c9 Mon Sep 17 00:00:00 2001 From: lukelewang Date: Fri, 25 Nov 2022 18:48:48 +0800 Subject: [PATCH 05/11] SetConnectionConfig --- go/base/context.go | 5 ++++- go/cmd/gh-ost/main.go | 14 +++++++------- go/mysql/connection.go | 16 ++++++++-------- go/mysql/connection_test.go | 1 - 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index cc5844ea4..b68bf7b1f 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -288,7 +288,7 @@ func NewMigrationContext() *MigrationContext { } } -func (this *MigrationContext) SetConnectionConfig(storageEngine string) error { +func (this *MigrationContext) SetConnectionConfig(storageEngine string, host string, port int, timeout float64) error { transactionIsolation := "REPEATABLE-READ" switch storageEngine { case "rocksdb": @@ -299,6 +299,9 @@ func (this *MigrationContext) SetConnectionConfig(storageEngine string) error { transactionIsolation = "REPEATABLE-READ" } this.InspectorConnectionConfig = mysql.NewConnectionConfig(transactionIsolation) + this.InspectorConnectionConfig.Key.Hostname = host + this.InspectorConnectionConfig.Key.Port = port + this.InspectorConnectionConfig.Timeout = timeout this.ApplierConnectionConfig = mysql.NewConnectionConfig(transactionIsolation) return nil } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 54083c390..145ef65b0 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -47,10 +47,10 @@ func acceptSignals(migrationContext *base.MigrationContext) { // main is the application's entry point. It will either spawn a CLI or HTTP interfaces. func main() { migrationContext := base.NewMigrationContext() - flag.StringVar(&migrationContext.InspectorConnectionConfig.Key.Hostname, "host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)") + host := flag.String("host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)") + port := flag.Int("port", 3306, "MySQL port (preferably a replica, not the master)") + timeout := flag.Float64("mysql-timeout", 0.0, "Connect, read and write timeout for MySQL") flag.StringVar(&migrationContext.AssumeMasterHostname, "assume-master-host", "", "(optional) explicitly tell gh-ost the identity of the master. Format: some.host.com[:port] This is useful in master-master setups where you wish to pick an explicit master, or in a tungsten-replicator where gh-ost is unable to determine the master") - flag.IntVar(&migrationContext.InspectorConnectionConfig.Key.Port, "port", 3306, "MySQL port (preferably a replica, not the master)") - flag.Float64Var(&migrationContext.InspectorConnectionConfig.Timeout, "mysql-timeout", 0.0, "Connect, read and write timeout for MySQL") flag.StringVar(&migrationContext.CliUser, "user", "", "MySQL user") flag.StringVar(&migrationContext.CliPassword, "password", "", "MySQL password") flag.StringVar(&migrationContext.CliMasterUser, "master-user", "", "MySQL user on master, if different from that on replica. Requires --assume-master-host") @@ -183,6 +183,10 @@ func main() { migrationContext.Log.SetLevel(log.ERROR) } + if err := migrationContext.SetConnectionConfig(*storageEngine, *host, *port, *timeout); err != nil { + migrationContext.Log.Fatale(err) + } + if migrationContext.AlterStatement == "" { log.Fatal("--alter must be provided and statement must not be empty") } @@ -249,10 +253,6 @@ func main() { migrationContext.Log.Warning("--replication-lag-query is deprecated") } - if err := migrationContext.SetConnectionConfig(*storageEngine); err != nil { - migrationContext.Log.Fatale(err) - } - switch *cutOver { case "atomic", "default", "": migrationContext.CutOverType = base.CutOverAtomic diff --git a/go/mysql/connection.go b/go/mysql/connection.go index dcf33ad29..15d7cb0a7 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -18,23 +18,23 @@ import ( ) const ( - TLS_CONFIG_KEY = "ghost" + TLS_CONFIG_KEY = "ghost" ) // ConnectionConfig is the minimal configuration required to connect to a MySQL server type ConnectionConfig struct { - Key InstanceKey - User string - Password string - ImpliedKey *InstanceKey - tlsConfig *tls.Config - Timeout float64 + Key InstanceKey + User string + Password string + ImpliedKey *InstanceKey + tlsConfig *tls.Config + Timeout float64 transactionIsolation string } func NewConnectionConfig(transactionIsolation string) *ConnectionConfig { config := &ConnectionConfig{ - Key: InstanceKey{}, + Key: InstanceKey{}, transactionIsolation: transactionIsolation, } config.ImpliedKey = &config.Key diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index 2f39c4f58..e8cdc2724 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -17,7 +17,6 @@ const ( transactionIsolation = "REPEATABLE-READ" ) - func init() { log.SetLevel(log.ERROR) } From da3514253f5cd848c1cbcaad63ad794427ed1833 Mon Sep 17 00:00:00 2001 From: lukelewang Date: Sat, 26 Nov 2022 01:38:12 +0800 Subject: [PATCH 06/11] add support for rocksdb --- go/base/context.go | 11 ++-- go/cmd/gh-ost/main.go | 8 +-- go/mysql/connection.go | 20 +++---- go/mysql/connection_test.go | 17 ++++-- localtests/test.sh | 11 ++-- script/cibuild-gh-ost-replica-tests | 88 +++++++++++++++-------------- 6 files changed, 84 insertions(+), 71 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index b68bf7b1f..be28ac2e7 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -270,6 +270,8 @@ func NewMigrationContext() *MigrationContext { Uuid: uuid.NewV4().String(), defaultNumRetries: 60, ChunkSize: 1000, + InspectorConnectionConfig: mysql.NewConnectionConfig(), + ApplierConnectionConfig: mysql.NewConnectionConfig(), MaxLagMillisecondsThrottleThreshold: 1500, CutOverLockTimeoutSeconds: 3, DMLBatchSize: 10, @@ -288,7 +290,7 @@ func NewMigrationContext() *MigrationContext { } } -func (this *MigrationContext) SetConnectionConfig(storageEngine string, host string, port int, timeout float64) error { +func (this *MigrationContext) SetConnectionConfig(storageEngine string) error { transactionIsolation := "REPEATABLE-READ" switch storageEngine { case "rocksdb": @@ -298,11 +300,8 @@ func (this *MigrationContext) SetConnectionConfig(storageEngine string, host str default: transactionIsolation = "REPEATABLE-READ" } - this.InspectorConnectionConfig = mysql.NewConnectionConfig(transactionIsolation) - this.InspectorConnectionConfig.Key.Hostname = host - this.InspectorConnectionConfig.Key.Port = port - this.InspectorConnectionConfig.Timeout = timeout - this.ApplierConnectionConfig = mysql.NewConnectionConfig(transactionIsolation) + this.InspectorConnectionConfig.TransactionIsolation = transactionIsolation + this.ApplierConnectionConfig.TransactionIsolation = transactionIsolation return nil } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 145ef65b0..98310f30b 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -47,10 +47,10 @@ func acceptSignals(migrationContext *base.MigrationContext) { // main is the application's entry point. It will either spawn a CLI or HTTP interfaces. func main() { migrationContext := base.NewMigrationContext() - host := flag.String("host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)") - port := flag.Int("port", 3306, "MySQL port (preferably a replica, not the master)") - timeout := flag.Float64("mysql-timeout", 0.0, "Connect, read and write timeout for MySQL") + flag.StringVar(&migrationContext.InspectorConnectionConfig.Key.Hostname, "host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)") flag.StringVar(&migrationContext.AssumeMasterHostname, "assume-master-host", "", "(optional) explicitly tell gh-ost the identity of the master. Format: some.host.com[:port] This is useful in master-master setups where you wish to pick an explicit master, or in a tungsten-replicator where gh-ost is unable to determine the master") + flag.IntVar(&migrationContext.InspectorConnectionConfig.Key.Port, "port", 3306, "MySQL port (preferably a replica, not the master)") + flag.Float64Var(&migrationContext.InspectorConnectionConfig.Timeout, "mysql-timeout", 0.0, "Connect, read and write timeout for MySQL") flag.StringVar(&migrationContext.CliUser, "user", "", "MySQL user") flag.StringVar(&migrationContext.CliPassword, "password", "", "MySQL password") flag.StringVar(&migrationContext.CliMasterUser, "master-user", "", "MySQL user on master, if different from that on replica. Requires --assume-master-host") @@ -183,7 +183,7 @@ func main() { migrationContext.Log.SetLevel(log.ERROR) } - if err := migrationContext.SetConnectionConfig(*storageEngine, *host, *port, *timeout); err != nil { + if err := migrationContext.SetConnectionConfig(*storageEngine); err != nil { migrationContext.Log.Fatale(err) } diff --git a/go/mysql/connection.go b/go/mysql/connection.go index 15d7cb0a7..6250925b7 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -29,13 +29,12 @@ type ConnectionConfig struct { ImpliedKey *InstanceKey tlsConfig *tls.Config Timeout float64 - transactionIsolation string + TransactionIsolation string } -func NewConnectionConfig(transactionIsolation string) *ConnectionConfig { +func NewConnectionConfig() *ConnectionConfig { config := &ConnectionConfig{ - Key: InstanceKey{}, - transactionIsolation: transactionIsolation, + Key: InstanceKey{}, } config.ImpliedKey = &config.Key return config @@ -44,11 +43,12 @@ func NewConnectionConfig(transactionIsolation string) *ConnectionConfig { // DuplicateCredentials creates a new connection config with given key and with same credentials as this config func (this *ConnectionConfig) DuplicateCredentials(key InstanceKey) *ConnectionConfig { config := &ConnectionConfig{ - Key: key, - User: this.User, - Password: this.Password, - tlsConfig: this.tlsConfig, - Timeout: this.Timeout, + Key: key, + User: this.User, + Password: this.Password, + tlsConfig: this.tlsConfig, + Timeout: this.Timeout, + TransactionIsolation: this.TransactionIsolation, } config.ImpliedKey = &config.Key return config @@ -127,7 +127,7 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string { "charset=utf8mb4,utf8,latin1", "interpolateParams=true", fmt.Sprintf("tls=%s", tlsOption), - fmt.Sprintf("transaction_isolation=%q", this.transactionIsolation), + fmt.Sprintf("transaction_isolation=%q", this.TransactionIsolation), fmt.Sprintf("timeout=%fs", this.Timeout), fmt.Sprintf("readTimeout=%fs", this.Timeout), fmt.Sprintf("writeTimeout=%fs", this.Timeout), diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index e8cdc2724..5667235f5 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -22,17 +22,18 @@ func init() { } func TestNewConnectionConfig(t *testing.T) { - c := NewConnectionConfig(transactionIsolation) + c := NewConnectionConfig() test.S(t).ExpectEquals(c.Key.Hostname, "") test.S(t).ExpectEquals(c.Key.Port, 0) test.S(t).ExpectEquals(c.ImpliedKey.Hostname, "") test.S(t).ExpectEquals(c.ImpliedKey.Port, 0) test.S(t).ExpectEquals(c.User, "") test.S(t).ExpectEquals(c.Password, "") + test.S(t).ExpectEquals(c.TransactionIsolation, "") } func TestDuplicateCredentials(t *testing.T) { - c := NewConnectionConfig(transactionIsolation) + c := NewConnectionConfig() c.Key = InstanceKey{Hostname: "myhost", Port: 3306} c.User = "gromit" c.Password = "penguin" @@ -40,6 +41,7 @@ func TestDuplicateCredentials(t *testing.T) { InsecureSkipVerify: true, ServerName: "feathers", } + c.TransactionIsolation = transactionIsolation dup := c.DuplicateCredentials(InstanceKey{Hostname: "otherhost", Port: 3310}) test.S(t).ExpectEquals(dup.Key.Hostname, "otherhost") @@ -49,13 +51,15 @@ func TestDuplicateCredentials(t *testing.T) { test.S(t).ExpectEquals(dup.User, "gromit") test.S(t).ExpectEquals(dup.Password, "penguin") test.S(t).ExpectEquals(dup.tlsConfig, c.tlsConfig) + test.S(t).ExpectEquals(dup.TransactionIsolation, c.TransactionIsolation) } func TestDuplicate(t *testing.T) { - c := NewConnectionConfig(transactionIsolation) + c := NewConnectionConfig() c.Key = InstanceKey{Hostname: "myhost", Port: 3306} c.User = "gromit" c.Password = "penguin" + c.TransactionIsolation = transactionIsolation dup := c.Duplicate() test.S(t).ExpectEquals(dup.Key.Hostname, "myhost") @@ -64,26 +68,29 @@ func TestDuplicate(t *testing.T) { test.S(t).ExpectEquals(dup.ImpliedKey.Port, 3306) test.S(t).ExpectEquals(dup.User, "gromit") test.S(t).ExpectEquals(dup.Password, "penguin") + test.S(t).ExpectEquals(dup.TransactionIsolation, transactionIsolation) } func TestGetDBUri(t *testing.T) { - c := NewConnectionConfig(transactionIsolation) + c := NewConnectionConfig() c.Key = InstanceKey{Hostname: "myhost", Port: 3306} c.User = "gromit" c.Password = "penguin" c.Timeout = 1.2345 + c.TransactionIsolation = transactionIsolation uri := c.GetDBUri("test") test.S(t).ExpectEquals(uri, `gromit:penguin@tcp(myhost:3306)/test?autocommit=true&charset=utf8mb4,utf8,latin1&interpolateParams=true&tls=false&transaction_isolation="REPEATABLE-READ"&timeout=1.234500s&readTimeout=1.234500s&writeTimeout=1.234500s`) } func TestGetDBUriWithTLSSetup(t *testing.T) { - c := NewConnectionConfig(transactionIsolation) + c := NewConnectionConfig() c.Key = InstanceKey{Hostname: "myhost", Port: 3306} c.User = "gromit" c.Password = "penguin" c.Timeout = 1.2345 c.tlsConfig = &tls.Config{} + c.TransactionIsolation = transactionIsolation uri := c.GetDBUri("test") test.S(t).ExpectEquals(uri, `gromit:penguin@tcp(myhost:3306)/test?autocommit=true&charset=utf8mb4,utf8,latin1&interpolateParams=true&tls=ghost&transaction_isolation="REPEATABLE-READ"&timeout=1.234500s&readTimeout=1.234500s&writeTimeout=1.234500s`) diff --git a/localtests/test.sh b/localtests/test.sh index d73dab273..14ecd83ea 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -11,6 +11,7 @@ tests_path=$(dirname $0) test_logfile=/tmp/gh-ost-test.log default_ghost_binary=/tmp/gh-ost-test ghost_binary="" +storage_engine=innodb exec_command_file=/tmp/gh-ost-test.bash ghost_structure_output_file=/tmp/gh-ost-test.ghost.structure.sql orig_content_output_file=/tmp/gh-ost-test.orig.content.csv @@ -24,12 +25,13 @@ replica_port= original_sql_mode= OPTIND=1 -while getopts "b:" OPTION +while getopts "b:s:" OPTION do case $OPTION in b) - ghost_binary="$OPTARG" - ;; + ghost_binary="$OPTARG";; + s) + storage_engine="$OPTARG";; esac done shift $((OPTIND-1)) @@ -158,7 +160,8 @@ test_single() { --assume-master-host=${master_host}:${master_port} --database=test \ --table=gh_ost_test \ - --alter='engine=innodb' \ + --storage-engine=${storage_engine} \ + --alter='engine=${storage_engine}' \ --exact-rowcount \ --assume-rbr \ --initially-drop-old-table \ diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index bb60d7140..31b313938 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -36,8 +36,16 @@ test_mysql_version() { mkdir -p sandbox/binary rm -rf sandbox/binary/* - gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/"$mysql_version".tar.xz --sandbox-binary ${PWD}/sandbox/binary - + local mysql_server=${mysql_version%-*} + if echo "$mysql_server" | egrep -i "percona" ; then + tarball_name=Percona-Server-${mysql_version#*-}-12-Linux.x86_64.glibc2.12-minimal.tar.gz + rm -f gh-ost-ci-env/mysql-tarballs/${tarball_name} + ln -s "$mysql_version".tar.xz gh-ost-ci-env/mysql-tarballs/${tarball_name} + gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/${tarball_name} --sandbox-binary ${PWD}/sandbox/binary + rm -f gh-ost-ci-env/mysql-tarballs/${tarball_name} + else + gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/"$mysql_version".tar.xz --sandbox-binary ${PWD}/sandbox/binary + fi mkdir -p sandboxes rm -rf sandboxes/* @@ -60,49 +68,45 @@ test_mysql_version() { gh-ost-test-mysql-master -uroot -e "create user 'gh-ost'@'%' identified by 'gh-ost'" gh-ost-test-mysql-master -uroot -e "grant all on *.* to 'gh-ost'@'%'" - local mysql_server=${mysql_version%-*} if echo "$mysql_server" | egrep -i "percona" ; then echo "### Preparing for rocksdb in PerconaServer" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_CFSTATS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_DBSTATS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_PERF_CONTEXT SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_PERF_CONTEXT_GLOBAL SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_CF_OPTIONS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_GLOBAL_INFO SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_COMPACTION_HISTORY SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_COMPACTION_STATS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_ACTIVE_COMPACTION_STATS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_DDL SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_INDEX_FILE_MAP SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_LOCKS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_TRX SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_DEADLOCK SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "set global default_storage_engine='ROCKSDB'" - gh-ost-test-mysql-master -uroot -e "set global transaction_isolation='READ-COMMITTED'" - - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_CFSTATS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_DBSTATS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_PERF_CONTEXT SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_PERF_CONTEXT_GLOBAL SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_CF_OPTIONS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_GLOBAL_INFO SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_COMPACTION_HISTORY SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_COMPACTION_STATS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_ACTIVE_COMPACTION_STATS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_DDL SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_INDEX_FILE_MAP SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_LOCKS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_TRX SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_DEADLOCK SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "set global default_storage_engine='ROCKSDB'" - gh-ost-test-mysql-replica -uroot -e "set global transaction_isolation='READ-COMMITTED'" + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_CFSTATS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_DBSTATS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_PERF_CONTEXT SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_PERF_CONTEXT_GLOBAL SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_CF_OPTIONS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_GLOBAL_INFO SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_COMPACTION_STATS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_DDL SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_INDEX_FILE_MAP SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_LOCKS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_TRX SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_DEADLOCK SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'set global default_storage_engine="ROCKSDB"' + gh-ost-test-mysql-master -uroot -e 'set global transaction_isolation="READ-COMMITTED"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_CFSTATS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_DBSTATS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_PERF_CONTEXT SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_PERF_CONTEXT_GLOBAL SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_CF_OPTIONS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_GLOBAL_INFO SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_COMPACTION_STATS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_DDL SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_INDEX_FILE_MAP SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_LOCKS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_TRX SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_DEADLOCK SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'set global default_storage_engine="ROCKSDB"' + gh-ost-test-mysql-replica -uroot -e 'set global transaction_isolation="READ-COMMITTED"' + + echo "### Running gh-ost tests for $mysql_version" + ./localtests/test.sh -b bin/gh-ost -s rocksdb + else + echo "### Running gh-ost tests for $mysql_version" + ./localtests/test.sh -b bin/gh-ost -s innodb fi - - echo "### Running gh-ost tests for $mysql_version" - ./localtests/test.sh -b bin/gh-ost - find sandboxes -name "stop_all" | bash } From af1e0d647f661e00b05b7701d36b025d535e6e01 Mon Sep 17 00:00:00 2001 From: lukelewang Date: Sat, 26 Nov 2022 01:43:11 +0800 Subject: [PATCH 07/11] add support for rocksdb --- go/base/context.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/base/context.go b/go/base/context.go index be28ac2e7..764a3e699 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -291,7 +291,7 @@ func NewMigrationContext() *MigrationContext { } func (this *MigrationContext) SetConnectionConfig(storageEngine string) error { - transactionIsolation := "REPEATABLE-READ" + var transactionIsolation string switch storageEngine { case "rocksdb": transactionIsolation = "READ-COMMITTED" From ed71099ce667e833533d95b883567babc5166473 Mon Sep 17 00:00:00 2001 From: lukelewang Date: Sat, 26 Nov 2022 13:08:19 +0800 Subject: [PATCH 08/11] add percona to versions in workflows --- .github/workflows/replica-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/replica-tests.yml b/.github/workflows/replica-tests.yml index 0a82f0010..f2a52ece8 100644 --- a/.github/workflows/replica-tests.yml +++ b/.github/workflows/replica-tests.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - version: [mysql-5.7.25,mysql-8.0.16] + version: [mysql-5.7.25,mysql-8.0.16,PerconaServer-8.0.21] steps: - uses: actions/checkout@v2 From 6e2be1d44fb5e19e4c6ecf044b220810b0c77927 Mon Sep 17 00:00:00 2001 From: lukelewang Date: Sun, 27 Nov 2022 13:54:01 +0800 Subject: [PATCH 09/11] add description and optimize tests --- doc/command-line-flags.md | 11 ++++++++++- go/base/context.go | 2 -- go/cmd/gh-ost/main.go | 3 +++ script/cibuild-gh-ost-replica-tests | 24 ------------------------ 4 files changed, 13 insertions(+), 27 deletions(-) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 5a6163189..82cb884c2 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -247,7 +247,16 @@ Allows `gh-ost` to connect to the MySQL servers using encrypted connections, but `--ssl-key=/path/to/ssl-key.key`: SSL private key file (in PEM format). ### storage-engine -default is `innodb`. When set to `rocksdb`, some necessary changes (e.g. sets isolation level to READ_COMMITTED) is made to support rocksdb as transactional engine. +default is `innodb`, and `rocksdb`as an optional. InnoDB and RocksDB are both transactional engines, supporting both shared and exclusive row locks. + +But RocksDB currently lacks a few features support compared to InnoDB: +- Gap Locks +- Foreign Key +- Generated Columns +- Spatial +- Geometry + +When `--storage-engine=rocksdb`, `gh-ost` will make some changes necessary (e.g. sets isolation level to `READ_COMMITTED`) to support RocksDB. ### test-on-replica diff --git a/go/base/context.go b/go/base/context.go index 764a3e699..e3472f5bd 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -295,8 +295,6 @@ func (this *MigrationContext) SetConnectionConfig(storageEngine string) error { switch storageEngine { case "rocksdb": transactionIsolation = "READ-COMMITTED" - case "innodb": - transactionIsolation = "REPEATABLE-READ" default: transactionIsolation = "REPEATABLE-READ" } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 98310f30b..e21430636 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -252,6 +252,9 @@ func main() { if *replicationLagQuery != "" { migrationContext.Log.Warning("--replication-lag-query is deprecated") } + if *storageEngine == "rocksdb" { + migrationContext.Log.Warning("RocksDB storage engine support is experimental") + } switch *cutOver { case "atomic", "default", "": diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index 31b313938..90eb856bc 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -71,33 +71,9 @@ test_mysql_version() { if echo "$mysql_server" | egrep -i "percona" ; then echo "### Preparing for rocksdb in PerconaServer" gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_CFSTATS SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_DBSTATS SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_PERF_CONTEXT SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_PERF_CONTEXT_GLOBAL SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_CF_OPTIONS SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_GLOBAL_INFO SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_COMPACTION_STATS SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_DDL SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_INDEX_FILE_MAP SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_LOCKS SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_TRX SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_DEADLOCK SONAME "ha_rocksdb.so"' gh-ost-test-mysql-master -uroot -e 'set global default_storage_engine="ROCKSDB"' gh-ost-test-mysql-master -uroot -e 'set global transaction_isolation="READ-COMMITTED"' gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_CFSTATS SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_DBSTATS SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_PERF_CONTEXT SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_PERF_CONTEXT_GLOBAL SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_CF_OPTIONS SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_GLOBAL_INFO SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_COMPACTION_STATS SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_DDL SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_INDEX_FILE_MAP SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_LOCKS SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_TRX SONAME "ha_rocksdb.so"' - gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_DEADLOCK SONAME "ha_rocksdb.so"' gh-ost-test-mysql-replica -uroot -e 'set global default_storage_engine="ROCKSDB"' gh-ost-test-mysql-replica -uroot -e 'set global transaction_isolation="READ-COMMITTED"' From 9f3cf74444823a53dc2d1a99c6c561a5a8b8abc1 Mon Sep 17 00:00:00 2001 From: wangzihuacool <47876169+wangzihuacool@users.noreply.github.com> Date: Tue, 29 Nov 2022 10:16:58 +0800 Subject: [PATCH 10/11] Apply suggestions from code review Co-authored-by: dm-2 <45519614+dm-2@users.noreply.github.com> --- doc/command-line-flags.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 82cb884c2..021462fa2 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -247,7 +247,7 @@ Allows `gh-ost` to connect to the MySQL servers using encrypted connections, but `--ssl-key=/path/to/ssl-key.key`: SSL private key file (in PEM format). ### storage-engine -default is `innodb`, and `rocksdb`as an optional. InnoDB and RocksDB are both transactional engines, supporting both shared and exclusive row locks. +Default is `innodb`, and `rocksdb` support is currently experimental. InnoDB and RocksDB are both transactional engines, supporting both shared and exclusive row locks. But RocksDB currently lacks a few features support compared to InnoDB: - Gap Locks From 20af3af283d88ec3825cc99e140d28411ae7418a Mon Sep 17 00:00:00 2001 From: wangzihuacool <47876169+wangzihuacool@users.noreply.github.com> Date: Tue, 29 Nov 2022 10:30:01 +0800 Subject: [PATCH 11/11] Apply suggestions from code review Co-authored-by: Tim Vaillancourt --- go/cmd/gh-ost/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index e21430636..3daf24441 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -68,7 +68,7 @@ func main() { flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)") flag.StringVar(&migrationContext.AlterStatement, "alter", "", "alter statement (mandatory)") flag.BoolVar(&migrationContext.AttemptInstantDDL, "attempt-instant-ddl", false, "Attempt to use instant DDL for this migration first") - storageEngine := flag.String("storage-engine", "innodb", "Specify table storage engine (default: 'innodb'). When 'rocksdb': change session transaction isolation level to READ_COMMITTED.") + storageEngine := flag.String("storage-engine", "innodb", "Specify table storage engine (default: 'innodb'). When 'rocksdb': the session transaction isolation level is changed from REPEATABLE_READ to READ_COMMITTED.") flag.BoolVar(&migrationContext.CountTableRows, "exact-rowcount", false, "actually count table rows as opposed to estimate them (results in more accurate progress estimation)") flag.BoolVar(&migrationContext.ConcurrentCountTableRows, "concurrent-rowcount", true, "(with --exact-rowcount), when true (default): count rows after row-copy begins, concurrently, and adjust row estimate later on; when false: first count rows, then start row copy")