diff --git a/cdc/Makefile b/cdc/Makefile index 6b81bea1..ce2342d9 100644 --- a/cdc/Makefile +++ b/cdc/Makefile @@ -32,7 +32,7 @@ PACKAGE_LIST := go list ./... | grep -vE 'vendor|proto|cdc\/tests|integration|te PACKAGES := $$($(PACKAGE_LIST)) FILES := $$(find . -name '*.go' -type f | grep -vE 'vendor|kv_gen|proto|pb\.go|pb\.gw\.go') TEST_FILES := $$(find . -name '*_test.go' -type f | grep -vE 'vendor|kv_gen|integration|testing_utils') -FAILPOINT_DIR := $$(for p in $(PACKAGES); do echo $${p\#"github.com/tikv/migration/cdc/"}|grep -v "github.com/pingcap/migration/cdc/"; done) +FAILPOINT_DIR := $$(for p in $(PACKAGES); do echo $${p\#"github.com/tikv/migration/cdc/"}|grep -v "github.com/tikv/migration/cdc/"; done) FAILPOINT := tools/bin/failpoint-ctl FAILPOINT_ENABLE := $$(echo $(FAILPOINT_DIR) | xargs $(FAILPOINT) enable >/dev/null) diff --git a/cdc/cdc/capture/capture.go b/cdc/cdc/capture/capture.go index 22c962e3..2aa15d2f 100644 --- a/cdc/cdc/capture/capture.go +++ b/cdc/cdc/capture/capture.go @@ -30,8 +30,8 @@ import ( "github.com/tikv/migration/cdc/cdc/model" "github.com/tikv/migration/cdc/cdc/owner" "github.com/tikv/migration/cdc/cdc/processor" - "github.com/tikv/migration/cdc/cdc/processor/pipeline/system" - ssystem "github.com/tikv/migration/cdc/cdc/sorter/leveldb/system" + + // ssystem "github.com/tikv/migration/cdc/cdc/sorter/leveldb/system" "github.com/tikv/migration/cdc/pkg/config" cdcContext "github.com/tikv/migration/cdc/pkg/context" cerror "github.com/tikv/migration/cdc/pkg/errors" @@ -66,10 +66,10 @@ type Capture struct { grpcPool kv.GrpcPool regionCache *tikv.RegionCache TimeAcquirer pdtime.TimeAcquirer - sorterSystem *ssystem.System + // sorterSystem *ssystem.System enableNewScheduler bool - tableActorSystem *system.System + // keyspanActorSystem *system.System // MessageServer is the receiver of the messages from the other nodes. // It should be recreated each time the capture is restarted. @@ -141,40 +141,6 @@ func (c *Capture) reset(ctx context.Context) error { c.TimeAcquirer.Stop() } c.TimeAcquirer = pdtime.NewTimeAcquirer(c.pdClient) - - if c.tableActorSystem != nil { - err := c.tableActorSystem.Stop() - if err != nil { - log.Warn("stop table actor system failed", zap.Error(err)) - } - } - if conf.Debug.EnableTableActor { - c.tableActorSystem = system.NewSystem() - err = c.tableActorSystem.Start(ctx) - if err != nil { - return errors.Annotate( - cerror.WrapError(cerror.ErrNewCaptureFailed, err), - "create table actor system") - } - } - if conf.Debug.EnableDBSorter { - if c.sorterSystem != nil { - err := c.sorterSystem.Stop() - if err != nil { - log.Warn("stop sorter system failed", zap.Error(err)) - } - } - // Sorter dir has been set and checked when server starts. - // See https://github.com/tikv/migration/cdc/blob/9dad09/cdc/server.go#L275 - sortDir := config.GetGlobalServerConfig().Sorter.SortDir - c.sorterSystem = ssystem.NewSystem(sortDir, conf.Debug.DB) - err = c.sorterSystem.Start(ctx) - if err != nil { - return errors.Annotate( - cerror.WrapError(cerror.ErrNewCaptureFailed, err), - "create sorter system") - } - } if c.grpcPool != nil { c.grpcPool.Close() } @@ -257,17 +223,17 @@ func (c *Capture) Run(ctx context.Context) error { func (c *Capture) run(stdCtx context.Context) error { ctx := cdcContext.NewContext(stdCtx, &cdcContext.GlobalVars{ - PDClient: c.pdClient, - KVStorage: c.kvStorage, - CaptureInfo: c.info, - EtcdClient: c.etcdClient, - GrpcPool: c.grpcPool, - RegionCache: c.regionCache, - TimeAcquirer: c.TimeAcquirer, - TableActorSystem: c.tableActorSystem, - SorterSystem: c.sorterSystem, - MessageServer: c.MessageServer, - MessageRouter: c.MessageRouter, + PDClient: c.pdClient, + KVStorage: c.kvStorage, + CaptureInfo: c.info, + EtcdClient: c.etcdClient, + GrpcPool: c.grpcPool, + RegionCache: c.regionCache, + TimeAcquirer: c.TimeAcquirer, + // KeySpanActorSystem: c.keyspanActorSystem, + // SorterSystem: c.sorterSystem, + MessageServer: c.MessageServer, + MessageRouter: c.MessageRouter, }) err := c.register(ctx) if err != nil { @@ -535,20 +501,6 @@ func (c *Capture) AsyncClose() { c.regionCache.Close() c.regionCache = nil } - if c.tableActorSystem != nil { - err := c.tableActorSystem.Stop() - if err != nil { - log.Warn("stop table actor system failed", zap.Error(err)) - } - c.tableActorSystem = nil - } - if c.sorterSystem != nil { - err := c.sorterSystem.Stop() - if err != nil { - log.Warn("stop sorter system failed", zap.Error(err)) - } - c.sorterSystem = nil - } if c.enableNewScheduler { c.grpcService.Reset(nil) diff --git a/cdc/cdc/capture/http_handler.go b/cdc/cdc/capture/http_handler.go index 80a75461..b4abf03b 100644 --- a/cdc/cdc/capture/http_handler.go +++ b/cdc/cdc/capture/http_handler.go @@ -166,11 +166,11 @@ func (h *HTTPHandler) GetChangefeed(c *gin.Context) { taskStatus := make([]model.CaptureTaskStatus, 0, len(processorInfos)) for captureID, status := range processorInfos { - tables := make([]int64, 0) - for tableID := range status.Tables { - tables = append(tables, tableID) + keyspans := make([]uint64, 0) + for keyspanID := range status.KeySpans { + keyspans = append(keyspans, keyspanID) } - taskStatus = append(taskStatus, model.CaptureTaskStatus{CaptureID: captureID, Tables: tables, Operation: status.Operation}) + taskStatus = append(taskStatus, model.CaptureTaskStatus{CaptureID: captureID, KeySpans: keyspans, Operation: status.Operation}) } changefeedDetail := &model.ChangefeedDetail{ @@ -424,17 +424,17 @@ func (h *HTTPHandler) RemoveChangefeed(c *gin.Context) { c.Status(http.StatusAccepted) } -// RebalanceTable rebalances tables -// @Summary rebalance tables -// @Description rebalance all tables of a changefeed +// RebalanceKeySpan rebalances keyspans +// @Summary rebalance keyspans +// @Description rebalance all keyspans of a changefeed // @Tags changefeed // @Accept json // @Produce json // @Param changefeed_id path string true "changefeed_id" // @Success 202 // @Failure 500,400 {object} model.HTTPError -// @Router /api/v1/changefeeds/{changefeed_id}/tables/rebalance_table [post] -func (h *HTTPHandler) RebalanceTable(c *gin.Context) { +// @Router /api/v1/changefeeds/{changefeed_id}/keyspans/rebalance_keyspan [post] +func (h *HTTPHandler) RebalanceKeySpan(c *gin.Context) { if !h.capture.IsOwner() { h.forwardToOwner(c) return @@ -462,19 +462,19 @@ func (h *HTTPHandler) RebalanceTable(c *gin.Context) { c.Status(http.StatusAccepted) } -// MoveTable moves a table to target capture -// @Summary move table -// @Description move one table to the target capture +// MoveKeySpan moves a keyspan to target capture +// @Summary move keyspan +// @Description move one keyspan to the target capture // @Tags changefeed // @Accept json // @Produce json // @Param changefeed_id path string true "changefeed_id" -// @Param table_id body integer true "table_id" +// @Param keyspan_id body integer true "keyspan_id" // @Param capture_id body string true "capture_id" // @Success 202 // @Failure 500,400 {object} model.HTTPError -// @Router /api/v1/changefeeds/{changefeed_id}/tables/move_table [post] -func (h *HTTPHandler) MoveTable(c *gin.Context) { +// @Router /api/v1/changefeeds/{changefeed_id}/keyspans/move_keyspan [post] +func (h *HTTPHandler) MoveKeySpan(c *gin.Context) { if !h.capture.IsOwner() { h.forwardToOwner(c) return @@ -495,7 +495,7 @@ func (h *HTTPHandler) MoveTable(c *gin.Context) { data := struct { CaptureID string `json:"capture_id"` - TableID int64 `json:"table_id"` + KeySpanID uint64 `json:"keyspan_id"` }{} err = c.BindJSON(&data) if err != nil { @@ -509,7 +509,7 @@ func (h *HTTPHandler) MoveTable(c *gin.Context) { } _ = h.capture.OperateOwnerUnderLock(func(owner *owner.Owner) error { - owner.ManualSchedule(changefeedID, data.CaptureID, data.TableID) + owner.ManualSchedule(changefeedID, data.CaptureID, data.KeySpanID) return nil }) @@ -586,7 +586,7 @@ func (h *HTTPHandler) GetProcessor(c *gin.Context) { return } position, exist := positions[captureID] - // Note: for the case that no tables are attached to a newly created changefeed, + // Note: for the case that no keyspans are attached to a newly created changefeed, // we just do not report an error. var processorDetail model.ProcessorDetail if exist { @@ -596,11 +596,11 @@ func (h *HTTPHandler) GetProcessor(c *gin.Context) { Count: position.Count, Error: position.Error, } - tables := make([]int64, 0) - for tableID := range status.Tables { - tables = append(tables, tableID) + keyspans := make([]uint64, 0) + for keyspanID := range status.KeySpans { + keyspans = append(keyspans, keyspanID) } - processorDetail.Tables = tables + processorDetail.KeySpans = keyspans } c.IndentedJSON(http.StatusOK, &processorDetail) } diff --git a/cdc/cdc/capture/http_validator.go b/cdc/cdc/capture/http_validator.go index b32630db..a29f2363 100644 --- a/cdc/cdc/capture/http_validator.go +++ b/cdc/cdc/capture/http_validator.go @@ -19,16 +19,13 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" - tidbkv "github.com/pingcap/tidb/kv" "github.com/r3labs/diff" "github.com/tikv/client-go/v2/oracle" - "github.com/tikv/migration/cdc/cdc/entry" - "github.com/tikv/migration/cdc/cdc/kv" "github.com/tikv/migration/cdc/cdc/model" "github.com/tikv/migration/cdc/cdc/sink" "github.com/tikv/migration/cdc/pkg/config" cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/filter" + "github.com/tikv/migration/cdc/pkg/txnutil/gc" "github.com/tikv/migration/cdc/pkg/util" "github.com/tikv/migration/cdc/pkg/version" @@ -80,19 +77,23 @@ func verifyCreateChangefeedConfig(ctx context.Context, changefeedConfig model.Ch // init replicaConfig replicaConfig := config.GetDefaultReplicaConfig() - replicaConfig.ForceReplicate = changefeedConfig.ForceReplicate - if changefeedConfig.MounterWorkerNum != 0 { - replicaConfig.Mounter.WorkerNum = changefeedConfig.MounterWorkerNum - } + /* + replicaConfig.ForceReplicate = changefeedConfig.ForceReplicate + if changefeedConfig.MounterWorkerNum != 0 { + replicaConfig.Mounter.WorkerNum = changefeedConfig.MounterWorkerNum + } + */ if changefeedConfig.SinkConfig != nil { replicaConfig.Sink = changefeedConfig.SinkConfig } - if len(changefeedConfig.IgnoreTxnStartTs) != 0 { - replicaConfig.Filter.IgnoreTxnStartTs = changefeedConfig.IgnoreTxnStartTs - } - if len(changefeedConfig.FilterRules) != 0 { - replicaConfig.Filter.Rules = changefeedConfig.FilterRules - } + /* + if len(changefeedConfig.IgnoreTxnStartTs) != 0 { + replicaConfig.Filter.IgnoreTxnStartTs = changefeedConfig.IgnoreTxnStartTs + } + if len(changefeedConfig.FilterRules) != 0 { + replicaConfig.Filter.Rules = changefeedConfig.FilterRules + } + */ captureInfos, err := capture.owner.StatusProvider().GetCaptures(ctx) if err != nil { @@ -127,16 +128,6 @@ func verifyCreateChangefeedConfig(ctx context.Context, changefeedConfig model.Ch CreatorVersion: version.ReleaseVersion, } - if !replicaConfig.ForceReplicate && !changefeedConfig.IgnoreIneligibleTable { - ineligibleTables, _, err := verifyTables(replicaConfig, capture.kvStorage, changefeedConfig.StartTS) - if err != nil { - return nil, err - } - if len(ineligibleTables) != 0 { - return nil, cerror.ErrTableIneligible.GenWithStackByArgs(ineligibleTables) - } - } - tz, err := util.GetTimezone(changefeedConfig.TimeZone) if err != nil { return nil, cerror.ErrAPIInvalidParam.Wrap(errors.Annotatef(err, "invalid timezone:%s", changefeedConfig.TimeZone)) @@ -164,22 +155,6 @@ func verifyUpdateChangefeedConfig(ctx context.Context, changefeedConfig model.Ch } // verify rules - if len(changefeedConfig.FilterRules) != 0 { - newInfo.Config.Filter.Rules = changefeedConfig.FilterRules - _, err = filter.VerifyRules(newInfo.Config) - if err != nil { - return nil, cerror.ErrChangefeedUpdateRefused.GenWithStackByArgs(err.Error()) - } - } - - if len(changefeedConfig.IgnoreTxnStartTs) != 0 { - newInfo.Config.Filter.IgnoreTxnStartTs = changefeedConfig.IgnoreTxnStartTs - } - - if changefeedConfig.MounterWorkerNum != 0 { - newInfo.Config.Mounter.WorkerNum = changefeedConfig.MounterWorkerNum - } - if changefeedConfig.SinkConfig != nil { newInfo.Config.Sink = changefeedConfig.SinkConfig } @@ -198,30 +173,3 @@ func verifyUpdateChangefeedConfig(ctx context.Context, changefeedConfig model.Ch return newInfo, nil } - -func verifyTables(replicaConfig *config.ReplicaConfig, storage tidbkv.Storage, startTs uint64) (ineligibleTables, eligibleTables []model.TableName, err error) { - filter, err := filter.NewFilter(replicaConfig) - if err != nil { - return nil, nil, errors.Trace(err) - } - meta, err := kv.GetSnapshotMeta(storage, startTs) - if err != nil { - return nil, nil, errors.Trace(err) - } - snap, err := entry.NewSingleSchemaSnapshotFromMeta(meta, startTs, false /* explicitTables */) - if err != nil { - return nil, nil, errors.Trace(err) - } - - for _, tableInfo := range snap.Tables() { - if filter.ShouldIgnoreTable(tableInfo.TableName.Schema, tableInfo.TableName.Table) { - continue - } - if !tableInfo.IsEligible(false /* forceReplicate */) { - ineligibleTables = append(ineligibleTables, tableInfo.TableName) - } else { - eligibleTables = append(eligibleTables, tableInfo.TableName) - } - } - return -} diff --git a/cdc/cdc/capture/http_validator_test.go b/cdc/cdc/capture/http_validator_test.go index 0c260f36..77cedc7e 100644 --- a/cdc/cdc/capture/http_validator_test.go +++ b/cdc/cdc/capture/http_validator_test.go @@ -44,7 +44,7 @@ func TestVerifyUpdateChangefeedConfig(t *testing.T) { require.Nil(t, newInfo) // test verify success - changefeedConfig = model.ChangefeedConfig{MounterWorkerNum: 32} + changefeedConfig = model.ChangefeedConfig{SinkConfig: &config.SinkConfig{Protocol: "test"}} newInfo, err = verifyUpdateChangefeedConfig(ctx, changefeedConfig, oldInfo) require.Nil(t, err) require.NotNil(t, newInfo) diff --git a/cdc/cdc/entry/codec.go b/cdc/cdc/entry/codec.go deleted file mode 100644 index 393d3739..00000000 --- a/cdc/cdc/entry/codec.go +++ /dev/null @@ -1,302 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package entry - -import ( - "bytes" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/parser/mysql" - "github.com/pingcap/tidb/tablecodec" - "github.com/pingcap/tidb/types" - "github.com/pingcap/tidb/util/codec" - "github.com/pingcap/tidb/util/rowcodec" - "github.com/tikv/migration/cdc/cdc/model" - cerror "github.com/tikv/migration/cdc/pkg/errors" -) - -var ( - tablePrefix = []byte{'t'} - recordPrefix = []byte("_r") - metaPrefix = []byte("m") -) - -var ( - intLen = 8 - tablePrefixLen = len(tablePrefix) - recordPrefixLen = len(recordPrefix) - metaPrefixLen = len(metaPrefix) - prefixTableIDLen = tablePrefixLen + intLen /*tableID*/ - prefixRecordIDLen = recordPrefixLen + intLen /*recordID*/ -) - -// MetaType is for data structure meta/data flag. -type MetaType byte - -const ( - // UnknownMetaType is used for all unknown meta types - UnknownMetaType MetaType = 0 - // StringMeta is the flag for string meta. - StringMeta MetaType = 'S' - // StringData is the flag for string data. - StringData MetaType = 's' - // HashMeta is the flag for hash meta. - HashMeta MetaType = 'H' - // HashData is the flag for hash data. - HashData MetaType = 'h' - // ListMeta is the flag for list meta. - ListMeta MetaType = 'L' - // ListData is the flag for list data. - ListData MetaType = 'l' -) - -type meta interface { - getType() MetaType -} - -type metaHashData struct { - key string - field []byte -} - -func (d metaHashData) getType() MetaType { - return HashData -} - -type metaListData struct { - key string - index int64 -} - -func (d metaListData) getType() MetaType { - return ListData -} - -type other struct { - tp MetaType -} - -func (d other) getType() MetaType { - return d.tp -} - -func decodeTableID(key []byte) (rest []byte, tableID int64, err error) { - if len(key) < prefixTableIDLen || !bytes.HasPrefix(key, tablePrefix) { - return nil, 0, cerror.ErrInvalidRecordKey.GenWithStackByArgs(key) - } - key = key[tablePrefixLen:] - rest, tableID, err = codec.DecodeInt(key) - if err != nil { - return nil, 0, cerror.WrapError(cerror.ErrCodecDecode, err) - } - return -} - -func decodeRecordID(key []byte) (rest []byte, recordID int64, err error) { - if len(key) < prefixRecordIDLen || !bytes.HasPrefix(key, recordPrefix) { - return nil, 0, cerror.ErrInvalidRecordKey.GenWithStackByArgs(key) - } - key = key[recordPrefixLen:] - rest, recordID, err = codec.DecodeInt(key) - if err != nil { - return nil, 0, cerror.WrapError(cerror.ErrCodecDecode, err) - } - return -} - -func decodeMetaKey(ek []byte) (meta, error) { - if !bytes.HasPrefix(ek, metaPrefix) { - return nil, cerror.ErrInvalidRecordKey.GenWithStackByArgs(ek) - } - - ek = ek[metaPrefixLen:] - ek, rawKey, err := codec.DecodeBytes(ek, nil) - if err != nil { - return nil, cerror.WrapError(cerror.ErrCodecDecode, err) - } - key := string(rawKey) - - ek, rawTp, err := codec.DecodeUint(ek) - if err != nil { - return nil, cerror.WrapError(cerror.ErrCodecDecode, err) - } - switch MetaType(rawTp) { - case HashData: - if len(ek) > 0 { - var field []byte - _, field, err = codec.DecodeBytes(ek, nil) - if err != nil { - return nil, cerror.WrapError(cerror.ErrCodecDecode, err) - } - return metaHashData{key: key, field: field}, nil - } - if len(ek) > 0 { - // TODO: warning hash key decode failure - panic("hash key decode failure, should never happen") - } - case ListData: - if len(ek) == 0 { - panic("list key decode failure") - } - var index int64 - _, index, err = codec.DecodeInt(ek) - if err != nil { - return nil, cerror.WrapError(cerror.ErrCodecDecode, err) - } - return metaListData{key: key, index: index}, nil - // TODO decode other key - default: - return other{tp: MetaType(rawTp)}, nil - } - return nil, cerror.ErrUnknownMetaType.GenWithStackByArgs(rawTp) -} - -// decodeRow decodes a byte slice into datums with a existing row map. -func decodeRow(b []byte, recordID kv.Handle, tableInfo *model.TableInfo, tz *time.Location) (map[int64]types.Datum, error) { - if len(b) == 0 { - return map[int64]types.Datum{}, nil - } - handleColIDs, handleColFt, reqCols := tableInfo.GetRowColInfos() - var datums map[int64]types.Datum - var err error - if rowcodec.IsNewFormat(b) { - datums, err = decodeRowV2(b, reqCols, tz) - } else { - datums, err = decodeRowV1(b, tableInfo, tz) - } - if err != nil { - return nil, errors.Trace(err) - } - return tablecodec.DecodeHandleToDatumMap(recordID, handleColIDs, handleColFt, tz, datums) -} - -// decodeRowV1 decodes value data using old encoding format. -// Row layout: colID1, value1, colID2, value2, ..... -func decodeRowV1(b []byte, tableInfo *model.TableInfo, tz *time.Location) (map[int64]types.Datum, error) { - row := make(map[int64]types.Datum) - if len(b) == 1 && b[0] == codec.NilFlag { - b = b[1:] - } - var err error - var data []byte - for len(b) > 0 { - // Get col id. - data, b, err = codec.CutOne(b) - if err != nil { - return nil, cerror.WrapError(cerror.ErrCodecDecode, err) - } - _, cid, err := codec.DecodeOne(data) - if err != nil { - return nil, cerror.WrapError(cerror.ErrCodecDecode, err) - } - id := cid.GetInt64() - - // Get col value. - data, b, err = codec.CutOne(b) - if err != nil { - return nil, cerror.WrapError(cerror.ErrCodecDecode, err) - } - _, v, err := codec.DecodeOne(data) - if err != nil { - return nil, cerror.WrapError(cerror.ErrCodecDecode, err) - } - - // unflatten value - colInfo, exist := tableInfo.GetColumnInfo(id) - if !exist { - // can not find column info, ignore this column because the column should be in WRITE ONLY state - continue - } - fieldType := &colInfo.FieldType - datum, err := unflatten(v, fieldType, tz) - if err != nil { - return nil, cerror.WrapError(cerror.ErrCodecDecode, err) - } - row[id] = datum - } - return row, nil -} - -// decodeRowV2 decodes value data using new encoding format. -// Ref: https://github.com/pingcap/tidb/pull/12634 -// https://github.com/pingcap/tidb/blob/master/docs/design/2018-07-19-row-format.md -func decodeRowV2(data []byte, columns []rowcodec.ColInfo, tz *time.Location) (map[int64]types.Datum, error) { - decoder := rowcodec.NewDatumMapDecoder(columns, tz) - datums, err := decoder.DecodeToDatumMap(data, nil) - if err != nil { - return datums, cerror.WrapError(cerror.ErrDecodeRowToDatum, err) - } - return datums, nil -} - -// unflatten converts a raw datum to a column datum. -func unflatten(datum types.Datum, ft *types.FieldType, loc *time.Location) (types.Datum, error) { - if datum.IsNull() { - return datum, nil - } - switch ft.Tp { - case mysql.TypeFloat: - datum.SetFloat32(float32(datum.GetFloat64())) - return datum, nil - case mysql.TypeVarchar, mysql.TypeString, mysql.TypeVarString, mysql.TypeTinyBlob, - mysql.TypeMediumBlob, mysql.TypeBlob, mysql.TypeLongBlob: - datum.SetString(datum.GetString(), ft.Collate) - case mysql.TypeTiny, mysql.TypeShort, mysql.TypeYear, mysql.TypeInt24, - mysql.TypeLong, mysql.TypeLonglong, mysql.TypeDouble: - return datum, nil - case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeTimestamp: - t := types.NewTime(types.ZeroCoreTime, ft.Tp, int8(ft.Decimal)) - var err error - err = t.FromPackedUint(datum.GetUint64()) - if err != nil { - return datum, cerror.WrapError(cerror.ErrDatumUnflatten, err) - } - if ft.Tp == mysql.TypeTimestamp && !t.IsZero() { - err = t.ConvertTimeZone(time.UTC, loc) - if err != nil { - return datum, cerror.WrapError(cerror.ErrDatumUnflatten, err) - } - } - datum.SetUint64(0) - datum.SetMysqlTime(t) - return datum, nil - case mysql.TypeDuration: // duration should read fsp from column meta data - dur := types.Duration{Duration: time.Duration(datum.GetInt64()), Fsp: int8(ft.Decimal)} - datum.SetMysqlDuration(dur) - return datum, nil - case mysql.TypeEnum: - // ignore error deliberately, to read empty enum value. - enum, err := types.ParseEnumValue(ft.Elems, datum.GetUint64()) - if err != nil { - enum = types.Enum{} - } - datum.SetMysqlEnum(enum, ft.Collate) - return datum, nil - case mysql.TypeSet: - set, err := types.ParseSetValue(ft.Elems, datum.GetUint64()) - if err != nil { - return datum, cerror.WrapError(cerror.ErrDatumUnflatten, err) - } - datum.SetMysqlSet(set, ft.Collate) - return datum, nil - case mysql.TypeBit: - val := datum.GetUint64() - byteSize := (ft.Flen + 7) >> 3 - datum.SetUint64(0) - datum.SetMysqlBit(types.NewBinaryLiteralFromUint(val, byteSize)) - } - return datum, nil -} diff --git a/cdc/cdc/entry/codec_test.go b/cdc/cdc/entry/codec_test.go deleted file mode 100644 index e2a92b42..00000000 --- a/cdc/cdc/entry/codec_test.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package entry - -import ( - "testing" - - "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/tablecodec" - "github.com/pingcap/tidb/util/codec" - "github.com/stretchr/testify/require" -) - -func TestDecodeRecordKey(t *testing.T) { - t.Parallel() - recordPrefix := tablecodec.GenTableRecordPrefix(12345) - key := tablecodec.EncodeRecordKey(recordPrefix, kv.IntHandle(67890)) - key, tableID, err := decodeTableID(key) - require.Nil(t, err) - require.Equal(t, tableID, int64(12345)) - key, recordID, err := decodeRecordID(key) - require.Nil(t, err) - require.Equal(t, recordID, int64(67890)) - require.Equal(t, len(key), 0) -} - -func TestDecodeListData(t *testing.T) { - t.Parallel() - key := []byte("hello") - var index int64 = 3 - - meta, err := decodeMetaKey(buildMetaKey(key, index)) - require.Nil(t, err) - require.Equal(t, meta.getType(), ListData) - list := meta.(metaListData) - require.Equal(t, list.key, string(key)) - require.Equal(t, list.index, index) -} - -func buildMetaKey(key []byte, index int64) []byte { - ek := make([]byte, 0, len(metaPrefix)+len(key)+36) - ek = append(ek, metaPrefix...) - ek = codec.EncodeBytes(ek, key) - ek = codec.EncodeUint(ek, uint64(ListData)) - return codec.EncodeInt(ek, index) -} diff --git a/cdc/cdc/entry/metrics.go b/cdc/cdc/entry/metrics.go index cacd8308..bb1ca3bf 100644 --- a/cdc/cdc/entry/metrics.go +++ b/cdc/cdc/entry/metrics.go @@ -20,14 +20,14 @@ import ( var ( mounterInputChanSizeGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "mounter", Name: "input_chan_size", Help: "mounter input chan size", }, []string{"capture", "changefeed"}) mountDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "mounter", Name: "unmarshal_and_mount", Help: "Bucketed histogram of processing time (s) of unmarshal and mount in mounter.", @@ -35,7 +35,7 @@ var ( }, []string{"capture", "changefeed"}) totalRowsCountGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "mounter", Name: "total_rows_count", Help: "The total count of rows that are processed by mounter", diff --git a/cdc/cdc/entry/mounter.go b/cdc/cdc/entry/mounter.go deleted file mode 100644 index 211a58e4..00000000 --- a/cdc/cdc/entry/mounter.go +++ /dev/null @@ -1,556 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package entry - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "math" - "math/rand" - "time" - "unsafe" - - "github.com/pingcap/errors" - "github.com/pingcap/log" - "github.com/pingcap/tidb/kv" - timodel "github.com/pingcap/tidb/parser/model" - "github.com/pingcap/tidb/parser/mysql" - "github.com/pingcap/tidb/table" - "github.com/pingcap/tidb/tablecodec" - "github.com/pingcap/tidb/types" - "github.com/tikv/migration/cdc/cdc/model" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/util" - "go.uber.org/zap" - "golang.org/x/sync/errgroup" -) - -const ( - defaultOutputChanSize = 128000 -) - -type baseKVEntry struct { - StartTs uint64 - // Commit or resolved TS - CRTs uint64 - - PhysicalTableID int64 - RecordID kv.Handle - Delete bool -} - -type rowKVEntry struct { - baseKVEntry - Row map[int64]types.Datum - PreRow map[int64]types.Datum - - // In some cases, row data may exist but not contain any Datum, - // use this RowExist/PreRowExist variable to distinguish between row data that does not exist - // or row data that does not contain any Datum. - RowExist bool - PreRowExist bool -} - -// Mounter is used to parse SQL events from KV events -type Mounter interface { - Run(ctx context.Context) error - Input() chan<- *model.PolymorphicEvent -} - -type mounterImpl struct { - schemaStorage SchemaStorage - rawRowChangedChs []chan *model.PolymorphicEvent - tz *time.Location - workerNum int - enableOldValue bool -} - -// NewMounter creates a mounter -func NewMounter(schemaStorage SchemaStorage, workerNum int, enableOldValue bool) Mounter { - if workerNum <= 0 { - workerNum = defaultMounterWorkerNum - } - chs := make([]chan *model.PolymorphicEvent, workerNum) - for i := 0; i < workerNum; i++ { - chs[i] = make(chan *model.PolymorphicEvent, defaultOutputChanSize) - } - return &mounterImpl{ - schemaStorage: schemaStorage, - rawRowChangedChs: chs, - workerNum: workerNum, - enableOldValue: enableOldValue, - } -} - -const defaultMounterWorkerNum = 32 - -func (m *mounterImpl) Run(ctx context.Context) error { - m.tz = util.TimezoneFromCtx(ctx) - errg, ctx := errgroup.WithContext(ctx) - errg.Go(func() error { - m.collectMetrics(ctx) - return nil - }) - for i := 0; i < m.workerNum; i++ { - index := i - errg.Go(func() error { - return m.codecWorker(ctx, index) - }) - } - return errg.Wait() -} - -func (m *mounterImpl) codecWorker(ctx context.Context, index int) error { - captureAddr := util.CaptureAddrFromCtx(ctx) - changefeedID := util.ChangefeedIDFromCtx(ctx) - metricMountDuration := mountDuration.WithLabelValues(captureAddr, changefeedID) - metricTotalRows := totalRowsCountGauge.WithLabelValues(captureAddr, changefeedID) - defer func() { - mountDuration.DeleteLabelValues(captureAddr, changefeedID) - totalRowsCountGauge.DeleteLabelValues(captureAddr, changefeedID) - }() - - for { - var pEvent *model.PolymorphicEvent - select { - case <-ctx.Done(): - return errors.Trace(ctx.Err()) - case pEvent = <-m.rawRowChangedChs[index]: - } - if pEvent.RawKV.OpType == model.OpTypeResolved { - pEvent.PrepareFinished() - continue - } - startTime := time.Now() - rowEvent, err := m.unmarshalAndMountRowChanged(ctx, pEvent.RawKV) - if err != nil { - return errors.Trace(err) - } - pEvent.Row = rowEvent - pEvent.RawKV.Value = nil - pEvent.RawKV.OldValue = nil - pEvent.PrepareFinished() - metricMountDuration.Observe(time.Since(startTime).Seconds()) - metricTotalRows.Inc() - } -} - -func (m *mounterImpl) Input() chan<- *model.PolymorphicEvent { - return m.rawRowChangedChs[rand.Intn(m.workerNum)] -} - -func (m *mounterImpl) collectMetrics(ctx context.Context) { - captureAddr := util.CaptureAddrFromCtx(ctx) - changefeedID := util.ChangefeedIDFromCtx(ctx) - metricMounterInputChanSize := mounterInputChanSizeGauge.WithLabelValues(captureAddr, changefeedID) - - for { - select { - case <-ctx.Done(): - return - case <-time.After(time.Second * 15): - chSize := 0 - for _, ch := range m.rawRowChangedChs { - chSize += len(ch) - } - metricMounterInputChanSize.Set(float64(chSize)) - } - } -} - -func (m *mounterImpl) unmarshalAndMountRowChanged(ctx context.Context, raw *model.RawKVEntry) (*model.RowChangedEvent, error) { - if !bytes.HasPrefix(raw.Key, tablePrefix) { - return nil, nil - } - key, physicalTableID, err := decodeTableID(raw.Key) - if err != nil { - return nil, err - } - baseInfo := baseKVEntry{ - StartTs: raw.StartTs, - CRTs: raw.CRTs, - PhysicalTableID: physicalTableID, - Delete: raw.OpType == model.OpTypeDelete, - } - // when async commit is enabled, the commitTs of DMLs may be equals with DDL finishedTs - // a DML whose commitTs is equal to a DDL finishedTs using the schema info before the DDL - snap, err := m.schemaStorage.GetSnapshot(ctx, raw.CRTs-1) - if err != nil { - return nil, errors.Trace(err) - } - row, err := func() (*model.RowChangedEvent, error) { - if snap.IsIneligibleTableID(physicalTableID) { - log.Debug("skip the DML of ineligible table", zap.Uint64("ts", raw.CRTs), zap.Int64("tableID", physicalTableID)) - return nil, nil - } - tableInfo, exist := snap.PhysicalTableByID(physicalTableID) - if !exist { - if snap.IsTruncateTableID(physicalTableID) { - log.Debug("skip the DML of truncated table", zap.Uint64("ts", raw.CRTs), zap.Int64("tableID", physicalTableID)) - return nil, nil - } - return nil, cerror.ErrSnapshotTableNotFound.GenWithStackByArgs(physicalTableID) - } - if bytes.HasPrefix(key, recordPrefix) { - rowKV, err := m.unmarshalRowKVEntry(tableInfo, raw.Key, raw.Value, raw.OldValue, baseInfo) - if err != nil { - return nil, errors.Trace(err) - } - if rowKV == nil { - return nil, nil - } - return m.mountRowKVEntry(tableInfo, rowKV, raw.ApproximateDataSize()) - } - return nil, nil - }() - if err != nil { - log.Error("failed to mount and unmarshals entry, start to print debug info", zap.Error(err)) - snap.PrintStatus(log.Error) - } - return row, err -} - -func (m *mounterImpl) unmarshalRowKVEntry(tableInfo *model.TableInfo, rawKey []byte, rawValue []byte, rawOldValue []byte, base baseKVEntry) (*rowKVEntry, error) { - recordID, err := tablecodec.DecodeRowKey(rawKey) - if err != nil { - return nil, errors.Trace(err) - } - decodeRow := func(rawColValue []byte) (map[int64]types.Datum, bool, error) { - if len(rawColValue) == 0 { - return nil, false, nil - } - row, err := decodeRow(rawColValue, recordID, tableInfo, m.tz) - if err != nil { - return nil, false, errors.Trace(err) - } - return row, true, nil - } - - row, rowExist, err := decodeRow(rawValue) - if err != nil { - return nil, errors.Trace(err) - } - preRow, preRowExist, err := decodeRow(rawOldValue) - if err != nil { - return nil, errors.Trace(err) - } - - if base.Delete && !m.enableOldValue && (tableInfo.PKIsHandle || tableInfo.IsCommonHandle) { - handleColIDs, fieldTps, _ := tableInfo.GetRowColInfos() - preRow, err = tablecodec.DecodeHandleToDatumMap(recordID, handleColIDs, fieldTps, m.tz, nil) - if err != nil { - return nil, errors.Trace(err) - } - preRowExist = true - } - - base.RecordID = recordID - return &rowKVEntry{ - baseKVEntry: base, - Row: row, - PreRow: preRow, - RowExist: rowExist, - PreRowExist: preRowExist, - }, nil -} - -const ( - ddlJobListKey = "DDLJobList" - ddlAddIndexJobListKey = "DDLJobAddIdxList" -) - -// UnmarshalDDL unmarshals the ddl job from RawKVEntry -func UnmarshalDDL(raw *model.RawKVEntry) (*timodel.Job, error) { - if raw.OpType != model.OpTypePut || !bytes.HasPrefix(raw.Key, metaPrefix) { - return nil, nil - } - meta, err := decodeMetaKey(raw.Key) - if err != nil { - return nil, errors.Trace(err) - } - if meta.getType() != ListData { - return nil, nil - } - k := meta.(metaListData) - if k.key != ddlJobListKey && k.key != ddlAddIndexJobListKey { - return nil, nil - } - job := &timodel.Job{} - err = json.Unmarshal(raw.Value, job) - if err != nil { - return nil, errors.Trace(err) - } - log.Debug("get new DDL job", zap.String("detail", job.String())) - if !job.IsDone() && !job.IsSynced() { - return nil, nil - } - // FinishedTS is only set when the job is synced, - // but we can use the entry's ts here - job.StartTS = raw.StartTs - job.BinlogInfo.FinishedTS = raw.CRTs - return job, nil -} - -func datum2Column(tableInfo *model.TableInfo, datums map[int64]types.Datum, fillWithDefaultValue bool) ([]*model.Column, error) { - cols := make([]*model.Column, len(tableInfo.RowColumnsOffset)) - for _, colInfo := range tableInfo.Columns { - colSize := 0 - if !model.IsColCDCVisible(colInfo) { - continue - } - colName := colInfo.Name.O - colDatums, exist := datums[colInfo.ID] - var colValue interface{} - if !exist && !fillWithDefaultValue { - continue - } - var err error - var warn string - var size int - if exist { - colValue, size, warn, err = formatColVal(colDatums, colInfo.Tp) - } else if fillWithDefaultValue { - colValue, size, warn, err = getDefaultOrZeroValue(colInfo) - } - if err != nil { - return nil, errors.Trace(err) - } - if warn != "" { - log.Warn(warn, zap.String("table", tableInfo.TableName.String()), zap.String("column", colInfo.Name.String())) - } - colSize += size - cols[tableInfo.RowColumnsOffset[colInfo.ID]] = &model.Column{ - Name: colName, - Type: colInfo.Tp, - Value: colValue, - Flag: tableInfo.ColumnsFlag[colInfo.ID], - // ApproximateBytes = column data size + column struct size - ApproximateBytes: colSize + sizeOfEmptyColumn, - } - } - return cols, nil -} - -func (m *mounterImpl) mountRowKVEntry(tableInfo *model.TableInfo, row *rowKVEntry, dataSize int64) (*model.RowChangedEvent, error) { - var err error - // Decode previous columns. - var preCols []*model.Column - // Since we now always use old value internally, - // we need to control the output(sink will use the PreColumns field to determine whether to output old value). - // Normally old value is output when only enableOldValue is on, - // but for the Delete event, when the old value feature is off, - // the HandleKey column needs to be included as well. So we need to do the following filtering. - if row.PreRowExist { - // FIXME(leoppro): using pre table info to mounter pre column datum - // the pre column and current column in one event may using different table info - preCols, err = datum2Column(tableInfo, row.PreRow, m.enableOldValue) - if err != nil { - return nil, errors.Trace(err) - } - - // NOTICE: When the old Value feature is off, - // the Delete event only needs to keep the handle key column. - if row.Delete && !m.enableOldValue { - for i := range preCols { - col := preCols[i] - if col != nil && !col.Flag.IsHandleKey() { - preCols[i] = nil - } - } - } - } - - var cols []*model.Column - if row.RowExist { - cols, err = datum2Column(tableInfo, row.Row, true) - if err != nil { - return nil, errors.Trace(err) - } - } - - schemaName := tableInfo.TableName.Schema - tableName := tableInfo.TableName.Table - var intRowID int64 - if row.RecordID.IsInt() { - intRowID = row.RecordID.IntValue() - } - - var tableInfoVersion uint64 - // Align with the old format if old value disabled. - if row.Delete && !m.enableOldValue { - tableInfoVersion = 0 - } else { - tableInfoVersion = tableInfo.TableInfoVersion - } - - return &model.RowChangedEvent{ - StartTs: row.StartTs, - CommitTs: row.CRTs, - RowID: intRowID, - TableInfoVersion: tableInfoVersion, - Table: &model.TableName{ - Schema: schemaName, - Table: tableName, - TableID: row.PhysicalTableID, - IsPartition: tableInfo.GetPartitionInfo() != nil, - }, - Columns: cols, - PreColumns: preCols, - IndexColumns: tableInfo.IndexColumnsOffset, - ApproximateDataSize: dataSize, - }, nil -} - -var emptyBytes = make([]byte, 0) - -const ( - sizeOfEmptyColumn = int(unsafe.Sizeof(model.Column{})) - sizeOfEmptyBytes = int(unsafe.Sizeof(emptyBytes)) - sizeOfEmptyString = int(unsafe.Sizeof("")) -) - -func sizeOfDatum(d types.Datum) int { - array := [...]types.Datum{d} - return int(types.EstimatedMemUsage(array[:], 1)) -} - -func sizeOfString(s string) int { - // string data size + string struct size. - return len(s) + sizeOfEmptyString -} - -func sizeOfBytes(b []byte) int { - // bytes data size + bytes struct size. - return len(b) + sizeOfEmptyBytes -} - -// formatColVal return interface{} need to meet the same requirement as getDefaultOrZeroValue -func formatColVal(datum types.Datum, tp byte) ( - value interface{}, size int, warn string, err error, -) { - if datum.IsNull() { - return nil, 0, "", nil - } - switch tp { - case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeNewDate, mysql.TypeTimestamp: - v := datum.GetMysqlTime().String() - return v, sizeOfString(v), "", nil - case mysql.TypeDuration: - v := datum.GetMysqlDuration().String() - return v, sizeOfString(v), "", nil - case mysql.TypeJSON: - v := datum.GetMysqlJSON().String() - return v, sizeOfString(v), "", nil - case mysql.TypeNewDecimal: - d := datum.GetMysqlDecimal() - if d == nil { - // nil takes 0 byte. - return nil, 0, "", nil - } - v := d.String() - return v, sizeOfString(v), "", nil - case mysql.TypeEnum: - v := datum.GetMysqlEnum().Value - const sizeOfV = unsafe.Sizeof(v) - return v, int(sizeOfV), "", nil - case mysql.TypeSet: - v := datum.GetMysqlSet().Value - const sizeOfV = unsafe.Sizeof(v) - return v, int(sizeOfV), "", nil - case mysql.TypeBit: - // Encode bits as integers to avoid pingcap/tidb#10988 (which also affects MySQL itself) - v, err := datum.GetBinaryLiteral().ToInt(nil) - const sizeOfV = unsafe.Sizeof(v) - return v, int(sizeOfV), "", err - case mysql.TypeString, mysql.TypeVarString, mysql.TypeVarchar, - mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, mysql.TypeBlob: - b := datum.GetBytes() - if b == nil { - b = emptyBytes - } - return b, sizeOfBytes(b), "", nil - case mysql.TypeFloat, mysql.TypeDouble: - v := datum.GetFloat64() - if math.IsNaN(v) || math.IsInf(v, 1) || math.IsInf(v, -1) { - warn = fmt.Sprintf("the value is invalid in column: %f", v) - v = 0 - } - const sizeOfV = unsafe.Sizeof(v) - return v, int(sizeOfV), warn, nil - default: - // NOTICE: GetValue() may return some types that go sql not support, which will cause sink DML fail - // Make specified convert upper if you need - // Go sql support type ref to: https://github.com/golang/go/blob/go1.17.4/src/database/sql/driver/types.go#L236 - return datum.GetValue(), sizeOfDatum(datum), "", nil - } -} - -// Scenarios when call this function: -// (1) column define default null at creating + insert without explicit column -// (2) alter table add column default xxx + old existing data -// (3) amend + insert without explicit column + alter table add column default xxx -// (4) online DDL drop column + data insert at state delete-only -// -// getDefaultOrZeroValue return interface{} need to meet to require type in -// https://github.com/golang/go/blob/go1.17.4/src/database/sql/driver/types.go#L236 -// Supported type is: nil, basic type(Int, Int8,..., Float32, Float64, String), Slice(uint8), other types not support -// TODO: Check default expr support -func getDefaultOrZeroValue(col *timodel.ColumnInfo) (interface{}, int, string, error) { - var d types.Datum - // NOTICE: SHOULD use OriginDefaultValue here, more info pls ref to - // https://github.com/tikv/migration/cdc/issues/4048 - // FIXME: Too many corner cases may hit here, like type truncate, timezone - // (1) If this column is uk(no pk), will cause data inconsistency in Scenarios(2) - // (2) If not fix here, will cause data inconsistency in Scenarios(3) directly - // Ref: https://github.com/pingcap/tidb/blob/d2c352980a43bb593db81fd1db996f47af596d91/table/column.go#L489 - if col.GetOriginDefaultValue() != nil { - d = types.NewDatum(col.GetOriginDefaultValue()) - return d.GetValue(), sizeOfDatum(d), "", nil - } - - if !mysql.HasNotNullFlag(col.Flag) { - // NOTICE: NotNullCheck need do after OriginDefaultValue check, as when TiDB meet "amend + add column default xxx", - // ref: https://github.com/pingcap/ticdc/issues/3929 - // must use null if TiDB not write the column value when default value is null - // and the value is null, see https://github.com/pingcap/tidb/issues/9304 - d = types.NewDatum(nil) - } else { - switch col.Tp { - case mysql.TypeEnum: - // For enum type, if no default value and not null is set, - // the default value is the first element of the enum list - d = types.NewDatum(col.FieldType.Elems[0]) - case mysql.TypeString, mysql.TypeVarString, mysql.TypeVarchar: - return emptyBytes, sizeOfEmptyBytes, "", nil - default: - d = table.GetZeroValue(col) - if d.IsNull() { - log.Error("meet unsupported column type", zap.String("column info", col.String())) - } - } - } - - return formatColVal(d, col.Tp) -} - -// DecodeTableID decodes the raw key to a table ID -func DecodeTableID(key []byte) (model.TableID, error) { - _, physicalTableID, err := decodeTableID(key) - if err != nil { - return 0, errors.Trace(err) - } - return physicalTableID, nil -} diff --git a/cdc/cdc/entry/mounter_test.go b/cdc/cdc/entry/mounter_test.go deleted file mode 100644 index b6308374..00000000 --- a/cdc/cdc/entry/mounter_test.go +++ /dev/null @@ -1,939 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package entry - -import ( - "context" - "strings" - "testing" - "time" - - "github.com/pingcap/log" - ticonfig "github.com/pingcap/tidb/config" - tidbkv "github.com/pingcap/tidb/kv" - timodel "github.com/pingcap/tidb/parser/model" - "github.com/pingcap/tidb/parser/mysql" - "github.com/pingcap/tidb/session" - "github.com/pingcap/tidb/store/mockstore" - "github.com/pingcap/tidb/testkit" - "github.com/pingcap/tidb/types" - "github.com/stretchr/testify/require" - "github.com/tikv/client-go/v2/oracle" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/regionspan" - "go.uber.org/zap" -) - -func TestMounterDisableOldValue(t *testing.T) { - testCases := []struct { - tableName string - createTableDDL string - // [] for rows, []infterface{} for columns. - values [][]interface{} - // [] for table partition if there is any, - // []int for approximateBytes of rows. - putApproximateBytes [][]int - delApproximateBytes [][]int - }{{ - tableName: "simple", - createTableDDL: "create table simple(id int primary key)", - values: [][]interface{}{{1}, {2}, {3}, {4}, {5}}, - putApproximateBytes: [][]int{{346, 346, 346, 346, 346}}, - delApproximateBytes: [][]int{{346, 346, 346, 346, 346}}, - }, { - tableName: "no_pk", - createTableDDL: "create table no_pk(id int not null unique key)", - values: [][]interface{}{{1}, {2}, {3}, {4}, {5}}, - putApproximateBytes: [][]int{{345, 345, 345, 345, 345}}, - delApproximateBytes: [][]int{{217, 217, 217, 217, 217}}, - }, { - tableName: "many_index", - createTableDDL: "create table many_index(id int not null unique key, c1 int unique key, c2 int, INDEX (c2))", - values: [][]interface{}{{1, 1, 1}, {2, 2, 2}, {3, 3, 3}, {4, 4, 4}, {5, 5, 5}}, - putApproximateBytes: [][]int{{638, 638, 638, 638, 638}}, - delApproximateBytes: [][]int{{254, 254, 254, 254, 254}}, - }, { - tableName: "default_value", - createTableDDL: "create table default_value(id int primary key, c1 int, c2 int not null default 5, c3 varchar(20), c4 varchar(20) not null default '666')", - values: [][]interface{}{{1}, {2}, {3}, {4}, {5}}, - putApproximateBytes: [][]int{{676, 676, 676, 676, 676}}, - delApproximateBytes: [][]int{{353, 353, 353, 353, 353}}, - }, { - tableName: "partition_table", - createTableDDL: `CREATE TABLE partition_table ( - id INT NOT NULL AUTO_INCREMENT UNIQUE KEY, - fname VARCHAR(25) NOT NULL, - lname VARCHAR(25) NOT NULL, - store_id INT NOT NULL, - department_id INT NOT NULL, - INDEX (department_id) - ) - - PARTITION BY RANGE(id) ( - PARTITION p0 VALUES LESS THAN (5), - PARTITION p1 VALUES LESS THAN (10), - PARTITION p2 VALUES LESS THAN (15), - PARTITION p3 VALUES LESS THAN (20) - )`, - values: [][]interface{}{ - {1, "aa", "bb", 12, 12}, - {6, "aac", "bab", 51, 51}, - {11, "aad", "bsb", 71, 61}, - {18, "aae", "bbf", 21, 14}, - {15, "afa", "bbc", 11, 12}, - }, - putApproximateBytes: [][]int{{775}, {777}, {777}, {777, 777}}, - delApproximateBytes: [][]int{{227}, {227}, {227}, {227, 227}}, - }, { - tableName: "tp_int", - createTableDDL: `create table tp_int - ( - id int auto_increment, - c_tinyint tinyint null, - c_smallint smallint null, - c_mediumint mediumint null, - c_int int null, - c_bigint bigint null, - constraint pk - primary key (id) - );`, - values: [][]interface{}{ - {1, 1, 2, 3, 4, 5}, - {2}, - {3, 3, 4, 5, 6, 7}, - {4, 127, 32767, 8388607, 2147483647, 9223372036854775807}, - {5, -128, -32768, -8388608, -2147483648, -9223372036854775808}, - }, - putApproximateBytes: [][]int{{986, 626, 986, 986, 986}}, - delApproximateBytes: [][]int{{346, 346, 346, 346, 346}}, - }, { - tableName: "tp_text", - createTableDDL: `create table tp_text - ( - id int auto_increment, - c_tinytext tinytext null, - c_text text null, - c_mediumtext mediumtext null, - c_longtext longtext null, - c_varchar varchar(16) null, - c_char char(16) null, - c_tinyblob tinyblob null, - c_blob blob null, - c_mediumblob mediumblob null, - c_longblob longblob null, - c_binary binary(16) null, - c_varbinary varbinary(16) null, - constraint pk - primary key (id) - );`, - values: [][]interface{}{ - {1}, - { - 2, "89504E470D0A1A0A", "89504E470D0A1A0A", "89504E470D0A1A0A", "89504E470D0A1A0A", "89504E470D0A1A0A", - "89504E470D0A1A0A", - []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}, - []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}, - []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}, - []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}, - []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}, - []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}, - }, - { - 3, "bug free", "bug free", "bug free", "bug free", "bug free", "bug free", "bug free", "bug free", - "bug free", "bug free", "bug free", "bug free", - }, - {4, "", "", "", "", "", "", "", "", "", "", "", ""}, - {5, "你好", "我好", "大家好", "道路", "千万条", "安全", "第一条", "行车", "不规范", "亲人", "两行泪", "！"}, - {6, "😀", "😃", "😄", "😁", "😆", "😅", "😂", "🤣", "☺️", "😊", "😇", "🙂"}, - }, - putApproximateBytes: [][]int{{1019, 1459, 1411, 1323, 1398, 1369}}, - delApproximateBytes: [][]int{{347, 347, 347, 347, 347, 347}}, - }, { - tableName: "tp_time", - createTableDDL: `create table tp_time - ( - id int auto_increment, - c_date date null, - c_datetime datetime null, - c_timestamp timestamp null, - c_time time null, - c_year year null, - constraint pk - primary key (id) - );`, - values: [][]interface{}{ - {1}, - {2, "2020-02-20", "2020-02-20 02:20:20", "2020-02-20 02:20:20", "02:20:20", "2020"}, - }, - putApproximateBytes: [][]int{{627, 819}}, - delApproximateBytes: [][]int{{347, 347}}, - }, { - tableName: "tp_real", - createTableDDL: `create table tp_real - ( - id int auto_increment, - c_float float null, - c_double double null, - c_decimal decimal null, - constraint pk - primary key (id) - );`, - values: [][]interface{}{ - {1}, - {2, "2020.0202", "2020.0303", "2020.0404"}, - }, - putApproximateBytes: [][]int{{563, 551}}, - delApproximateBytes: [][]int{{347, 347}}, - }, { - tableName: "tp_other", - createTableDDL: `create table tp_other - ( - id int auto_increment, - c_enum enum ('a','b','c') null, - c_set set ('a','b','c') null, - c_bit bit(64) null, - c_json json null, - constraint pk - primary key (id) - );`, - values: [][]interface{}{ - {1}, - {2, "a", "a,c", 888, `{"aa":"bb"}`}, - }, - putApproximateBytes: [][]int{{636, 624}}, - delApproximateBytes: [][]int{{348, 348}}, - }, { - tableName: "clustered_index1", - createTableDDL: "CREATE TABLE clustered_index1 (id VARCHAR(255) PRIMARY KEY, data INT);", - values: [][]interface{}{ - {"hhh"}, - {"你好😘", 666}, - {"世界🤪", 888}, - }, - putApproximateBytes: [][]int{{383, 446, 446}}, - delApproximateBytes: [][]int{{311, 318, 318}}, - }, { - tableName: "clustered_index2", - createTableDDL: "CREATE TABLE clustered_index2 (id VARCHAR(255), data INT, ddaa date, PRIMARY KEY (id, data, ddaa), UNIQUE KEY (id, data, ddaa));", - values: [][]interface{}{ - {"你好😘", 666, "2020-11-20"}, - {"世界🤪", 888, "2020-05-12"}, - }, - putApproximateBytes: [][]int{{592, 592}}, - delApproximateBytes: [][]int{{592, 592}}, - }} - for _, tc := range testCases { - testMounterDisableOldValue(t, tc) - } -} - -func testMounterDisableOldValue(t *testing.T, tc struct { - tableName string - createTableDDL string - values [][]interface{} - putApproximateBytes [][]int - delApproximateBytes [][]int -}) { - store, err := mockstore.NewMockStore() - require.Nil(t, err) - defer store.Close() //nolint:errcheck - ticonfig.UpdateGlobal(func(conf *ticonfig.Config) { - // we can update the tidb config here - }) - session.SetSchemaLease(0) - session.DisableStats4Test() - domain, err := session.BootstrapSession(store) - require.Nil(t, err) - defer domain.Close() - domain.SetStatsUpdating(true) - tk := testkit.NewTestKit(t, store) - tk.MustExec("set @@tidb_enable_clustered_index=1;") - tk.MustExec("use test;") - - tk.MustExec(tc.createTableDDL) - - jobs, err := getAllHistoryDDLJob(store) - require.Nil(t, err) - scheamStorage, err := NewSchemaStorage(nil, 0, nil, false) - require.Nil(t, err) - for _, job := range jobs { - err := scheamStorage.HandleDDLJob(job) - require.Nil(t, err) - } - tableInfo, ok := scheamStorage.GetLastSnapshot().GetTableByName("test", tc.tableName) - require.True(t, ok) - if tableInfo.IsCommonHandle { - // we can check this log to make sure if the clustered-index is enabled - log.Info("this table is enable the clustered index", zap.String("tableName", tableInfo.Name.L)) - } - - for _, params := range tc.values { - insertSQL := prepareInsertSQL(t, tableInfo, len(params)) - tk.MustExec(insertSQL, params...) - } - - ver, err := store.CurrentVersion(oracle.GlobalTxnScope) - require.Nil(t, err) - scheamStorage.AdvanceResolvedTs(ver.Ver) - mounter := NewMounter(scheamStorage, 1, false).(*mounterImpl) - mounter.tz = time.Local - ctx := context.Background() - - // [TODO] check size and readd rowBytes - mountAndCheckRowInTable := func(tableID int64, _ []int, f func(key []byte, value []byte) *model.RawKVEntry) int { - var rows int - walkTableSpanInStore(t, store, tableID, func(key []byte, value []byte) { - rawKV := f(key, value) - row, err := mounter.unmarshalAndMountRowChanged(ctx, rawKV) - require.Nil(t, err) - if row == nil { - return - } - rows++ - require.Equal(t, row.Table.Table, tc.tableName) - require.Equal(t, row.Table.Schema, "test") - // [TODO] check size and reopen this check - // require.Equal(t, rowBytes[rows-1], row.ApproximateBytes(), row) - t.Log("ApproximateBytes", tc.tableName, rows-1, row.ApproximateBytes()) - // TODO: test column flag, column type and index columns - if len(row.Columns) != 0 { - checkSQL, params := prepareCheckSQL(t, tc.tableName, row.Columns) - result := tk.MustQuery(checkSQL, params...) - result.Check([][]interface{}{{"1"}}) - } - if len(row.PreColumns) != 0 { - checkSQL, params := prepareCheckSQL(t, tc.tableName, row.PreColumns) - result := tk.MustQuery(checkSQL, params...) - result.Check([][]interface{}{{"1"}}) - } - }) - return rows - } - - mountAndCheckRow := func(rowsBytes [][]int, f func(key []byte, value []byte) *model.RawKVEntry) int { - partitionInfo := tableInfo.GetPartitionInfo() - if partitionInfo == nil { - return mountAndCheckRowInTable(tableInfo.ID, rowsBytes[0], f) - } - var rows int - for i, p := range partitionInfo.Definitions { - rows += mountAndCheckRowInTable(p.ID, rowsBytes[i], f) - } - return rows - } - - rows := mountAndCheckRow(tc.putApproximateBytes, func(key []byte, value []byte) *model.RawKVEntry { - return &model.RawKVEntry{ - OpType: model.OpTypePut, - Key: key, - Value: value, - StartTs: ver.Ver - 1, - CRTs: ver.Ver, - } - }) - require.Equal(t, rows, len(tc.values)) - - rows = mountAndCheckRow(tc.delApproximateBytes, func(key []byte, value []byte) *model.RawKVEntry { - return &model.RawKVEntry{ - OpType: model.OpTypeDelete, - Key: key, - Value: nil, // delete event doesn't include a value when old-value is disabled - StartTs: ver.Ver - 1, - CRTs: ver.Ver, - } - }) - require.Equal(t, rows, len(tc.values)) -} - -func prepareInsertSQL(t *testing.T, tableInfo *model.TableInfo, columnLens int) string { - var sb strings.Builder - _, err := sb.WriteString("INSERT INTO " + tableInfo.Name.O + "(") - require.Nil(t, err) - for i := 0; i < columnLens; i++ { - col := tableInfo.Columns[i] - if i != 0 { - _, err = sb.WriteString(", ") - require.Nil(t, err) - } - _, err = sb.WriteString(col.Name.O) - require.Nil(t, err) - } - _, err = sb.WriteString(") VALUES (") - require.Nil(t, err) - for i := 0; i < columnLens; i++ { - if i != 0 { - _, err = sb.WriteString(", ") - require.Nil(t, err) - } - _, err = sb.WriteString("?") - require.Nil(t, err) - } - _, err = sb.WriteString(")") - require.Nil(t, err) - return sb.String() -} - -func prepareCheckSQL(t *testing.T, tableName string, cols []*model.Column) (string, []interface{}) { - var sb strings.Builder - _, err := sb.WriteString("SELECT count(1) FROM " + tableName + " WHERE ") - require.Nil(t, err) - params := make([]interface{}, 0, len(cols)) - for i, col := range cols { - if col == nil { - continue - } - if i != 0 { - _, err = sb.WriteString(" AND ") - require.Nil(t, err) - } - if col.Value == nil { - _, err = sb.WriteString(col.Name + " IS NULL") - require.Nil(t, err) - continue - } - params = append(params, col.Value) - if col.Type == mysql.TypeJSON { - _, err = sb.WriteString(col.Name + " = CAST(? AS JSON)") - } else { - _, err = sb.WriteString(col.Name + " = ?") - } - require.Nil(t, err) - } - return sb.String(), params -} - -func walkTableSpanInStore(t *testing.T, store tidbkv.Storage, tableID int64, f func(key []byte, value []byte)) { - txn, err := store.Begin() - require.Nil(t, err) - defer txn.Rollback() //nolint:errcheck - tableSpan := regionspan.GetTableSpan(tableID) - kvIter, err := txn.Iter(tableSpan.Start, tableSpan.End) - require.Nil(t, err) - defer kvIter.Close() - for kvIter.Valid() { - f(kvIter.Key(), kvIter.Value()) - err = kvIter.Next() - require.Nil(t, err) - } -} - -// Check following MySQL type, ref to: -// https://github.com/pingcap/tidb/blob/master/parser/mysql/type.go -type columnInfoAndResult struct { - ColInfo timodel.ColumnInfo - Res interface{} -} - -// We use OriginDefaultValue instead of DefaultValue in the ut, pls ref to -// https://github.com/tikv/migration/cdc/issues/4048 -// FIXME: OriginDefaultValue seems always to be string, and test more corner case -// Ref: https://github.com/pingcap/tidb/blob/d2c352980a43bb593db81fd1db996f47af596d91/table/column.go#L489 -func TestGetDefaultZeroValue(t *testing.T) { - colAndRess := []columnInfoAndResult{ - // mysql flag null - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Flag: uint(0), - }, - }, - Res: nil, - }, - // mysql.TypeTiny + notnull + nodefault - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeTiny, - Flag: mysql.NotNullFlag, - }, - }, - Res: int64(0), - }, - // mysql.TypeTiny + notnull + default - { - ColInfo: timodel.ColumnInfo{ - OriginDefaultValue: -1314, - FieldType: types.FieldType{ - Tp: mysql.TypeTiny, - Flag: mysql.NotNullFlag, - }, - }, - Res: int64(-1314), - }, - // mysql.TypeTiny + notnull + default + unsigned - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeTiny, - Flag: mysql.NotNullFlag | mysql.UnsignedFlag, - }, - }, - Res: uint64(0), - }, - // mysql.TypeTiny + notnull + unsigned - { - ColInfo: timodel.ColumnInfo{ - OriginDefaultValue: uint64(1314), - FieldType: types.FieldType{ - Tp: mysql.TypeTiny, - Flag: mysql.NotNullFlag | mysql.UnsignedFlag, - }, - }, - Res: uint64(1314), - }, - // mysql.TypeTiny + null + default - { - ColInfo: timodel.ColumnInfo{ - OriginDefaultValue: -1314, - FieldType: types.FieldType{ - Tp: mysql.TypeTiny, - Flag: uint(0), - }, - }, - Res: int64(-1314), - }, - // mysql.TypeTiny + null + nodefault - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeTiny, - Flag: uint(0), - }, - }, - Res: nil, - }, - // mysql.TypeShort, others testCases same as tiny - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeShort, - Flag: mysql.NotNullFlag, - }, - }, - Res: int64(0), - }, - // mysql.TypeLong, others testCases same as tiny - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeLong, - Flag: mysql.NotNullFlag, - }, - }, - Res: int64(0), - }, - // mysql.TypeLonglong, others testCases same as tiny - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeLonglong, - Flag: mysql.NotNullFlag, - }, - }, - Res: int64(0), - }, - // mysql.TypeInt24, others testCases same as tiny - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeInt24, - Flag: mysql.NotNullFlag, - }, - }, - Res: int64(0), - }, - // mysql.TypeFloat + notnull + nodefault - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeFloat, - Flag: mysql.NotNullFlag, - }, - }, - Res: float64(0), - }, - // mysql.TypeFloat + notnull + default - { - ColInfo: timodel.ColumnInfo{ - OriginDefaultValue: -3.1415, - FieldType: types.FieldType{ - Tp: mysql.TypeFloat, - Flag: mysql.NotNullFlag, - }, - }, - Res: float64(-3.1415), - }, - // mysql.TypeFloat + notnull + default + unsigned - { - ColInfo: timodel.ColumnInfo{ - OriginDefaultValue: 3.1415, - FieldType: types.FieldType{ - Tp: mysql.TypeFloat, - Flag: mysql.NotNullFlag | mysql.UnsignedFlag, - }, - }, - Res: float64(3.1415), - }, - // mysql.TypeFloat + notnull + unsigned - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeFloat, - Flag: mysql.NotNullFlag | mysql.UnsignedFlag, - }, - }, - Res: float64(0), - }, - // mysql.TypeFloat + null + default - { - ColInfo: timodel.ColumnInfo{ - OriginDefaultValue: -3.1415, - FieldType: types.FieldType{ - Tp: mysql.TypeFloat, - Flag: uint(0), - }, - }, - Res: float64(-3.1415), - }, - // mysql.TypeFloat + null + nodefault - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeFloat, - Flag: uint(0), - }, - }, - Res: nil, - }, - // mysql.TypeDouble, other testCases same as float - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeDouble, - Flag: mysql.NotNullFlag, - }, - }, - Res: float64(0), - }, - // mysql.TypeNewDecimal + notnull + nodefault - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeNewDecimal, - Flag: mysql.NotNullFlag, - Flen: 5, - Decimal: 2, - }, - }, - Res: "0", // related with Flen and Decimal - }, - // mysql.TypeNewDecimal + null + nodefault - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeNewDecimal, - Flag: uint(0), - Flen: 5, - Decimal: 2, - }, - }, - Res: nil, - }, - // mysql.TypeNewDecimal + null + default - { - ColInfo: timodel.ColumnInfo{ - OriginDefaultValue: "-3.14", // no float - FieldType: types.FieldType{ - Tp: mysql.TypeNewDecimal, - Flag: uint(0), - Flen: 5, - Decimal: 2, - }, - }, - Res: "-3.14", - }, - // mysql.TypeNull - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeNull, - }, - }, - Res: nil, - }, - // mysql.TypeTimestamp + notnull + nodefault - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeTimestamp, - Flag: mysql.NotNullFlag, - }, - }, - Res: "0000-00-00 00:00:00", - }, - // mysql.TypeTimestamp + notnull + default - { - ColInfo: timodel.ColumnInfo{ - OriginDefaultValue: "2020-11-19 12:12:12", - FieldType: types.FieldType{ - Tp: mysql.TypeTimestamp, - Flag: mysql.NotNullFlag, - }, - }, - Res: "2020-11-19 12:12:12", - }, - // mysql.TypeTimestamp + null + default - { - ColInfo: timodel.ColumnInfo{ - OriginDefaultValue: "2020-11-19 12:12:12", - FieldType: types.FieldType{ - Tp: mysql.TypeTimestamp, - Flag: mysql.NotNullFlag, - }, - }, - Res: "2020-11-19 12:12:12", - }, - // mysql.TypeDate, other testCases same as TypeTimestamp - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeDate, - Flag: mysql.NotNullFlag, - }, - }, - Res: "0000-00-00", - }, - // mysql.TypeDuration, other testCases same as TypeTimestamp - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeDuration, - Flag: mysql.NotNullFlag, - }, - }, - Res: "00:00:00", - }, - // mysql.TypeDatetime, other testCases same as TypeTimestamp - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeDatetime, - Flag: mysql.NotNullFlag, - }, - }, - Res: "0000-00-00 00:00:00", - }, - // mysql.TypeYear + notnull + nodefault - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeYear, - Flag: mysql.NotNullFlag, - }, - }, - Res: int64(0), - }, - // mysql.TypeYear + notnull + default - { - ColInfo: timodel.ColumnInfo{ - OriginDefaultValue: "2021", - FieldType: types.FieldType{ - Tp: mysql.TypeYear, - Flag: mysql.NotNullFlag, - }, - }, - // TypeYear default value will be a string and then translate to []byte - Res: "2021", - }, - // mysql.TypeNewDate - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeNewDate, - Flag: mysql.NotNullFlag, - }, - }, - Res: nil, // [TODO] seems not support by TiDB, need check - }, - // mysql.TypeVarchar + notnull + nodefault - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeVarchar, - Flag: mysql.NotNullFlag, - }, - }, - Res: []byte{}, - }, - // mysql.TypeVarchar + notnull + default - { - ColInfo: timodel.ColumnInfo{ - OriginDefaultValue: "e0", - FieldType: types.FieldType{ - Tp: mysql.TypeVarchar, - Flag: mysql.NotNullFlag, - }, - }, - // TypeVarchar default value will be a string and then translate to []byte - Res: "e0", - }, - // mysql.TypeTinyBlob - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeTinyBlob, - Flag: mysql.NotNullFlag, - }, - }, - Res: []byte{}, - }, - // mysql.TypeMediumBlob - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeMediumBlob, - Flag: mysql.NotNullFlag, - }, - }, - Res: []byte{}, - }, - // mysql.TypeLongBlob - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeLongBlob, - Flag: mysql.NotNullFlag, - }, - }, - Res: []byte{}, - }, - // mysql.TypeBlob - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeBlob, - Flag: mysql.NotNullFlag, - }, - }, - Res: []byte{}, - }, - // mysql.TypeVarString - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeVarString, - Flag: mysql.NotNullFlag, - }, - }, - Res: []byte{}, - }, - // mysql.TypeString - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeString, - Flag: mysql.NotNullFlag, - }, - }, - Res: []byte{}, - }, - // mysql.TypeBit - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Flag: mysql.NotNullFlag, - Tp: mysql.TypeBit, - }, - }, - Res: uint64(0), - }, - // BLOB, TEXT, GEOMETRY or JSON column can't have a default value - // mysql.TypeJSON - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeJSON, - Flag: mysql.NotNullFlag, - }, - }, - Res: "null", - }, - // mysql.TypeEnum + notnull + nodefault - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeEnum, - Flag: mysql.NotNullFlag, - Elems: []string{"e0", "e1"}, - }, - }, - // TypeEnum value will be a string and then translate to []byte - // NotNull && no default will choose first element - Res: uint64(0), - }, - // mysql.TypeEnum + notnull + default - { - ColInfo: timodel.ColumnInfo{ - OriginDefaultValue: "e1", - FieldType: types.FieldType{ - Tp: mysql.TypeEnum, - Flag: mysql.NotNullFlag, - Elems: []string{"e0", "e1"}, - }, - }, - // TypeEnum default value will be a string and then translate to []byte - Res: "e1", - }, - // mysql.TypeSet + notnull - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeSet, - Flag: mysql.NotNullFlag, - }, - }, - Res: uint64(0), - }, - // mysql.TypeSet + notnull + default - { - ColInfo: timodel.ColumnInfo{ - OriginDefaultValue: "1,e", - FieldType: types.FieldType{ - Tp: mysql.TypeSet, - Flag: mysql.NotNullFlag, - }, - }, - // TypeSet default value will be a string and then translate to []byte - Res: "1,e", - }, - // mysql.TypeGeometry - { - ColInfo: timodel.ColumnInfo{ - FieldType: types.FieldType{ - Tp: mysql.TypeGeometry, - Flag: mysql.NotNullFlag, - }, - }, - Res: nil, // not support yet - }, - } - testGetDefaultZeroValue(t, colAndRess) -} - -func testGetDefaultZeroValue(t *testing.T, colAndRess []columnInfoAndResult) { - for _, colAndRes := range colAndRess { - val, _, _, _ := getDefaultOrZeroValue(&colAndRes.ColInfo) - require.Equal(t, colAndRes.Res, val) - } -} diff --git a/cdc/cdc/entry/schema_storage.go b/cdc/cdc/entry/schema_storage.go deleted file mode 100644 index 9b27258b..00000000 --- a/cdc/cdc/entry/schema_storage.go +++ /dev/null @@ -1,887 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package entry - -import ( - "context" - "sort" - "sync" - "sync/atomic" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/log" - timeta "github.com/pingcap/tidb/meta" - timodel "github.com/pingcap/tidb/parser/model" - "github.com/tikv/migration/cdc/cdc/model" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/filter" - "github.com/tikv/migration/cdc/pkg/retry" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -// schemaSnapshot stores the source TiDB all schema information -// schemaSnapshot is a READ ONLY struct -type schemaSnapshot struct { - tableNameToID map[model.TableName]int64 - schemaNameToID map[string]int64 - - schemas map[int64]*timodel.DBInfo - tables map[int64]*model.TableInfo - partitionTable map[int64]*model.TableInfo - - // key is schemaID and value is tableIDs - tableInSchema map[int64][]int64 - - truncateTableID map[int64]struct{} - ineligibleTableID map[int64]struct{} - - currentTs uint64 - - // if explicit is true, treat tables without explicit row id as eligible - explicitTables bool -} - -// SingleSchemaSnapshot is a single schema snapshot independent of schema storage -type SingleSchemaSnapshot = schemaSnapshot - -// HandleDDL handles the ddl job -func (s *SingleSchemaSnapshot) HandleDDL(job *timodel.Job) error { - return s.handleDDL(job) -} - -// PreTableInfo returns the table info which will be overwritten by the specified job -func (s *SingleSchemaSnapshot) PreTableInfo(job *timodel.Job) (*model.TableInfo, error) { - switch job.Type { - case timodel.ActionCreateSchema, timodel.ActionModifySchemaCharsetAndCollate, timodel.ActionDropSchema: - return nil, nil - case timodel.ActionCreateTable, timodel.ActionCreateView, timodel.ActionRecoverTable: - // no pre table info - return nil, nil - case timodel.ActionRenameTable, timodel.ActionDropTable, timodel.ActionDropView, timodel.ActionTruncateTable: - // get the table will be dropped - table, ok := s.TableByID(job.TableID) - if !ok { - return nil, cerror.ErrSchemaStorageTableMiss.GenWithStackByArgs(job.TableID) - } - return table, nil - case timodel.ActionRenameTables: - // DDL on multiple tables, ignore pre table info - return nil, nil - default: - binlogInfo := job.BinlogInfo - if binlogInfo == nil { - log.Warn("ignore a invalid DDL job", zap.Reflect("job", job)) - return nil, nil - } - tbInfo := binlogInfo.TableInfo - if tbInfo == nil { - log.Warn("ignore a invalid DDL job", zap.Reflect("job", job)) - return nil, nil - } - tableID := tbInfo.ID - table, ok := s.TableByID(tableID) - if !ok { - return nil, cerror.ErrSchemaStorageTableMiss.GenWithStackByArgs(job.TableID) - } - return table, nil - } -} - -// NewSingleSchemaSnapshotFromMeta creates a new single schema snapshot from a tidb meta -func NewSingleSchemaSnapshotFromMeta(meta *timeta.Meta, currentTs uint64, explicitTables bool) (*SingleSchemaSnapshot, error) { - // meta is nil only in unit tests - if meta == nil { - snap := newEmptySchemaSnapshot(explicitTables) - snap.currentTs = currentTs - return snap, nil - } - return newSchemaSnapshotFromMeta(meta, currentTs, explicitTables) -} - -func newEmptySchemaSnapshot(explicitTables bool) *schemaSnapshot { - return &schemaSnapshot{ - tableNameToID: make(map[model.TableName]int64), - schemaNameToID: make(map[string]int64), - - schemas: make(map[int64]*timodel.DBInfo), - tables: make(map[int64]*model.TableInfo), - partitionTable: make(map[int64]*model.TableInfo), - - tableInSchema: make(map[int64][]int64), - truncateTableID: make(map[int64]struct{}), - ineligibleTableID: make(map[int64]struct{}), - - explicitTables: explicitTables, - } -} - -func newSchemaSnapshotFromMeta(meta *timeta.Meta, currentTs uint64, explicitTables bool) (*schemaSnapshot, error) { - snap := newEmptySchemaSnapshot(explicitTables) - dbinfos, err := meta.ListDatabases() - if err != nil { - return nil, cerror.WrapError(cerror.ErrMetaListDatabases, err) - } - for _, dbinfo := range dbinfos { - snap.schemas[dbinfo.ID] = dbinfo - snap.schemaNameToID[dbinfo.Name.O] = dbinfo.ID - } - for schemaID, dbinfo := range snap.schemas { - tableInfos, err := meta.ListTables(schemaID) - if err != nil { - return nil, cerror.WrapError(cerror.ErrMetaListDatabases, err) - } - snap.tableInSchema[schemaID] = make([]int64, 0, len(tableInfos)) - for _, tableInfo := range tableInfos { - snap.tableInSchema[schemaID] = append(snap.tableInSchema[schemaID], tableInfo.ID) - tableInfo := model.WrapTableInfo(dbinfo.ID, dbinfo.Name.O, currentTs, tableInfo) - snap.tables[tableInfo.ID] = tableInfo - snap.tableNameToID[model.TableName{Schema: dbinfo.Name.O, Table: tableInfo.Name.O}] = tableInfo.ID - isEligible := tableInfo.IsEligible(explicitTables) - if !isEligible { - snap.ineligibleTableID[tableInfo.ID] = struct{}{} - } - if pi := tableInfo.GetPartitionInfo(); pi != nil { - for _, partition := range pi.Definitions { - snap.partitionTable[partition.ID] = tableInfo - if !isEligible { - snap.ineligibleTableID[partition.ID] = struct{}{} - } - } - } - } - } - snap.currentTs = currentTs - return snap, nil -} - -func (s *schemaSnapshot) PrintStatus(logger func(msg string, fields ...zap.Field)) { - logger("[SchemaSnap] Start to print status", zap.Uint64("currentTs", s.currentTs)) - for id, dbInfo := range s.schemas { - logger("[SchemaSnap] --> Schemas", zap.Int64("schemaID", id), zap.Reflect("dbInfo", dbInfo)) - // check schemaNameToID - if schemaID, exist := s.schemaNameToID[dbInfo.Name.O]; !exist || schemaID != id { - logger("[SchemaSnap] ----> schemaNameToID item lost", zap.String("name", dbInfo.Name.O), zap.Int64("schemaNameToID", s.schemaNameToID[dbInfo.Name.O])) - } - } - if len(s.schemaNameToID) != len(s.schemas) { - logger("[SchemaSnap] schemaNameToID length mismatch schemas") - for schemaName, schemaID := range s.schemaNameToID { - logger("[SchemaSnap] --> schemaNameToID", zap.String("schemaName", schemaName), zap.Int64("schemaID", schemaID)) - } - } - for id, tableInfo := range s.tables { - logger("[SchemaSnap] --> Tables", zap.Int64("tableID", id), zap.Stringer("tableInfo", tableInfo)) - // check tableNameToID - if tableID, exist := s.tableNameToID[tableInfo.TableName]; !exist || tableID != id { - logger("[SchemaSnap] ----> tableNameToID item lost", zap.Stringer("name", tableInfo.TableName), zap.Int64("tableNameToID", s.tableNameToID[tableInfo.TableName])) - } - } - if len(s.tableNameToID) != len(s.tables) { - logger("[SchemaSnap] tableNameToID length mismatch tables") - for tableName, tableID := range s.tableNameToID { - logger("[SchemaSnap] --> tableNameToID", zap.Stringer("tableName", tableName), zap.Int64("tableID", tableID)) - } - } - for pid, table := range s.partitionTable { - logger("[SchemaSnap] --> Partitions", zap.Int64("partitionID", pid), zap.Int64("tableID", table.ID)) - } - truncateTableID := make([]int64, 0, len(s.truncateTableID)) - for id := range s.truncateTableID { - truncateTableID = append(truncateTableID, id) - } - logger("[SchemaSnap] TruncateTableIDs", zap.Int64s("ids", truncateTableID)) - - ineligibleTableID := make([]int64, 0, len(s.ineligibleTableID)) - for id := range s.ineligibleTableID { - ineligibleTableID = append(ineligibleTableID, id) - } - logger("[SchemaSnap] IneligibleTableIDs", zap.Int64s("ids", ineligibleTableID)) -} - -// Clone clones Storage -func (s *schemaSnapshot) Clone() *schemaSnapshot { - clone := *s - - tableNameToID := make(map[model.TableName]int64, len(s.tableNameToID)) - for k, v := range s.tableNameToID { - tableNameToID[k] = v - } - clone.tableNameToID = tableNameToID - - schemaNameToID := make(map[string]int64, len(s.schemaNameToID)) - for k, v := range s.schemaNameToID { - schemaNameToID[k] = v - } - clone.schemaNameToID = schemaNameToID - - schemas := make(map[int64]*timodel.DBInfo, len(s.schemas)) - for k, v := range s.schemas { - // DBInfo is readonly in TiCDC, shallow copy to reduce memory - schemas[k] = v.Copy() - } - clone.schemas = schemas - - tables := make(map[int64]*model.TableInfo, len(s.tables)) - for k, v := range s.tables { - tables[k] = v - } - clone.tables = tables - - tableInSchema := make(map[int64][]int64, len(s.tableInSchema)) - for k, v := range s.tableInSchema { - cloneV := make([]int64, len(v)) - copy(cloneV, v) - tableInSchema[k] = cloneV - } - clone.tableInSchema = tableInSchema - - partitionTable := make(map[int64]*model.TableInfo, len(s.partitionTable)) - for k, v := range s.partitionTable { - partitionTable[k] = v - } - clone.partitionTable = partitionTable - - truncateTableID := make(map[int64]struct{}, len(s.truncateTableID)) - for k, v := range s.truncateTableID { - truncateTableID[k] = v - } - clone.truncateTableID = truncateTableID - - ineligibleTableID := make(map[int64]struct{}, len(s.ineligibleTableID)) - for k, v := range s.ineligibleTableID { - ineligibleTableID[k] = v - } - clone.ineligibleTableID = ineligibleTableID - - return &clone -} - -// GetTableNameByID looks up a TableName with the given table id -func (s *schemaSnapshot) GetTableNameByID(id int64) (model.TableName, bool) { - tableInfo, ok := s.tables[id] - if !ok { - // Try partition, it could be a partition table. - partInfo, ok := s.partitionTable[id] - if !ok { - return model.TableName{}, false - } - // Must exists an table that contains the partition. - tableInfo = s.tables[partInfo.ID] - } - return tableInfo.TableName, true -} - -// GetTableIDByName returns the tableID by table schemaName and tableName -func (s *schemaSnapshot) GetTableIDByName(schemaName string, tableName string) (int64, bool) { - id, ok := s.tableNameToID[model.TableName{ - Schema: schemaName, - Table: tableName, - }] - return id, ok -} - -// GetTableByName queries a table by name, -// the second returned value is false if no table with the specified name is found. -func (s *schemaSnapshot) GetTableByName(schema, table string) (info *model.TableInfo, ok bool) { - id, ok := s.GetTableIDByName(schema, table) - if !ok { - return nil, ok - } - return s.TableByID(id) -} - -// SchemaByID returns the DBInfo by schema id -func (s *schemaSnapshot) SchemaByID(id int64) (val *timodel.DBInfo, ok bool) { - val, ok = s.schemas[id] - return -} - -// SchemaByTableID returns the schema ID by table ID -func (s *schemaSnapshot) SchemaByTableID(tableID int64) (*timodel.DBInfo, bool) { - tableInfo, ok := s.tables[tableID] - if !ok { - return nil, false - } - schemaID, ok := s.schemaNameToID[tableInfo.TableName.Schema] - if !ok { - return nil, false - } - return s.SchemaByID(schemaID) -} - -// TableByID returns the TableInfo by table id -func (s *schemaSnapshot) TableByID(id int64) (val *model.TableInfo, ok bool) { - val, ok = s.tables[id] - return -} - -// PhysicalTableByID returns the TableInfo by table id or partition ID. -func (s *schemaSnapshot) PhysicalTableByID(id int64) (val *model.TableInfo, ok bool) { - val, ok = s.tables[id] - if !ok { - val, ok = s.partitionTable[id] - } - return -} - -// IsTruncateTableID returns true if the table id have been truncated by truncate table DDL -func (s *schemaSnapshot) IsTruncateTableID(id int64) bool { - _, ok := s.truncateTableID[id] - return ok -} - -// IsIneligibleTableID returns true if the table is ineligible -func (s *schemaSnapshot) IsIneligibleTableID(id int64) bool { - _, ok := s.ineligibleTableID[id] - return ok -} - -// FillSchemaName fills the schema name in ddl job -func (s *schemaSnapshot) FillSchemaName(job *timodel.Job) error { - if job.Type == timodel.ActionRenameTables { - // DDLs on multiple schema or tables, ignore them. - return nil - } - if job.Type == timodel.ActionCreateSchema || - job.Type == timodel.ActionDropSchema { - job.SchemaName = job.BinlogInfo.DBInfo.Name.O - return nil - } - dbInfo, exist := s.SchemaByID(job.SchemaID) - if !exist { - return cerror.ErrSnapshotSchemaNotFound.GenWithStackByArgs(job.SchemaID) - } - job.SchemaName = dbInfo.Name.O - return nil -} - -func (s *schemaSnapshot) dropSchema(id int64) error { - schema, ok := s.schemas[id] - if !ok { - return cerror.ErrSnapshotSchemaNotFound.GenWithStackByArgs(id) - } - - for _, tableID := range s.tableInSchema[id] { - tableName := s.tables[tableID].TableName - if pi := s.tables[tableID].GetPartitionInfo(); pi != nil { - for _, partition := range pi.Definitions { - delete(s.partitionTable, partition.ID) - } - } - delete(s.tables, tableID) - delete(s.tableNameToID, tableName) - } - - delete(s.schemas, id) - delete(s.tableInSchema, id) - delete(s.schemaNameToID, schema.Name.O) - - return nil -} - -func (s *schemaSnapshot) createSchema(db *timodel.DBInfo) error { - if _, ok := s.schemas[db.ID]; ok { - return cerror.ErrSnapshotSchemaExists.GenWithStackByArgs(db.Name, db.ID) - } - - s.schemas[db.ID] = db.Copy() - s.schemaNameToID[db.Name.O] = db.ID - s.tableInSchema[db.ID] = []int64{} - - log.Debug("create schema success, schema id", zap.String("name", db.Name.O), zap.Int64("id", db.ID)) - return nil -} - -func (s *schemaSnapshot) replaceSchema(db *timodel.DBInfo) error { - _, ok := s.schemas[db.ID] - if !ok { - return cerror.ErrSnapshotSchemaNotFound.GenWithStack("schema %s(%d) not found", db.Name, db.ID) - } - s.schemas[db.ID] = db.Copy() - s.schemaNameToID[db.Name.O] = db.ID - return nil -} - -func (s *schemaSnapshot) dropTable(id int64) error { - table, ok := s.tables[id] - if !ok { - return cerror.ErrSnapshotTableNotFound.GenWithStackByArgs(id) - } - tableInSchema, ok := s.tableInSchema[table.SchemaID] - if !ok { - return cerror.ErrSnapshotSchemaNotFound.GenWithStack("table(%d)'s schema", id) - } - - for i, tableID := range tableInSchema { - if tableID == id { - copy(tableInSchema[i:], tableInSchema[i+1:]) - s.tableInSchema[table.SchemaID] = tableInSchema[:len(tableInSchema)-1] - break - } - } - - tableName := s.tables[id].TableName - delete(s.tables, id) - if pi := table.GetPartitionInfo(); pi != nil { - for _, partition := range pi.Definitions { - delete(s.partitionTable, partition.ID) - delete(s.ineligibleTableID, partition.ID) - } - } - delete(s.tableNameToID, tableName) - delete(s.ineligibleTableID, id) - - log.Debug("drop table success", zap.String("name", table.Name.O), zap.Int64("id", id)) - return nil -} - -func (s *schemaSnapshot) updatePartition(tbl *model.TableInfo) error { - id := tbl.ID - table, ok := s.tables[id] - if !ok { - return cerror.ErrSnapshotTableNotFound.GenWithStackByArgs(id) - } - oldPi := table.GetPartitionInfo() - if oldPi == nil { - return cerror.ErrSnapshotTableNotFound.GenWithStack("table %d is not a partition table", id) - } - oldIDs := make(map[int64]struct{}, len(oldPi.Definitions)) - for _, p := range oldPi.Definitions { - oldIDs[p.ID] = struct{}{} - } - - newPi := tbl.GetPartitionInfo() - if newPi == nil { - return cerror.ErrSnapshotTableNotFound.GenWithStack("table %d is not a partition table", id) - } - s.tables[id] = tbl - for _, partition := range newPi.Definitions { - // update table info. - if _, ok := s.partitionTable[partition.ID]; ok { - log.Debug("add table partition success", - zap.String("name", tbl.Name.O), zap.Int64("tid", id), - zap.Int64("add partition id", partition.ID)) - } - s.partitionTable[partition.ID] = tbl - if !tbl.IsEligible(s.explicitTables) { - s.ineligibleTableID[partition.ID] = struct{}{} - } - delete(oldIDs, partition.ID) - } - - // drop old partition. - for pid := range oldIDs { - s.truncateTableID[pid] = struct{}{} - delete(s.partitionTable, pid) - delete(s.ineligibleTableID, pid) - log.Debug("drop table partition success", - zap.String("name", tbl.Name.O), zap.Int64("tid", id), - zap.Int64("truncated partition id", pid)) - } - - return nil -} - -func (s *schemaSnapshot) createTable(table *model.TableInfo) error { - schema, ok := s.schemas[table.SchemaID] - if !ok { - return cerror.ErrSnapshotSchemaNotFound.GenWithStack("table's schema(%d)", table.SchemaID) - } - tableInSchema, ok := s.tableInSchema[table.SchemaID] - if !ok { - return cerror.ErrSnapshotSchemaNotFound.GenWithStack("table's schema(%d)", table.SchemaID) - } - _, ok = s.tables[table.ID] - if ok { - return cerror.ErrSnapshotTableExists.GenWithStackByArgs(schema.Name, table.Name) - } - tableInSchema = append(tableInSchema, table.ID) - s.tableInSchema[table.SchemaID] = tableInSchema - - s.tables[table.ID] = table - if !table.IsEligible(s.explicitTables) { - log.Warn("this table is not eligible to replicate", zap.String("tableName", table.Name.O), zap.Int64("tableID", table.ID)) - s.ineligibleTableID[table.ID] = struct{}{} - } - if pi := table.GetPartitionInfo(); pi != nil { - for _, partition := range pi.Definitions { - s.partitionTable[partition.ID] = table - if !table.IsEligible(s.explicitTables) { - s.ineligibleTableID[partition.ID] = struct{}{} - } - } - } - s.tableNameToID[table.TableName] = table.ID - - log.Debug("create table success", zap.String("name", schema.Name.O+"."+table.Name.O), zap.Int64("id", table.ID)) - return nil -} - -// ReplaceTable replace the table by new tableInfo -func (s *schemaSnapshot) replaceTable(table *model.TableInfo) error { - _, ok := s.tables[table.ID] - if !ok { - return cerror.ErrSnapshotTableNotFound.GenWithStack("table %s(%d)", table.Name, table.ID) - } - s.tables[table.ID] = table - if !table.IsEligible(s.explicitTables) { - log.Warn("this table is not eligible to replicate", zap.String("tableName", table.Name.O), zap.Int64("tableID", table.ID)) - s.ineligibleTableID[table.ID] = struct{}{} - } - if pi := table.GetPartitionInfo(); pi != nil { - for _, partition := range pi.Definitions { - s.partitionTable[partition.ID] = table - if !table.IsEligible(s.explicitTables) { - s.ineligibleTableID[partition.ID] = struct{}{} - } - } - } - - return nil -} - -func (s *schemaSnapshot) handleDDL(job *timodel.Job) error { - if err := s.FillSchemaName(job); err != nil { - return errors.Trace(err) - } - log.Info("handle DDL", zap.String("DDL", job.Query), zap.Stringer("job", job)) - getWrapTableInfo := func(job *timodel.Job) *model.TableInfo { - return model.WrapTableInfo(job.SchemaID, job.SchemaName, - job.BinlogInfo.FinishedTS, - job.BinlogInfo.TableInfo) - } - switch job.Type { - case timodel.ActionCreateSchema: - // get the DBInfo from job rawArgs - err := s.createSchema(job.BinlogInfo.DBInfo) - if err != nil { - return errors.Trace(err) - } - case timodel.ActionModifySchemaCharsetAndCollate: - err := s.replaceSchema(job.BinlogInfo.DBInfo) - if err != nil { - return errors.Trace(err) - } - case timodel.ActionDropSchema: - err := s.dropSchema(job.SchemaID) - if err != nil { - return errors.Trace(err) - } - case timodel.ActionRenameTable: - // first drop the table - err := s.dropTable(job.TableID) - if err != nil { - return errors.Trace(err) - } - // create table - err = s.createTable(getWrapTableInfo(job)) - if err != nil { - return errors.Trace(err) - } - case timodel.ActionRenameTables: - return s.renameTables(job) - case timodel.ActionCreateTable, timodel.ActionCreateView, timodel.ActionRecoverTable: - err := s.createTable(getWrapTableInfo(job)) - if err != nil { - return errors.Trace(err) - } - case timodel.ActionDropTable, timodel.ActionDropView: - err := s.dropTable(job.TableID) - if err != nil { - return errors.Trace(err) - } - - case timodel.ActionTruncateTable: - // job.TableID is the old table id, different from table.ID - err := s.dropTable(job.TableID) - if err != nil { - return errors.Trace(err) - } - - err = s.createTable(getWrapTableInfo(job)) - if err != nil { - return errors.Trace(err) - } - - s.truncateTableID[job.TableID] = struct{}{} - case timodel.ActionTruncateTablePartition, timodel.ActionAddTablePartition, timodel.ActionDropTablePartition: - err := s.updatePartition(getWrapTableInfo(job)) - if err != nil { - return errors.Trace(err) - } - default: - binlogInfo := job.BinlogInfo - if binlogInfo == nil { - log.Warn("ignore a invalid DDL job", zap.Reflect("job", job)) - return nil - } - tbInfo := binlogInfo.TableInfo - if tbInfo == nil { - log.Warn("ignore a invalid DDL job", zap.Reflect("job", job)) - return nil - } - err := s.replaceTable(getWrapTableInfo(job)) - if err != nil { - return errors.Trace(err) - } - } - s.currentTs = job.BinlogInfo.FinishedTS - return nil -} - -func (s *schemaSnapshot) renameTables(job *timodel.Job) error { - var oldSchemaIDs, newSchemaIDs, oldTableIDs []int64 - var newTableNames, oldSchemaNames []*timodel.CIStr - err := job.DecodeArgs(&oldSchemaIDs, &newSchemaIDs, &newTableNames, &oldTableIDs, &oldSchemaNames) - if err != nil { - return errors.Trace(err) - } - if len(job.BinlogInfo.MultipleTableInfos) < len(newTableNames) { - return cerror.ErrInvalidDDLJob.GenWithStackByArgs(job.ID) - } - // NOTE: should handle failures in halfway better. - for _, tableID := range oldTableIDs { - if err := s.dropTable(tableID); err != nil { - return errors.Trace(err) - } - } - for i, tableInfo := range job.BinlogInfo.MultipleTableInfos { - newSchema, ok := s.SchemaByID(newSchemaIDs[i]) - if !ok { - return cerror.ErrSnapshotSchemaNotFound.GenWithStackByArgs(newSchemaIDs[i]) - } - newSchemaName := newSchema.Name.L - err = s.createTable(model.WrapTableInfo( - newSchemaIDs[i], newSchemaName, job.BinlogInfo.FinishedTS, tableInfo)) - if err != nil { - return errors.Trace(err) - } - } - return nil -} - -// CloneTables return a clone of the existing tables. -func (s *schemaSnapshot) CloneTables() map[model.TableID]model.TableName { - mp := make(map[model.TableID]model.TableName, len(s.tables)) - - for id, table := range s.tables { - mp[id] = table.TableName - } - - return mp -} - -// Tables return a map between table id and table info -// the returned map must be READ-ONLY. Any modified of this map will lead to the internal state confusion in schema storage -func (s *schemaSnapshot) Tables() map[model.TableID]*model.TableInfo { - return s.tables -} - -// SchemaStorage stores the schema information with multi-version -type SchemaStorage interface { - // GetSnapshot returns the snapshot which of ts is specified. - // It may block caller when ts is larger than ResolvedTs. - GetSnapshot(ctx context.Context, ts uint64) (*SingleSchemaSnapshot, error) - // GetLastSnapshot returns the last snapshot - GetLastSnapshot() *schemaSnapshot - // HandleDDLJob creates a new snapshot in storage and handles the ddl job - HandleDDLJob(job *timodel.Job) error - // AdvanceResolvedTs advances the resolved - AdvanceResolvedTs(ts uint64) - // ResolvedTs returns the resolved ts of the schema storage - ResolvedTs() uint64 - // DoGC removes snaps that are no longer needed at the specified TS. - // It returns the TS from which the oldest maintained snapshot is valid. - DoGC(ts uint64) (lastSchemaTs uint64) -} - -type schemaStorageImpl struct { - snaps []*schemaSnapshot - snapsMu sync.RWMutex - gcTs uint64 - resolvedTs uint64 - - filter *filter.Filter - explicitTables bool -} - -// NewSchemaStorage creates a new schema storage -func NewSchemaStorage(meta *timeta.Meta, startTs uint64, filter *filter.Filter, forceReplicate bool) (SchemaStorage, error) { - var snap *schemaSnapshot - var err error - if meta == nil { - snap = newEmptySchemaSnapshot(forceReplicate) - } else { - snap, err = newSchemaSnapshotFromMeta(meta, startTs, forceReplicate) - } - if err != nil { - return nil, errors.Trace(err) - } - schema := &schemaStorageImpl{ - snaps: []*schemaSnapshot{snap}, - resolvedTs: startTs, - filter: filter, - explicitTables: forceReplicate, - } - return schema, nil -} - -func (s *schemaStorageImpl) getSnapshot(ts uint64) (*schemaSnapshot, error) { - gcTs := atomic.LoadUint64(&s.gcTs) - if ts < gcTs { - // Unexpected error, caller should fail immediately. - return nil, cerror.ErrSchemaStorageGCed.GenWithStackByArgs(ts, gcTs) - } - resolvedTs := atomic.LoadUint64(&s.resolvedTs) - if ts > resolvedTs { - // Caller should retry. - return nil, cerror.ErrSchemaStorageUnresolved.GenWithStackByArgs(ts, resolvedTs) - } - s.snapsMu.RLock() - defer s.snapsMu.RUnlock() - i := sort.Search(len(s.snaps), func(i int) bool { - return s.snaps[i].currentTs > ts - }) - if i <= 0 { - // Unexpected error, caller should fail immediately. - return nil, cerror.ErrSchemaSnapshotNotFound.GenWithStackByArgs(ts) - } - return s.snaps[i-1], nil -} - -// GetSnapshot returns the snapshot which of ts is specified -func (s *schemaStorageImpl) GetSnapshot(ctx context.Context, ts uint64) (*schemaSnapshot, error) { - var snap *schemaSnapshot - - // The infinite retry here is a temporary solution to the `ErrSchemaStorageUnresolved` caused by - // DDL puller lagging too much. - startTime := time.Now() - logTime := startTime - err := retry.Do(ctx, func() error { - var err error - snap, err = s.getSnapshot(ts) - now := time.Now() - if now.Sub(logTime) >= 30*time.Second && isRetryable(err) { - log.Warn("GetSnapshot is taking too long, DDL puller stuck?", - zap.Uint64("ts", ts), zap.Duration("duration", now.Sub(startTime))) - logTime = now - } - return err - }, retry.WithBackoffBaseDelay(10), retry.WithInfiniteTries(), retry.WithIsRetryableErr(isRetryable)) - - return snap, err -} - -func isRetryable(err error) bool { - return cerror.IsRetryableError(err) && cerror.ErrSchemaStorageUnresolved.Equal(err) -} - -// GetLastSnapshot returns the last snapshot -func (s *schemaStorageImpl) GetLastSnapshot() *schemaSnapshot { - s.snapsMu.RLock() - defer s.snapsMu.RUnlock() - return s.snaps[len(s.snaps)-1] -} - -// HandleDDLJob creates a new snapshot in storage and handles the ddl job -func (s *schemaStorageImpl) HandleDDLJob(job *timodel.Job) error { - if s.skipJob(job) { - s.AdvanceResolvedTs(job.BinlogInfo.FinishedTS) - return nil - } - s.snapsMu.Lock() - defer s.snapsMu.Unlock() - var snap *schemaSnapshot - if len(s.snaps) > 0 { - lastSnap := s.snaps[len(s.snaps)-1] - if job.BinlogInfo.FinishedTS <= lastSnap.currentTs { - log.Info("ignore foregone DDL", - zap.Int64("jobID", job.ID), zap.String("DDL", job.Query)) - return nil - } - snap = lastSnap.Clone() - } else { - snap = newEmptySchemaSnapshot(s.explicitTables) - } - if err := snap.handleDDL(job); err != nil { - return errors.Trace(err) - } - s.snaps = append(s.snaps, snap) - s.AdvanceResolvedTs(job.BinlogInfo.FinishedTS) - return nil -} - -// AdvanceResolvedTs advances the resolved -func (s *schemaStorageImpl) AdvanceResolvedTs(ts uint64) { - var swapped bool - for !swapped { - oldResolvedTs := atomic.LoadUint64(&s.resolvedTs) - if ts < oldResolvedTs { - return - } - swapped = atomic.CompareAndSwapUint64(&s.resolvedTs, oldResolvedTs, ts) - } -} - -// ResolvedTs returns the resolved ts of the schema storage -func (s *schemaStorageImpl) ResolvedTs() uint64 { - return atomic.LoadUint64(&s.resolvedTs) -} - -// DoGC removes snaps which of ts less than this specified ts -func (s *schemaStorageImpl) DoGC(ts uint64) (lastSchemaTs uint64) { - s.snapsMu.Lock() - defer s.snapsMu.Unlock() - var startIdx int - for i, snap := range s.snaps { - if snap.currentTs > ts { - break - } - startIdx = i - } - if startIdx == 0 { - return s.snaps[0].currentTs - } - if log.GetLevel() == zapcore.DebugLevel { - log.Debug("Do GC in schema storage") - for i := 0; i < startIdx; i++ { - s.snaps[i].PrintStatus(log.Debug) - } - } - - // copy the part of the slice that is needed instead of re-slicing it - // to maximize efficiency of Go runtime GC. - newSnaps := make([]*schemaSnapshot, len(s.snaps)-startIdx) - copy(newSnaps, s.snaps[startIdx:]) - s.snaps = newSnaps - - lastSchemaTs = s.snaps[0].currentTs - atomic.StoreUint64(&s.gcTs, lastSchemaTs) - return -} - -// SkipJob skip the job should not be executed -// TiDB write DDL Binlog for every DDL Job, we must ignore jobs that are cancelled or rollback -// For older version TiDB, it write DDL Binlog in the txn that the state of job is changed to *synced* -// Now, it write DDL Binlog in the txn that the state of job is changed to *done* (before change to *synced*) -// At state *done*, it will be always and only changed to *synced*. -func (s *schemaStorageImpl) skipJob(job *timodel.Job) bool { - if s.filter != nil && s.filter.ShouldDiscardDDL(job.Type) { - log.Info("discard DDL", zap.Int64("jobID", job.ID), zap.String("DDL", job.Query)) - return true - } - return !job.IsSynced() && !job.IsDone() -} diff --git a/cdc/cdc/entry/schema_storage_test.go b/cdc/cdc/entry/schema_storage_test.go deleted file mode 100644 index dda7081b..00000000 --- a/cdc/cdc/entry/schema_storage_test.go +++ /dev/null @@ -1,1038 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package entry - -import ( - "context" - "encoding/json" - "fmt" - "sort" - "testing" - - "github.com/pingcap/errors" - ticonfig "github.com/pingcap/tidb/config" - "github.com/pingcap/tidb/domain" - tidbkv "github.com/pingcap/tidb/kv" - timeta "github.com/pingcap/tidb/meta" - timodel "github.com/pingcap/tidb/parser/model" - "github.com/pingcap/tidb/parser/mysql" - "github.com/pingcap/tidb/session" - "github.com/pingcap/tidb/sessionctx" - "github.com/pingcap/tidb/store/mockstore" - "github.com/pingcap/tidb/testkit" - "github.com/pingcap/tidb/types" - "github.com/stretchr/testify/require" - "github.com/tikv/client-go/v2/oracle" - "github.com/tikv/migration/cdc/cdc/kv" - "github.com/tikv/migration/cdc/cdc/model" -) - -func TestSchema(t *testing.T) { - dbName := timodel.NewCIStr("Test") - // db and ignoreDB info - dbInfo := &timodel.DBInfo{ - ID: 1, - Name: dbName, - State: timodel.StatePublic, - } - // `createSchema` job1 - job := &timodel.Job{ - ID: 3, - State: timodel.JobStateSynced, - SchemaID: 1, - Type: timodel.ActionCreateSchema, - BinlogInfo: &timodel.HistoryInfo{SchemaVersion: 1, DBInfo: dbInfo, FinishedTS: 123}, - Query: "create database test", - } - // reconstruct the local schema - snap := newEmptySchemaSnapshot(false) - err := snap.handleDDL(job) - require.Nil(t, err) - _, exist := snap.SchemaByID(job.SchemaID) - require.True(t, exist) - - // test drop schema - job = &timodel.Job{ - ID: 6, - State: timodel.JobStateSynced, - SchemaID: 1, - Type: timodel.ActionDropSchema, - BinlogInfo: &timodel.HistoryInfo{SchemaVersion: 3, DBInfo: dbInfo, FinishedTS: 124}, - Query: "drop database test", - } - err = snap.handleDDL(job) - require.Nil(t, err) - _, exist = snap.SchemaByID(job.SchemaID) - require.False(t, exist) - - job = &timodel.Job{ - ID: 3, - State: timodel.JobStateSynced, - SchemaID: 1, - Type: timodel.ActionCreateSchema, - BinlogInfo: &timodel.HistoryInfo{SchemaVersion: 2, DBInfo: dbInfo, FinishedTS: 124}, - Query: "create database test", - } - - err = snap.handleDDL(job) - require.Nil(t, err) - err = snap.handleDDL(job) - require.True(t, errors.IsAlreadyExists(err)) - - // test schema drop schema error - job = &timodel.Job{ - ID: 9, - State: timodel.JobStateSynced, - SchemaID: 1, - Type: timodel.ActionDropSchema, - BinlogInfo: &timodel.HistoryInfo{SchemaVersion: 1, DBInfo: dbInfo, FinishedTS: 123}, - Query: "drop database test", - } - err = snap.handleDDL(job) - require.Nil(t, err) - err = snap.handleDDL(job) - require.True(t, errors.IsNotFound(err)) -} - -func TestTable(t *testing.T) { - var jobs []*timodel.Job - dbName := timodel.NewCIStr("Test") - tbName := timodel.NewCIStr("T") - colName := timodel.NewCIStr("A") - idxName := timodel.NewCIStr("idx") - // column info - colInfo := &timodel.ColumnInfo{ - ID: 1, - Name: colName, - Offset: 0, - FieldType: *types.NewFieldType(mysql.TypeLonglong), - State: timodel.StatePublic, - } - // index info - idxInfo := &timodel.IndexInfo{ - Name: idxName, - Table: tbName, - Columns: []*timodel.IndexColumn{ - { - Name: colName, - Offset: 0, - Length: 10, - }, - }, - Unique: true, - Primary: true, - State: timodel.StatePublic, - } - // table info - tblInfo := &timodel.TableInfo{ - ID: 2, - Name: tbName, - State: timodel.StatePublic, - } - // db info - dbInfo := &timodel.DBInfo{ - ID: 3, - Name: dbName, - State: timodel.StatePublic, - } - - // `createSchema` job - job := &timodel.Job{ - ID: 5, - State: timodel.JobStateSynced, - SchemaID: 3, - Type: timodel.ActionCreateSchema, - BinlogInfo: &timodel.HistoryInfo{SchemaVersion: 1, DBInfo: dbInfo, FinishedTS: 123}, - Query: "create database " + dbName.O, - } - jobs = append(jobs, job) - - // `createTable` job - job = &timodel.Job{ - ID: 6, - State: timodel.JobStateSynced, - SchemaID: 3, - TableID: 2, - Type: timodel.ActionCreateTable, - BinlogInfo: &timodel.HistoryInfo{SchemaVersion: 2, TableInfo: tblInfo, FinishedTS: 124}, - Query: "create table " + tbName.O, - } - jobs = append(jobs, job) - - // `addColumn` job - tblInfo.Columns = []*timodel.ColumnInfo{colInfo} - job = &timodel.Job{ - ID: 7, - State: timodel.JobStateSynced, - SchemaID: 3, - TableID: 2, - Type: timodel.ActionAddColumn, - BinlogInfo: &timodel.HistoryInfo{SchemaVersion: 3, TableInfo: tblInfo, FinishedTS: 125}, - Query: "alter table " + tbName.O + " add column " + colName.O, - } - jobs = append(jobs, job) - - // construct a historical `addIndex` job - tblInfo = tblInfo.Clone() - tblInfo.Indices = []*timodel.IndexInfo{idxInfo} - job = &timodel.Job{ - ID: 8, - State: timodel.JobStateSynced, - SchemaID: 3, - TableID: 2, - Type: timodel.ActionAddIndex, - BinlogInfo: &timodel.HistoryInfo{SchemaVersion: 4, TableInfo: tblInfo, FinishedTS: 126}, - Query: fmt.Sprintf("alter table %s add index %s(%s)", tbName, idxName, colName), - } - jobs = append(jobs, job) - - // reconstruct the local schema - snap := newEmptySchemaSnapshot(false) - for _, job := range jobs { - err := snap.handleDDL(job) - require.Nil(t, err) - } - - // check the historical db that constructed above whether in the schema list of local schema - _, ok := snap.SchemaByID(dbInfo.ID) - require.True(t, ok) - // check the historical table that constructed above whether in the table list of local schema - table, ok := snap.TableByID(tblInfo.ID) - require.True(t, ok) - require.Len(t, table.Columns, 1) - require.Len(t, table.Indices, 1) - - // test ineligible tables - require.True(t, snap.IsIneligibleTableID(2)) - - // check truncate table - tblInfo1 := &timodel.TableInfo{ - ID: 9, - Name: tbName, - State: timodel.StatePublic, - } - job = &timodel.Job{ - ID: 9, - State: timodel.JobStateSynced, - SchemaID: 3, - TableID: 2, - Type: timodel.ActionTruncateTable, - BinlogInfo: &timodel.HistoryInfo{SchemaVersion: 5, TableInfo: tblInfo1, FinishedTS: 127}, - Query: "truncate table " + tbName.O, - } - preTableInfo, err := snap.PreTableInfo(job) - require.Nil(t, err) - require.Equal(t, preTableInfo.TableName, model.TableName{Schema: "Test", Table: "T"}) - require.Equal(t, preTableInfo.ID, int64(2)) - - err = snap.handleDDL(job) - require.Nil(t, err) - - _, ok = snap.TableByID(tblInfo1.ID) - require.True(t, ok) - - _, ok = snap.TableByID(2) - require.False(t, ok) - - // test ineligible tables - require.True(t, snap.IsIneligibleTableID(9)) - require.False(t, snap.IsIneligibleTableID(2)) - // check drop table - job = &timodel.Job{ - ID: 9, - State: timodel.JobStateSynced, - SchemaID: 3, - TableID: 9, - Type: timodel.ActionDropTable, - BinlogInfo: &timodel.HistoryInfo{SchemaVersion: 6, FinishedTS: 128}, - Query: "drop table " + tbName.O, - } - preTableInfo, err = snap.PreTableInfo(job) - require.Nil(t, err) - require.Equal(t, preTableInfo.TableName, model.TableName{Schema: "Test", Table: "T"}) - require.Equal(t, preTableInfo.ID, int64(9)) - - err = snap.handleDDL(job) - require.Nil(t, err) - - _, ok = snap.TableByID(tblInfo.ID) - require.False(t, ok) - - // test ineligible tables - require.False(t, snap.IsIneligibleTableID(9)) - - // drop schema - err = snap.dropSchema(3) - require.Nil(t, err) -} - -func TestHandleDDL(t *testing.T) { - snap := newEmptySchemaSnapshot(false) - dbName := timodel.NewCIStr("Test") - colName := timodel.NewCIStr("A") - tbName := timodel.NewCIStr("T") - newTbName := timodel.NewCIStr("RT") - - // db info - dbInfo := &timodel.DBInfo{ - ID: 2, - Name: dbName, - State: timodel.StatePublic, - } - // table Info - tblInfo := &timodel.TableInfo{ - ID: 6, - Name: tbName, - State: timodel.StatePublic, - } - // column info - colInfo := &timodel.ColumnInfo{ - ID: 8, - Name: colName, - Offset: 0, - FieldType: *types.NewFieldType(mysql.TypeLonglong), - State: timodel.StatePublic, - } - tblInfo.Columns = []*timodel.ColumnInfo{colInfo} - - testCases := []struct { - name string - jobID int64 - schemaID int64 - tableID int64 - jobType timodel.ActionType - binlogInfo *timodel.HistoryInfo - query string - resultQuery string - schemaName string - tableName string - }{ - {name: "createSchema", jobID: 3, schemaID: 2, tableID: 0, jobType: timodel.ActionCreateSchema, binlogInfo: &timodel.HistoryInfo{SchemaVersion: 1, DBInfo: dbInfo, TableInfo: nil, FinishedTS: 123}, query: "create database Test", resultQuery: "create database Test", schemaName: dbInfo.Name.O, tableName: ""}, - {name: "updateSchema", jobID: 4, schemaID: 2, tableID: 0, jobType: timodel.ActionModifySchemaCharsetAndCollate, binlogInfo: &timodel.HistoryInfo{SchemaVersion: 8, DBInfo: dbInfo, TableInfo: nil, FinishedTS: 123}, query: "ALTER DATABASE Test CHARACTER SET utf8mb4;", resultQuery: "ALTER DATABASE Test CHARACTER SET utf8mb4;", schemaName: dbInfo.Name.O}, - {name: "createTable", jobID: 7, schemaID: 2, tableID: 6, jobType: timodel.ActionCreateTable, binlogInfo: &timodel.HistoryInfo{SchemaVersion: 3, DBInfo: nil, TableInfo: tblInfo, FinishedTS: 123}, query: "create table T(id int);", resultQuery: "create table T(id int);", schemaName: dbInfo.Name.O, tableName: tblInfo.Name.O}, - {name: "addColumn", jobID: 9, schemaID: 2, tableID: 6, jobType: timodel.ActionAddColumn, binlogInfo: &timodel.HistoryInfo{SchemaVersion: 4, DBInfo: nil, TableInfo: tblInfo, FinishedTS: 123}, query: "alter table T add a varchar(45);", resultQuery: "alter table T add a varchar(45);", schemaName: dbInfo.Name.O, tableName: tblInfo.Name.O}, - {name: "truncateTable", jobID: 10, schemaID: 2, tableID: 6, jobType: timodel.ActionTruncateTable, binlogInfo: &timodel.HistoryInfo{SchemaVersion: 5, DBInfo: nil, TableInfo: tblInfo, FinishedTS: 123}, query: "truncate table T;", resultQuery: "truncate table T;", schemaName: dbInfo.Name.O, tableName: tblInfo.Name.O}, - {name: "renameTable", jobID: 11, schemaID: 2, tableID: 10, jobType: timodel.ActionRenameTable, binlogInfo: &timodel.HistoryInfo{SchemaVersion: 6, DBInfo: nil, TableInfo: tblInfo, FinishedTS: 123}, query: "rename table T to RT;", resultQuery: "rename table T to RT;", schemaName: dbInfo.Name.O, tableName: newTbName.O}, - {name: "dropTable", jobID: 12, schemaID: 2, tableID: 12, jobType: timodel.ActionDropTable, binlogInfo: &timodel.HistoryInfo{SchemaVersion: 7, DBInfo: nil, TableInfo: nil, FinishedTS: 123}, query: "drop table RT;", resultQuery: "drop table RT;", schemaName: dbInfo.Name.O, tableName: newTbName.O}, - {name: "dropSchema", jobID: 13, schemaID: 2, tableID: 0, jobType: timodel.ActionDropSchema, binlogInfo: &timodel.HistoryInfo{SchemaVersion: 8, DBInfo: dbInfo, TableInfo: nil, FinishedTS: 123}, query: "drop database test;", resultQuery: "drop database test;", schemaName: dbInfo.Name.O, tableName: ""}, - } - - for _, testCase := range testCases { - // prepare for ddl - switch testCase.name { - case "addColumn": - tblInfo.Columns = []*timodel.ColumnInfo{colInfo} - case "truncateTable": - tblInfo.ID = 10 - case "renameTable": - tblInfo.ID = 12 - tblInfo.Name = newTbName - } - - job := &timodel.Job{ - ID: testCase.jobID, - State: timodel.JobStateDone, - SchemaID: testCase.schemaID, - TableID: testCase.tableID, - Type: testCase.jobType, - BinlogInfo: testCase.binlogInfo, - Query: testCase.query, - } - testDoDDLAndCheck(t, snap, job, false) - - // custom check after ddl - switch testCase.name { - case "createSchema": - _, ok := snap.SchemaByID(dbInfo.ID) - require.True(t, ok) - case "createTable": - _, ok := snap.TableByID(tblInfo.ID) - require.True(t, ok) - case "renameTable": - tb, ok := snap.TableByID(tblInfo.ID) - require.True(t, ok) - require.Equal(t, tblInfo.Name, tb.Name) - case "addColumn", "truncateTable": - tb, ok := snap.TableByID(tblInfo.ID) - require.True(t, ok) - require.Len(t, tb.Columns, 1) - case "dropTable": - _, ok := snap.TableByID(tblInfo.ID) - require.False(t, ok) - case "dropSchema": - _, ok := snap.SchemaByID(job.SchemaID) - require.False(t, ok) - } - } -} - -func TestHandleRenameTables(t *testing.T) { - // Initial schema: db_1.table_1 and db_2.table_2. - snap := newEmptySchemaSnapshot(true) - var i int64 - for i = 1; i < 3; i++ { - dbInfo := &timodel.DBInfo{ - ID: i, - Name: timodel.NewCIStr(fmt.Sprintf("db_%d", i)), - State: timodel.StatePublic, - } - err := snap.createSchema(dbInfo) - require.Nil(t, err) - } - for i = 1; i < 3; i++ { - tblInfo := &timodel.TableInfo{ - ID: 10 + i, - Name: timodel.NewCIStr(fmt.Sprintf("table_%d", i)), - State: timodel.StatePublic, - } - err := snap.createTable(model.WrapTableInfo(i, fmt.Sprintf("db_%d", i), 1, tblInfo)) - require.Nil(t, err) - } - - // rename table db1.table_1 to db2.x, db2.table_2 to db1.y - oldSchemaIDs := []int64{1, 2} - newSchemaIDs := []int64{2, 1} - oldTableIDs := []int64{11, 12} - newTableNames := []timodel.CIStr{timodel.NewCIStr("x"), timodel.NewCIStr("y")} - oldSchemaNames := []timodel.CIStr{timodel.NewCIStr("db_1"), timodel.NewCIStr("db_2")} - args := []interface{}{oldSchemaIDs, newSchemaIDs, newTableNames, oldTableIDs, oldSchemaNames} - rawArgs, err := json.Marshal(args) - require.Nil(t, err) - var job *timodel.Job = &timodel.Job{ - Type: timodel.ActionRenameTables, - RawArgs: rawArgs, - BinlogInfo: &timodel.HistoryInfo{}, - } - job.BinlogInfo.MultipleTableInfos = append(job.BinlogInfo.MultipleTableInfos, - &timodel.TableInfo{ - ID: 13, - Name: timodel.NewCIStr("x"), - State: timodel.StatePublic, - }) - job.BinlogInfo.MultipleTableInfos = append(job.BinlogInfo.MultipleTableInfos, - &timodel.TableInfo{ - ID: 14, - Name: timodel.NewCIStr("y"), - State: timodel.StatePublic, - }) - testDoDDLAndCheck(t, snap, job, false) - - var ok bool - _, ok = snap.TableByID(13) - require.True(t, ok) - _, ok = snap.TableByID(14) - require.True(t, ok) - _, ok = snap.TableByID(11) - require.False(t, ok) - _, ok = snap.TableByID(12) - require.False(t, ok) - - t1 := model.TableName{Schema: "db_2", Table: "x"} - t2 := model.TableName{Schema: "db_1", Table: "y"} - require.Equal(t, snap.tableNameToID[t1], int64(13)) - require.Equal(t, snap.tableNameToID[t2], int64(14)) -} - -func testDoDDLAndCheck(t *testing.T, snap *schemaSnapshot, job *timodel.Job, isErr bool) { - err := snap.handleDDL(job) - require.Equal(t, err != nil, isErr) -} - -func TestPKShouldBeInTheFirstPlaceWhenPKIsNotHandle(t *testing.T) { - tblInfo := timodel.TableInfo{ - Columns: []*timodel.ColumnInfo{ - { - Name: timodel.CIStr{O: "name"}, - FieldType: types.FieldType{ - Flag: mysql.NotNullFlag, - }, - }, - {Name: timodel.CIStr{O: "id"}}, - }, - Indices: []*timodel.IndexInfo{ - { - Name: timodel.CIStr{ - O: "name", - }, - Columns: []*timodel.IndexColumn{ - { - Name: timodel.CIStr{O: "name"}, - Offset: 0, - }, - }, - Unique: true, - }, - { - Name: timodel.CIStr{ - O: "PRIMARY", - }, - Columns: []*timodel.IndexColumn{ - { - Name: timodel.CIStr{O: "id"}, - Offset: 1, - }, - }, - Primary: true, - }, - }, - PKIsHandle: false, - } - info := model.WrapTableInfo(1, "", 0, &tblInfo) - cols := info.GetUniqueKeys() - require.Equal(t, cols, [][]string{ - {"id"}, {"name"}, - }) -} - -func TestPKShouldBeInTheFirstPlaceWhenPKIsHandle(t *testing.T) { - tblInfo := timodel.TableInfo{ - Indices: []*timodel.IndexInfo{ - { - Name: timodel.CIStr{ - O: "uniq_job", - }, - Columns: []*timodel.IndexColumn{ - {Name: timodel.CIStr{O: "job"}}, - }, - Unique: true, - }, - }, - Columns: []*timodel.ColumnInfo{ - { - Name: timodel.CIStr{ - O: "job", - }, - FieldType: types.FieldType{ - Flag: mysql.NotNullFlag, - }, - }, - { - Name: timodel.CIStr{ - O: "uid", - }, - FieldType: types.FieldType{ - Flag: mysql.PriKeyFlag, - }, - }, - }, - PKIsHandle: true, - } - info := model.WrapTableInfo(1, "", 0, &tblInfo) - cols := info.GetUniqueKeys() - require.Equal(t, cols, [][]string{ - {"uid"}, {"job"}, - }) -} - -func TestMultiVersionStorage(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - dbName := timodel.NewCIStr("Test") - tbName := timodel.NewCIStr("T1") - // db and ignoreDB info - dbInfo := &timodel.DBInfo{ - ID: 1, - Name: dbName, - State: timodel.StatePublic, - } - var jobs []*timodel.Job - // `createSchema` job1 - job := &timodel.Job{ - ID: 3, - State: timodel.JobStateSynced, - SchemaID: 1, - Type: timodel.ActionCreateSchema, - BinlogInfo: &timodel.HistoryInfo{SchemaVersion: 1, DBInfo: dbInfo, FinishedTS: 100}, - Query: "create database test", - } - jobs = append(jobs, job) - - // table info - tblInfo := &timodel.TableInfo{ - ID: 2, - Name: tbName, - State: timodel.StatePublic, - } - - // `createTable` job - job = &timodel.Job{ - ID: 6, - State: timodel.JobStateSynced, - SchemaID: 1, - TableID: 2, - Type: timodel.ActionCreateTable, - BinlogInfo: &timodel.HistoryInfo{SchemaVersion: 2, TableInfo: tblInfo, FinishedTS: 110}, - Query: "create table " + tbName.O, - } - - jobs = append(jobs, job) - - tbName = timodel.NewCIStr("T2") - // table info - tblInfo = &timodel.TableInfo{ - ID: 3, - Name: tbName, - State: timodel.StatePublic, - } - // `createTable` job - job = &timodel.Job{ - ID: 6, - State: timodel.JobStateSynced, - SchemaID: 1, - TableID: 3, - Type: timodel.ActionCreateTable, - BinlogInfo: &timodel.HistoryInfo{SchemaVersion: 2, TableInfo: tblInfo, FinishedTS: 120}, - Query: "create table " + tbName.O, - } - - jobs = append(jobs, job) - storage, err := NewSchemaStorage(nil, 0, nil, false) - require.Nil(t, err) - for _, job := range jobs { - err := storage.HandleDDLJob(job) - require.Nil(t, err) - } - - // `dropTable` job - job = &timodel.Job{ - ID: 6, - State: timodel.JobStateSynced, - SchemaID: 1, - TableID: 2, - Type: timodel.ActionDropTable, - BinlogInfo: &timodel.HistoryInfo{FinishedTS: 130}, - } - - err = storage.HandleDDLJob(job) - require.Nil(t, err) - - // `dropSchema` job - job = &timodel.Job{ - ID: 6, - State: timodel.JobStateSynced, - SchemaID: 1, - Type: timodel.ActionDropSchema, - BinlogInfo: &timodel.HistoryInfo{FinishedTS: 140, DBInfo: dbInfo}, - } - - err = storage.HandleDDLJob(job) - require.Nil(t, err) - - require.Equal(t, storage.(*schemaStorageImpl).resolvedTs, uint64(140)) - snap, err := storage.GetSnapshot(ctx, 100) - require.Nil(t, err) - _, exist := snap.SchemaByID(1) - require.True(t, exist) - _, exist = snap.TableByID(2) - require.False(t, exist) - _, exist = snap.TableByID(3) - require.False(t, exist) - - snap, err = storage.GetSnapshot(ctx, 115) - require.Nil(t, err) - _, exist = snap.SchemaByID(1) - require.True(t, exist) - _, exist = snap.TableByID(2) - require.True(t, exist) - _, exist = snap.TableByID(3) - require.False(t, exist) - - snap, err = storage.GetSnapshot(ctx, 125) - require.Nil(t, err) - _, exist = snap.SchemaByID(1) - require.True(t, exist) - _, exist = snap.TableByID(2) - require.True(t, exist) - _, exist = snap.TableByID(3) - require.True(t, exist) - - snap, err = storage.GetSnapshot(ctx, 135) - require.Nil(t, err) - _, exist = snap.SchemaByID(1) - require.True(t, exist) - _, exist = snap.TableByID(2) - require.False(t, exist) - _, exist = snap.TableByID(3) - require.True(t, exist) - - snap, err = storage.GetSnapshot(ctx, 140) - require.Nil(t, err) - _, exist = snap.SchemaByID(1) - require.False(t, exist) - _, exist = snap.TableByID(2) - require.False(t, exist) - _, exist = snap.TableByID(3) - require.False(t, exist) - - lastSchemaTs := storage.DoGC(0) - require.Equal(t, uint64(0), lastSchemaTs) - - snap, err = storage.GetSnapshot(ctx, 100) - require.Nil(t, err) - _, exist = snap.SchemaByID(1) - require.True(t, exist) - _, exist = snap.TableByID(2) - require.False(t, exist) - _, exist = snap.TableByID(3) - require.False(t, exist) - storage.DoGC(115) - _, err = storage.GetSnapshot(ctx, 100) - require.NotNil(t, err) - snap, err = storage.GetSnapshot(ctx, 115) - require.Nil(t, err) - _, exist = snap.SchemaByID(1) - require.True(t, exist) - _, exist = snap.TableByID(2) - require.True(t, exist) - _, exist = snap.TableByID(3) - require.False(t, exist) - - lastSchemaTs = storage.DoGC(155) - require.Equal(t, uint64(140), lastSchemaTs) - - storage.AdvanceResolvedTs(185) - - snap, err = storage.GetSnapshot(ctx, 180) - require.Nil(t, err) - _, exist = snap.SchemaByID(1) - require.False(t, exist) - _, exist = snap.TableByID(2) - require.False(t, exist) - _, exist = snap.TableByID(3) - require.False(t, exist) - _, err = storage.GetSnapshot(ctx, 130) - require.NotNil(t, err) - - cancel() - _, err = storage.GetSnapshot(ctx, 200) - require.Equal(t, errors.Cause(err), context.Canceled) -} - -func TestCreateSnapFromMeta(t *testing.T) { - store, err := mockstore.NewMockStore() - require.Nil(t, err) - defer store.Close() //nolint:errcheck - - session.SetSchemaLease(0) - session.DisableStats4Test() - domain, err := session.BootstrapSession(store) - require.Nil(t, err) - defer domain.Close() - domain.SetStatsUpdating(true) - tk := testkit.NewTestKit(t, store) - tk.MustExec("create database test2") - tk.MustExec("create table test.simple_test1 (id bigint primary key)") - tk.MustExec("create table test.simple_test2 (id bigint primary key)") - tk.MustExec("create table test2.simple_test3 (id bigint primary key)") - tk.MustExec("create table test2.simple_test4 (id bigint primary key)") - tk.MustExec("create table test2.simple_test5 (a bigint)") - ver, err := store.CurrentVersion(oracle.GlobalTxnScope) - require.Nil(t, err) - meta, err := kv.GetSnapshotMeta(store, ver.Ver) - require.Nil(t, err) - snap, err := newSchemaSnapshotFromMeta(meta, ver.Ver, false) - require.Nil(t, err) - _, ok := snap.GetTableByName("test", "simple_test1") - require.True(t, ok) - tableID, ok := snap.GetTableIDByName("test2", "simple_test5") - require.True(t, ok) - require.True(t, snap.IsIneligibleTableID(tableID)) - dbInfo, ok := snap.SchemaByTableID(tableID) - require.True(t, ok) - require.Equal(t, dbInfo.Name.O, "test2") - require.Len(t, snap.tableInSchema, 3) -} - -func TestSnapshotClone(t *testing.T) { - store, err := mockstore.NewMockStore() - require.Nil(t, err) - defer store.Close() //nolint:errcheck - - session.SetSchemaLease(0) - session.DisableStats4Test() - domain, err := session.BootstrapSession(store) - require.Nil(t, err) - defer domain.Close() - domain.SetStatsUpdating(true) - tk := testkit.NewTestKit(t, store) - tk.MustExec("create database test2") - tk.MustExec("create table test.simple_test1 (id bigint primary key)") - tk.MustExec("create table test.simple_test2 (id bigint primary key)") - tk.MustExec("create table test2.simple_test3 (id bigint primary key)") - tk.MustExec("create table test2.simple_test4 (id bigint primary key)") - tk.MustExec("create table test2.simple_test5 (a bigint)") - ver, err := store.CurrentVersion(oracle.GlobalTxnScope) - require.Nil(t, err) - meta, err := kv.GetSnapshotMeta(store, ver.Ver) - require.Nil(t, err) - snap, err := newSchemaSnapshotFromMeta(meta, ver.Ver, false /* explicitTables */) - require.Nil(t, err) - - clone := snap.Clone() - require.Equal(t, clone.tableNameToID, snap.tableNameToID) - require.Equal(t, clone.schemaNameToID, snap.schemaNameToID) - require.Equal(t, clone.truncateTableID, snap.truncateTableID) - require.Equal(t, clone.ineligibleTableID, snap.ineligibleTableID) - require.Equal(t, clone.currentTs, snap.currentTs) - require.Equal(t, clone.explicitTables, snap.explicitTables) - require.Equal(t, len(clone.tables), len(snap.tables)) - require.Equal(t, len(clone.schemas), len(snap.schemas)) - require.Equal(t, len(clone.partitionTable), len(snap.partitionTable)) - - tableCount := len(snap.tables) - clone.tables = make(map[int64]*model.TableInfo) - require.Len(t, snap.tables, tableCount) -} - -func TestExplicitTables(t *testing.T) { - store, err := mockstore.NewMockStore() - require.Nil(t, err) - defer store.Close() //nolint:errcheck - - session.SetSchemaLease(0) - session.DisableStats4Test() - domain, err := session.BootstrapSession(store) - require.Nil(t, err) - defer domain.Close() - domain.SetStatsUpdating(true) - tk := testkit.NewTestKit(t, store) - ver1, err := store.CurrentVersion(oracle.GlobalTxnScope) - require.Nil(t, err) - tk.MustExec("create database test2") - tk.MustExec("create table test.simple_test1 (id bigint primary key)") - tk.MustExec("create table test.simple_test2 (id bigint unique key)") - tk.MustExec("create table test2.simple_test3 (a bigint)") - tk.MustExec("create table test2.simple_test4 (a varchar(20) unique key)") - tk.MustExec("create table test2.simple_test5 (a varchar(20))") - ver2, err := store.CurrentVersion(oracle.GlobalTxnScope) - require.Nil(t, err) - meta1, err := kv.GetSnapshotMeta(store, ver1.Ver) - require.Nil(t, err) - snap1, err := newSchemaSnapshotFromMeta(meta1, ver1.Ver, true /* explicitTables */) - require.Nil(t, err) - meta2, err := kv.GetSnapshotMeta(store, ver2.Ver) - require.Nil(t, err) - snap2, err := newSchemaSnapshotFromMeta(meta2, ver2.Ver, false /* explicitTables */) - require.Nil(t, err) - snap3, err := newSchemaSnapshotFromMeta(meta2, ver2.Ver, true /* explicitTables */) - require.Nil(t, err) - - require.Equal(t, len(snap2.tables)-len(snap1.tables), 5) - // some system tables are also ineligible - require.GreaterOrEqual(t, len(snap2.ineligibleTableID), 4) - - require.Equal(t, len(snap3.tables)-len(snap1.tables), 5) - require.Len(t, snap3.ineligibleTableID, 0) -} - -/* -TODO: Untested Action: - -ActionAddForeignKey ActionType = 9 -ActionDropForeignKey ActionType = 10 -ActionRebaseAutoID ActionType = 13 -ActionShardRowID ActionType = 16 -ActionLockTable ActionType = 27 -ActionUnlockTable ActionType = 28 -ActionRepairTable ActionType = 29 -ActionSetTiFlashReplica ActionType = 30 -ActionUpdateTiFlashReplicaStatus ActionType = 31 -ActionCreateSequence ActionType = 34 -ActionAlterSequence ActionType = 35 -ActionDropSequence ActionType = 36 -ActionModifyTableAutoIdCache ActionType = 39 -ActionRebaseAutoRandomBase ActionType = 40 -ActionExchangeTablePartition ActionType = 42 -ActionAddCheckConstraint ActionType = 43 -ActionDropCheckConstraint ActionType = 44 -ActionAlterCheckConstraint ActionType = 45 -ActionAlterTableAlterPartition ActionType = 46 - -... Any Action which of value is greater than 46 ... -*/ -func TestSchemaStorage(t *testing.T) { - ctx := context.Background() - testCases := [][]string{{ - "create database test_ddl1", // ActionCreateSchema - "create table test_ddl1.simple_test1 (id bigint primary key)", // ActionCreateTable - "create table test_ddl1.simple_test2 (id bigint)", // ActionCreateTable - "create table test_ddl1.simple_test3 (id bigint primary key)", // ActionCreateTable - "create table test_ddl1.simple_test4 (id bigint primary key)", // ActionCreateTable - "DROP TABLE test_ddl1.simple_test3", // ActionDropTable - "ALTER TABLE test_ddl1.simple_test1 ADD COLUMN c1 INT NOT NULL", // ActionAddColumn - "ALTER TABLE test_ddl1.simple_test1 ADD c2 INT NOT NULL AFTER id", // ActionAddColumn - "ALTER TABLE test_ddl1.simple_test1 ADD c3 INT NOT NULL, ADD c4 INT NOT NULL", // ActionAddColumns - "ALTER TABLE test_ddl1.simple_test1 DROP c1", // ActionDropColumn - "ALTER TABLE test_ddl1.simple_test1 DROP c2, DROP c3", // ActionDropColumns - "ALTER TABLE test_ddl1.simple_test1 ADD INDEX (c4)", // ActionAddIndex - "ALTER TABLE test_ddl1.simple_test1 DROP INDEX c4", // ActionDropIndex - "TRUNCATE test_ddl1.simple_test1", // ActionTruncateTable - "ALTER DATABASE test_ddl1 CHARACTER SET = binary COLLATE binary", // ActionModifySchemaCharsetAndCollate - "ALTER TABLE test_ddl1.simple_test2 ADD c1 INT NOT NULL, ADD c2 INT NOT NULL", // ActionAddColumns - "ALTER TABLE test_ddl1.simple_test2 ADD INDEX (c1)", // ActionAddIndex - "ALTER TABLE test_ddl1.simple_test2 ALTER INDEX c1 INVISIBLE", // ActionAlterIndexVisibility - "ALTER TABLE test_ddl1.simple_test2 RENAME INDEX c1 TO idx_c1", // ActionRenameIndex - "ALTER TABLE test_ddl1.simple_test2 MODIFY c2 BIGINT", // ActionModifyColumn - "CREATE VIEW test_ddl1.view_test2 AS SELECT * FROM test_ddl1.simple_test2 WHERE id > 2", // ActionCreateView - "DROP VIEW test_ddl1.view_test2", // ActionDropView - "RENAME TABLE test_ddl1.simple_test2 TO test_ddl1.simple_test5", // ActionRenameTable - "DROP DATABASE test_ddl1", // ActionDropSchema - "create database test_ddl2", // ActionCreateSchema - "create table test_ddl2.simple_test1 (id bigint primary key, c1 int not null unique key)", // ActionCreateTable - `CREATE TABLE test_ddl2.employees ( - id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, - fname VARCHAR(25) NOT NULL, - lname VARCHAR(25) NOT NULL, - store_id INT NOT NULL, - department_id INT NOT NULL - ) - - PARTITION BY RANGE(id) ( - PARTITION p0 VALUES LESS THAN (5), - PARTITION p1 VALUES LESS THAN (10), - PARTITION p2 VALUES LESS THAN (15), - PARTITION p3 VALUES LESS THAN (20) - )`, // ActionCreateTable - "ALTER TABLE test_ddl2.employees DROP PARTITION p2", // ActionDropTablePartition - "ALTER TABLE test_ddl2.employees ADD PARTITION (PARTITION p4 VALUES LESS THAN (25))", // ActionAddTablePartition - "ALTER TABLE test_ddl2.employees TRUNCATE PARTITION p3", // ActionTruncateTablePartition - "alter table test_ddl2.employees comment='modify comment'", // ActionModifyTableComment - "alter table test_ddl2.simple_test1 drop primary key", // ActionDropPrimaryKey - "alter table test_ddl2.simple_test1 add primary key pk(id)", // ActionAddPrimaryKey - "ALTER TABLE test_ddl2.simple_test1 ALTER id SET DEFAULT 18", // ActionSetDefaultValue - "ALTER TABLE test_ddl2.simple_test1 CHARACTER SET = utf8mb4", // ActionModifyTableCharsetAndCollate - // "recover table test_ddl2.employees", // ActionRecoverTable this ddl can't work on mock tikv - - "DROP TABLE test_ddl2.employees", - `CREATE TABLE test_ddl2.employees2 ( - id INT NOT NULL, - fname VARCHAR(25) NOT NULL, - lname VARCHAR(25) NOT NULL, - store_id INT NOT NULL, - department_id INT NOT NULL - ) - - PARTITION BY RANGE(id) ( - PARTITION p0 VALUES LESS THAN (5), - PARTITION p1 VALUES LESS THAN (10), - PARTITION p2 VALUES LESS THAN (15), - PARTITION p3 VALUES LESS THAN (20) - )`, - "ALTER TABLE test_ddl2.employees2 CHARACTER SET = utf8mb4", - "DROP DATABASE test_ddl2", - }} - - testOneGroup := func(tc []string) { - store, err := mockstore.NewMockStore() - require.Nil(t, err) - defer store.Close() //nolint:errcheck - ticonfig.UpdateGlobal(func(conf *ticonfig.Config) { - conf.AlterPrimaryKey = true - }) - session.SetSchemaLease(0) - session.DisableStats4Test() - domain, err := session.BootstrapSession(store) - require.Nil(t, err) - defer domain.Close() - domain.SetStatsUpdating(true) - tk := testkit.NewTestKit(t, store) - - for _, ddlSQL := range tc { - tk.MustExec(ddlSQL) - } - - jobs, err := getAllHistoryDDLJob(store) - require.Nil(t, err) - scheamStorage, err := NewSchemaStorage(nil, 0, nil, false) - require.Nil(t, err) - for _, job := range jobs { - err := scheamStorage.HandleDDLJob(job) - require.Nil(t, err) - } - - for _, job := range jobs { - ts := job.BinlogInfo.FinishedTS - meta, err := kv.GetSnapshotMeta(store, ts) - require.Nil(t, err) - snapFromMeta, err := newSchemaSnapshotFromMeta(meta, ts, false) - require.Nil(t, err) - snapFromSchemaStore, err := scheamStorage.GetSnapshot(ctx, ts) - require.Nil(t, err) - - tidySchemaSnapshot(snapFromMeta) - tidySchemaSnapshot(snapFromSchemaStore) - require.Equal(t, snapFromMeta, snapFromSchemaStore) - } - } - - for _, tc := range testCases { - testOneGroup(tc) - } -} - -func tidySchemaSnapshot(snap *schemaSnapshot) { - for _, dbInfo := range snap.schemas { - if len(dbInfo.Tables) == 0 { - dbInfo.Tables = nil - } - } - for _, tableInfo := range snap.tables { - tableInfo.TableInfoVersion = 0 - if len(tableInfo.Columns) == 0 { - tableInfo.Columns = nil - } - if len(tableInfo.Indices) == 0 { - tableInfo.Indices = nil - } - if len(tableInfo.ForeignKeys) == 0 { - tableInfo.ForeignKeys = nil - } - } - // the snapshot from meta doesn't know which ineligible tables that have existed in history - // so we delete the ineligible tables which are already not exist - for tableID := range snap.ineligibleTableID { - if _, ok := snap.tables[tableID]; !ok { - delete(snap.ineligibleTableID, tableID) - } - } - // the snapshot from meta doesn't know which tables are truncated, so we just ignore it - snap.truncateTableID = nil - for _, v := range snap.tableInSchema { - sort.Slice(v, func(i, j int) bool { return v[i] < v[j] }) - } -} - -func getAllHistoryDDLJob(storage tidbkv.Storage) ([]*timodel.Job, error) { - s, err := session.CreateSession(storage) - if err != nil { - return nil, errors.Trace(err) - } - - if s != nil { - defer s.Close() - } - - store := domain.GetDomain(s.(sessionctx.Context)).Store() - txn, err := store.Begin() - if err != nil { - return nil, errors.Trace(err) - } - defer txn.Rollback() //nolint:errcheck - txnMeta := timeta.NewMeta(txn) - - jobs, err := txnMeta.GetAllHistoryDDLJobs() - if err != nil { - return nil, errors.Trace(err) - } - return jobs, nil -} diff --git a/cdc/cdc/http_handler.go b/cdc/cdc/http_handler.go index 59685c0d..817af203 100644 --- a/cdc/cdc/http_handler.go +++ b/cdc/cdc/http_handler.go @@ -39,8 +39,8 @@ const ( APIOpVarChangefeedID = "cf-id" // APIOpVarTargetCaptureID is the key of to-capture ID in HTTP API APIOpVarTargetCaptureID = "target-cp-id" - // APIOpVarTableID is the key of table ID in HTTP API - APIOpVarTableID = "table-id" + // APIOpVarKeySpanID is the key of keyspan ID in HTTP API + APIOpVarKeySpanID = "keyspan-id" // APIOpForceRemoveChangefeed is used when remove a changefeed APIOpForceRemoveChangefeed = "force-remove" ) @@ -166,7 +166,7 @@ func (s *Server) handleRebalanceTrigger(w http.ResponseWriter, req *http.Request handleOwnerResp(w, err) } -func (s *Server) handleMoveTable(w http.ResponseWriter, req *http.Request) { +func (s *Server) handleMoveKeySpan(w http.ResponseWriter, req *http.Request) { if s.capture == nil { // for test only handleOwnerResp(w, concurrency.ErrElectionNotLeader) @@ -190,16 +190,16 @@ func (s *Server) handleMoveTable(w http.ResponseWriter, req *http.Request) { cerror.ErrAPIInvalidParam.GenWithStack("invalid target capture id: %s", to)) return } - tableIDStr := req.Form.Get(APIOpVarTableID) - tableID, err := strconv.ParseInt(tableIDStr, 10, 64) + keyspanIDStr := req.Form.Get(APIOpVarKeySpanID) + keyspanID, err := strconv.ParseInt(keyspanIDStr, 10, 64) if err != nil { writeError(w, http.StatusBadRequest, - cerror.ErrAPIInvalidParam.GenWithStack("invalid tableID: %s", tableIDStr)) + cerror.ErrAPIInvalidParam.GenWithStack("invalid keyspanID: %s", keyspanIDStr)) return } err = s.capture.OperateOwnerUnderLock(func(owner *owner.Owner) error { - owner.ManualSchedule(changefeedID, to, tableID) + owner.ManualSchedule(changefeedID, to, uint64(keyspanID)) return nil }) diff --git a/cdc/cdc/http_router.go b/cdc/cdc/http_router.go index 8e2a8f7f..7f2ab4f6 100644 --- a/cdc/cdc/http_router.go +++ b/cdc/cdc/http_router.go @@ -64,8 +64,8 @@ func newRouter(captureHandler capture.HTTPHandler) *gin.Engine { changefeedGroup.POST("/:changefeed_id/pause", captureHandler.PauseChangefeed) changefeedGroup.POST("/:changefeed_id/resume", captureHandler.ResumeChangefeed) changefeedGroup.DELETE("/:changefeed_id", captureHandler.RemoveChangefeed) - changefeedGroup.POST("/:changefeed_id/tables/rebalance_table", captureHandler.RebalanceTable) - changefeedGroup.POST("/:changefeed_id/tables/move_table", captureHandler.MoveTable) + changefeedGroup.POST("/:changefeed_id/keyspans/rebalance_keyspan", captureHandler.RebalanceKeySpan) + changefeedGroup.POST("/:changefeed_id/keyspans/move_keyspan", captureHandler.MoveKeySpan) } // owner API diff --git a/cdc/cdc/http_status.go b/cdc/cdc/http_status.go index 05c79be2..d56aaa0b 100644 --- a/cdc/cdc/http_status.go +++ b/cdc/cdc/http_status.go @@ -52,7 +52,7 @@ func (s *Server) startStatusHTTP(lis net.Listener) error { router.POST("/capture/owner/resign", gin.WrapF(s.handleResignOwner)) router.POST("/capture/owner/admin", gin.WrapF(s.handleChangefeedAdmin)) router.POST("/capture/owner/rebalance_trigger", gin.WrapF(s.handleRebalanceTrigger)) - router.POST("/capture/owner/move_table", gin.WrapF(s.handleMoveTable)) + router.POST("/capture/owner/move_keyspan", gin.WrapF(s.handleMoveKeySpan)) router.POST("/capture/owner/changefeed/query", gin.WrapF(s.handleChangefeedQuery)) router.POST("/admin/log", gin.WrapF(handleAdminLogLevel)) diff --git a/cdc/cdc/http_status_test.go b/cdc/cdc/http_status_test.go index cfad7786..79e37788 100644 --- a/cdc/cdc/http_status_test.go +++ b/cdc/cdc/http_status_test.go @@ -95,7 +95,7 @@ func (s *httpStatusSuite) TestHTTPStatus(c *check.C) { testReisgnOwner(c) testHandleChangefeedAdmin(c) testHandleRebalance(c) - testHandleMoveTable(c) + testHandleMoveKeySpan(c) testHandleChangefeedQuery(c) testHandleFailpoint(c) @@ -133,8 +133,8 @@ func testHandleRebalance(c *check.C) { testRequestNonOwnerFailed(c, uri) } -func testHandleMoveTable(c *check.C) { - uri := fmt.Sprintf("http://%s/capture/owner/move_table", advertiseAddr4Test) +func testHandleMoveKeySpan(c *check.C) { + uri := fmt.Sprintf("http://%s/capture/owner/move_keyspan", advertiseAddr4Test) testRequestNonOwnerFailed(c, uri) } diff --git a/cdc/cdc/kv/client.go b/cdc/cdc/kv/client.go index 29f366ff..590ba8e0 100644 --- a/cdc/cdc/kv/client.go +++ b/cdc/cdc/kv/client.go @@ -536,7 +536,7 @@ func (s *eventFeedSession) eventFeed(ctx context.Context, ts uint64) error { } }) - tableID, tableName := util.TableIDFromCtx(ctx) + // tableID, tableName := util.KeySpanIDFromCtx(ctx) cfID := util.ChangefeedIDFromCtx(ctx) g.Go(func() error { timer := time.NewTimer(defaultCheckRegionRateLimitInterval) @@ -564,7 +564,7 @@ func (s *eventFeedSession) eventFeed(ctx context.Context, ts uint64) error { zap.Uint64("regionID", errInfo.singleRegionInfo.verID.GetID()), zap.Uint64("ts", errInfo.singleRegionInfo.ts), zap.String("changefeed", cfID), zap.Stringer("span", errInfo.span), - zap.Int64("tableID", tableID), zap.String("tableName", tableName), + // zap.Int64("tableID", tableID), zap.String("tableName", tableName), zapFieldAddr) } // rate limit triggers, add the error info to the rate limit queue. diff --git a/cdc/cdc/kv/metrics.go b/cdc/cdc/kv/metrics.go index bb19c209..c76908d8 100644 --- a/cdc/cdc/kv/metrics.go +++ b/cdc/cdc/kv/metrics.go @@ -23,21 +23,21 @@ var ( eventFeedErrorCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "kvclient", Name: "event_feed_error_count", Help: "The number of error return by tikv", }, []string{"type"}) eventFeedGauge = prometheus.NewGauge( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "kvclient", Name: "event_feed_count", Help: "The number of event feed running", }) scanRegionsDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "kvclient", Name: "scan_regions_duration_seconds", Help: "The time it took to finish a scanRegions call.", @@ -45,7 +45,7 @@ var ( }, []string{"capture"}) eventSize = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "kvclient", Name: "event_size_bytes", Help: "Size of KV events.", @@ -53,42 +53,42 @@ var ( }, []string{"capture", "type"}) pullEventCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "kvclient", Name: "pull_event_count", Help: "event count received by this puller", }, []string{"type", "capture", "changefeed"}) sendEventCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "kvclient", Name: "send_event_count", Help: "event count sent to event channel by this puller", }, []string{"type", "capture", "changefeed"}) clientChannelSize = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "kvclient", Name: "channel_size", Help: "size of each channel in kv client", }, []string{"channel"}) clientRegionTokenSize = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "kvclient", Name: "region_token", Help: "size of region token in kv client", }, []string{"store", "changefeed", "capture"}) cachedRegionSize = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "kvclient", Name: "cached_region", Help: "cached region that has not requested to TiKV in kv client", }, []string{"store", "changefeed", "capture"}) batchResolvedEventSize = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "kvclient", Name: "batch_resolved_event_size", Help: "The number of region in one batch resolved ts event", @@ -96,7 +96,7 @@ var ( }, []string{"capture", "changefeed"}) grpcPoolStreamGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "kvclient", Name: "grpc_stream_count", Help: "active stream count of each gRPC connection", diff --git a/cdc/cdc/metrics.go b/cdc/cdc/metrics.go index e95d25e6..895cfdb2 100644 --- a/cdc/cdc/metrics.go +++ b/cdc/cdc/metrics.go @@ -19,15 +19,9 @@ import ( "github.com/tikv/migration/cdc/cdc/kv" "github.com/tikv/migration/cdc/cdc/owner" "github.com/tikv/migration/cdc/cdc/processor" - tablepipeline "github.com/tikv/migration/cdc/cdc/processor/pipeline" + keyspanpipeline "github.com/tikv/migration/cdc/cdc/processor/pipeline" "github.com/tikv/migration/cdc/cdc/puller" - redowriter "github.com/tikv/migration/cdc/cdc/redo/writer" "github.com/tikv/migration/cdc/cdc/sink" - "github.com/tikv/migration/cdc/cdc/sorter" - "github.com/tikv/migration/cdc/cdc/sorter/leveldb" - "github.com/tikv/migration/cdc/cdc/sorter/memory" - "github.com/tikv/migration/cdc/cdc/sorter/unified" - "github.com/tikv/migration/cdc/pkg/actor" "github.com/tikv/migration/cdc/pkg/db" "github.com/tikv/migration/cdc/pkg/etcd" "github.com/tikv/migration/cdc/pkg/orchestrator" @@ -45,18 +39,18 @@ func init() { sink.InitMetrics(registry) entry.InitMetrics(registry) processor.InitMetrics(registry) - tablepipeline.InitMetrics(registry) + keyspanpipeline.InitMetrics(registry) owner.InitMetrics(registry) etcd.InitMetrics(registry) initServerMetrics(registry) - actor.InitMetrics(registry) + // actor.InitMetrics(registry) orchestrator.InitMetrics(registry) p2p.InitMetrics(registry) // Sorter metrics - sorter.InitMetrics(registry) - memory.InitMetrics(registry) - unified.InitMetrics(registry) - leveldb.InitMetrics(registry) - redowriter.InitMetrics(registry) + // sorter.InitMetrics(registry) + // memory.InitMetrics(registry) + // unified.InitMetrics(registry) + // leveldb.InitMetrics(registry) + // redowriter.InitMetrics(registry) db.InitMetrics(registry) } diff --git a/cdc/cdc/metrics_server.go b/cdc/cdc/metrics_server.go index 8ebcab1f..1b10e514 100644 --- a/cdc/cdc/metrics_server.go +++ b/cdc/cdc/metrics_server.go @@ -24,7 +24,7 @@ import ( var ( etcdHealthCheckDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "server", Name: "etcd_health_check_duration", Help: "Bucketed histogram of processing time (s) of flushing events in processor", @@ -33,7 +33,7 @@ var ( goGC = prometheus.NewGauge( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "server", Name: "go_gc", Help: "The value of GOGC", @@ -41,7 +41,7 @@ var ( goMaxProcs = prometheus.NewGauge( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "server", Name: "go_max_procs", Help: "The value of GOMAXPROCS", diff --git a/cdc/cdc/model/changefeed.go b/cdc/cdc/model/changefeed.go index d652caa0..dfd141ae 100644 --- a/cdc/cdc/model/changefeed.go +++ b/cdc/cdc/model/changefeed.go @@ -25,13 +25,13 @@ import ( "github.com/pingcap/log" "github.com/tikv/client-go/v2/oracle" "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/cyclic/mark" cerror "github.com/tikv/migration/cdc/pkg/errors" cerrors "github.com/tikv/migration/cdc/pkg/errors" "github.com/tikv/migration/cdc/pkg/version" "go.uber.org/zap" ) +// TODO(zeminzhou): Maybe TiKV CDC don't need sort, cyclic // SortEngine is the sorter engine type SortEngine = string @@ -217,15 +217,6 @@ func (info *ChangeFeedInfo) Unmarshal(data []byte) error { return errors.Annotatef( cerror.WrapError(cerror.ErrUnmarshalFailed, err), "Unmarshal data: %v", data) } - // TODO(neil) find a better way to let sink know cyclic is enabled. - if info.Config != nil && info.Config.Cyclic.IsEnabled() { - cyclicCfg, err := info.Config.Cyclic.Marshal() - if err != nil { - return errors.Annotatef( - cerror.WrapError(cerror.ErrMarshalFailed, err), "Marshal data: %v", data) - } - info.Opts[mark.OptCyclicConfig] = cyclicCfg - } return nil } diff --git a/cdc/cdc/model/changefeed_test.go b/cdc/cdc/model/changefeed_test.go index 68e1f19b..c9acdc37 100644 --- a/cdc/cdc/model/changefeed_test.go +++ b/cdc/cdc/model/changefeed_test.go @@ -114,9 +114,7 @@ func TestFillV1(t *testing.T) { require.Nil(t, err) require.Equal(t, &ChangeFeedInfo{ SinkURI: "blackhole://", - Opts: map[string]string{ - "_cyclic_relax_sql_mode": `{"enable":true,"replica-id":1,"filter-replica-ids":[2,3],"id-buckets":4,"sync-ddl":true}`, - }, + Opts: map[string]string{}, StartTs: 417136892416622595, Engine: "memory", SortDir: ".", diff --git a/cdc/cdc/model/http_model.go b/cdc/cdc/model/http_model.go index 9fb5a792..e0a598f1 100644 --- a/cdc/cdc/model/http_model.go +++ b/cdc/cdc/model/http_model.go @@ -131,13 +131,13 @@ type ChangefeedConfig struct { SinkURI string `json:"sink_uri"` // timezone used when checking sink uri TimeZone string `json:"timezone" default:"system"` - // if true, force to replicate some ineligible tables - ForceReplicate bool `json:"force_replicate" default:"false"` - IgnoreIneligibleTable bool `json:"ignore_ineligible_table" default:"false"` - FilterRules []string `json:"filter_rules"` - IgnoreTxnStartTs []uint64 `json:"ignore_txn_start_ts"` - MounterWorkerNum int `json:"mounter_worker_num" default:"16"` - SinkConfig *config.SinkConfig `json:"sink_config"` + // if true, force to replicate some ineligible keyspans + // ForceReplicate bool `json:"force_replicate" default:"false"` + // IgnoreIneligibleKeySpan bool `json:"ignore_ineligible_keyspan" default:"false"` + // FilterRules []string `json:"filter_rules"` + // IgnoreTxnStartTs []uint64 `json:"ignore_txn_start_ts"` + // MounterWorkerNum int `json:"mounter_worker_num" default:"16"` + SinkConfig *config.SinkConfig `json:"sink_config"` } // ProcessorCommonInfo holds the common info of a processor @@ -152,8 +152,8 @@ type ProcessorDetail struct { CheckPointTs uint64 `json:"checkpoint_ts"` // The event that satisfies CommitTs <= ResolvedTs can be synchronized. ResolvedTs uint64 `json:"resolved_ts"` - // all table ids that this processor are replicating - Tables []int64 `json:"table_ids"` + // all keyspan ids that this processor are replicating + KeySpans []uint64 `json:"keyspan_ids"` // The count of events that have been replicated. Count uint64 `json:"count"` // Error code when error happens @@ -163,9 +163,9 @@ type ProcessorDetail struct { // CaptureTaskStatus holds TaskStatus of a capture type CaptureTaskStatus struct { CaptureID string `json:"capture_id"` - // Table list, containing tables that processor should process - Tables []int64 `json:"table_ids"` - Operation map[TableID]*TableOperation `json:"table_operations"` + // KeySpan list, containing keyspans that processor should process + KeySpans []uint64 `json:"keyspan_ids"` + Operation map[KeySpanID]*KeySpanOperation `json:"keyspan_operations"` } // Capture holds common information of a capture in cdc diff --git a/cdc/cdc/model/kv.go b/cdc/cdc/model/kv.go index b1afbb17..c5c11837 100644 --- a/cdc/cdc/model/kv.go +++ b/cdc/cdc/model/kv.go @@ -80,7 +80,8 @@ type RawKVEntry struct { CRTs uint64 `msg:"crts"` // Additional debug info - RegionID uint64 `msg:"region_id"` + RegionID uint64 `msg:"region_id"` + KeySpanID uint64 `msg:"keyspan_id"` } func (v *RawKVEntry) String() string { diff --git a/cdc/cdc/model/owner.go b/cdc/cdc/model/owner.go index 6fbdb0a2..791d3b18 100644 --- a/cdc/cdc/model/owner.go +++ b/cdc/cdc/model/owner.go @@ -123,42 +123,42 @@ func (tp *TaskPosition) Clone() *TaskPosition { return ret } -// MoveTableStatus represents for the status of a MoveTableJob -type MoveTableStatus int +// MoveKeySpanStatus represents for the status of a MoveKeySpanJob +type MoveKeySpanStatus int -// All MoveTable status +// All MoveKeySpan status const ( - MoveTableStatusNone MoveTableStatus = iota - MoveTableStatusDeleted - MoveTableStatusFinished + MoveKeySpanStatusNone MoveKeySpanStatus = iota + MoveKeySpanStatusDeleted + MoveKeySpanStatusFinished ) -// MoveTableJob records a move operation of a table -type MoveTableJob struct { - From CaptureID - To CaptureID - TableID TableID - TableReplicaInfo *TableReplicaInfo - Status MoveTableStatus +// MoveKeySpanJob records a move operation of a keyspan +type MoveKeySpanJob struct { + From CaptureID + To CaptureID + KeySpanID KeySpanID + KeySpanReplicaInfo *KeySpanReplicaInfo + Status MoveKeySpanStatus } -// All TableOperation flags +// All KeySpanOperation flags const ( - // Move means after the delete operation, the table will be re added. + // Move means after the delete operation, the kyespan will be re added. // This field is necessary since we must persist enough information to - // restore complete table operation in case of processor or owner crashes. - OperFlagMoveTable uint64 = 1 << iota + // restore complete keyspan operation in case of processor or owner crashes. + OperFlagMoveKeySpan uint64 = 1 << iota ) -// All TableOperation status +// All KeySpanOperation status const ( OperDispatched uint64 = iota OperProcessed OperFinished ) -// TableOperation records the current information of a table migration -type TableOperation struct { +// KeySpanOperation records the current information of a keyspan migration +type KeySpanOperation struct { Delete bool `json:"delete"` Flag uint64 `json:"flag,omitempty"` // if the operation is a delete operation, BoundaryTs is checkpoint ts @@ -167,19 +167,19 @@ type TableOperation struct { Status uint64 `json:"status,omitempty"` } -// TableProcessed returns whether the table has been processed by processor -func (o *TableOperation) TableProcessed() bool { +// KeySpanProcessed returns whether the keyspan has been processed by processor +func (o *KeySpanOperation) KeySpanProcessed() bool { return o.Status == OperProcessed || o.Status == OperFinished } -// TableApplied returns whether the table has finished the startup procedure. -// Returns true if table has been processed by processor and resolved ts reaches global resolved ts. -func (o *TableOperation) TableApplied() bool { +// KeySpanApplied returns whether the keyspan has finished the startup procedure. +// Returns true if keyspan has been processed by processor and resolved ts reaches global resolved ts. +func (o *KeySpanOperation) KeySpanApplied() bool { return o.Status == OperFinished } // Clone returns a deep-clone of the struct -func (o *TableOperation) Clone() *TableOperation { +func (o *KeySpanOperation) Clone() *KeySpanOperation { if o == nil { return nil } @@ -189,9 +189,9 @@ func (o *TableOperation) Clone() *TableOperation { // TaskWorkload records the workloads of a task // the value of the struct is the workload -type TaskWorkload map[TableID]WorkloadInfo +type TaskWorkload map[KeySpanID]WorkloadInfo -// WorkloadInfo records the workload info of a table +// WorkloadInfo records the workload info of a keyspan type WorkloadInfo struct { Workload uint64 `json:"workload"` } @@ -212,14 +212,16 @@ func (w *TaskWorkload) Marshal() (string, error) { return string(data), cerror.WrapError(cerror.ErrMarshalFailed, err) } -// TableReplicaInfo records the table replica info -type TableReplicaInfo struct { - StartTs Ts `json:"start-ts"` - MarkTableID TableID `json:"mark-table-id"` +// KeySpanReplicaInfo records the keyspan replica info +type KeySpanReplicaInfo struct { + StartTs Ts `json:"start-ts"` + Start []byte + End []byte + // MarkKeySpanID KeySpanID `json:"mark-keyspan-id"` } -// Clone clones a TableReplicaInfo -func (i *TableReplicaInfo) Clone() *TableReplicaInfo { +// Clone clones a KeySpanReplicaInfo +func (i *KeySpanReplicaInfo) Clone() *KeySpanReplicaInfo { if i == nil { return nil } @@ -229,11 +231,11 @@ func (i *TableReplicaInfo) Clone() *TableReplicaInfo { // TaskStatus records the task information of a capture type TaskStatus struct { - // Table information list, containing tables that processor should process, updated by ownrer, processor is read only. - Tables map[TableID]*TableReplicaInfo `json:"tables"` - Operation map[TableID]*TableOperation `json:"operation"` // Deprecated - AdminJobType AdminJobType `json:"admin-job-type"` - ModRevision int64 `json:"-"` + // KeySpan information list, containing keyspans that processor should process, updated by ownrer, processor is read only. + KeySpans map[KeySpanID]*KeySpanReplicaInfo `json:"keyspans"` + Operation map[KeySpanID]*KeySpanOperation `json:"operation"` // Deprecated + AdminJobType AdminJobType `json:"admin-job-type"` + ModRevision int64 `json:"-"` } // String implements fmt.Stringer interface. @@ -242,46 +244,46 @@ func (ts *TaskStatus) String() string { return data } -// RemoveTable remove the table in TableInfos and add a remove table operation. -func (ts *TaskStatus) RemoveTable(id TableID, boundaryTs Ts, isMoveTable bool) (*TableReplicaInfo, bool) { - if ts.Tables == nil { +// RemoveKeySpan remove the keyspan in KeySpanInfos and add a remove keyspan operation. +func (ts *TaskStatus) RemoveKeySpan(id KeySpanID, boundaryTs Ts, isMoveKeySpan bool) (*KeySpanReplicaInfo, bool) { + if ts.KeySpans == nil { return nil, false } - table, exist := ts.Tables[id] + keyspan, exist := ts.KeySpans[id] if !exist { return nil, false } - delete(ts.Tables, id) - log.Info("remove a table", zap.Int64("tableId", id), zap.Uint64("boundaryTs", boundaryTs), zap.Bool("isMoveTable", isMoveTable)) + delete(ts.KeySpans, id) + log.Info("remove a keyspan", zap.Uint64("keyspanId", id), zap.Uint64("boundaryTs", boundaryTs), zap.Bool("isMoveKeySpan", isMoveKeySpan)) if ts.Operation == nil { - ts.Operation = make(map[TableID]*TableOperation) + ts.Operation = make(map[KeySpanID]*KeySpanOperation) } - op := &TableOperation{ + op := &KeySpanOperation{ Delete: true, BoundaryTs: boundaryTs, } - if isMoveTable { - op.Flag |= OperFlagMoveTable + if isMoveKeySpan { + op.Flag |= OperFlagMoveKeySpan } ts.Operation[id] = op - return table, true + return keyspan, true } -// AddTable add the table in TableInfos and add a add table operation. -func (ts *TaskStatus) AddTable(id TableID, table *TableReplicaInfo, boundaryTs Ts) { - if ts.Tables == nil { - ts.Tables = make(map[TableID]*TableReplicaInfo) +// AddKeySpan add the keyspan in KeySpanInfos and add a add kyespan operation. +func (ts *TaskStatus) AddKeySpan(id KeySpanID, keyspan *KeySpanReplicaInfo, boundaryTs Ts) { + if ts.KeySpans == nil { + ts.KeySpans = make(map[KeySpanID]*KeySpanReplicaInfo) } - _, exist := ts.Tables[id] + _, exist := ts.KeySpans[id] if exist { return } - ts.Tables[id] = table - log.Info("add a table", zap.Int64("tableId", id), zap.Uint64("boundaryTs", boundaryTs)) + ts.KeySpans[id] = keyspan + log.Info("add a keyspan", zap.Uint64("keyspanId", id), zap.Uint64("boundaryTs", boundaryTs)) if ts.Operation == nil { - ts.Operation = make(map[TableID]*TableOperation) + ts.Operation = make(map[KeySpanID]*KeySpanOperation) } - ts.Operation[id] = &TableOperation{ + ts.Operation[id] = &KeySpanOperation{ Delete: false, BoundaryTs: boundaryTs, Status: OperDispatched, @@ -291,7 +293,7 @@ func (ts *TaskStatus) AddTable(id TableID, table *TableReplicaInfo, boundaryTs T // SomeOperationsUnapplied returns true if there are some operations not applied func (ts *TaskStatus) SomeOperationsUnapplied() bool { for _, o := range ts.Operation { - if !o.TableApplied() { + if !o.KeySpanApplied() { return true } } @@ -302,7 +304,7 @@ func (ts *TaskStatus) SomeOperationsUnapplied() bool { func (ts *TaskStatus) AppliedTs() Ts { appliedTs := uint64(math.MaxUint64) for _, o := range ts.Operation { - if !o.TableApplied() { + if !o.KeySpanApplied() { if appliedTs > o.BoundaryTs { appliedTs = o.BoundaryTs } @@ -316,16 +318,15 @@ func (ts *TaskStatus) Snapshot(cfID ChangeFeedID, captureID CaptureID, checkpoin snap := &ProcInfoSnap{ CfID: cfID, CaptureID: captureID, - Tables: make(map[TableID]*TableReplicaInfo, len(ts.Tables)), + KeySpans: make(map[KeySpanID]*KeySpanReplicaInfo, len(ts.KeySpans)), } - for tableID, table := range ts.Tables { + for keyspanID, keyspan := range ts.KeySpans { ts := checkpointTs - if ts < table.StartTs { - ts = table.StartTs + if ts < keyspan.StartTs { + ts = keyspan.StartTs } - snap.Tables[tableID] = &TableReplicaInfo{ - StartTs: ts, - MarkTableID: table.MarkTableID, + snap.KeySpans[keyspanID] = &KeySpanReplicaInfo{ + StartTs: ts, } } return snap @@ -347,14 +348,14 @@ func (ts *TaskStatus) Unmarshal(data []byte) error { // Clone returns a deep-clone of the struct func (ts *TaskStatus) Clone() *TaskStatus { clone := *ts - tables := make(map[TableID]*TableReplicaInfo, len(ts.Tables)) - for tableID, table := range ts.Tables { - tables[tableID] = table.Clone() + keyspans := make(map[KeySpanID]*KeySpanReplicaInfo, len(ts.KeySpans)) + for keyspanID, keyspan := range ts.KeySpans { + keyspans[keyspanID] = keyspan.Clone() } - clone.Tables = tables - operation := make(map[TableID]*TableOperation, len(ts.Operation)) - for tableID, opt := range ts.Operation { - operation[tableID] = opt.Clone() + clone.KeySpans = keyspans + operation := make(map[KeySpanID]*KeySpanOperation, len(ts.Operation)) + for keyspanID, opt := range ts.Operation { + operation[keyspanID] = opt.Clone() } clone.Operation = operation return &clone @@ -366,8 +367,8 @@ type CaptureID = string // ChangeFeedID is the type for change feed ID type ChangeFeedID = string -// TableID is the ID of the table -type TableID = int64 +// KeySpanID is the ID of the KeySpan +type KeySpanID = uint64 // SchemaID is the ID of the schema type SchemaID = int64 @@ -443,7 +444,7 @@ func (status *ChangeFeedStatus) Unmarshal(data []byte) error { // ProcInfoSnap holds most important replication information of a processor type ProcInfoSnap struct { - CfID string `json:"changefeed-id"` - CaptureID string `json:"capture-id"` - Tables map[TableID]*TableReplicaInfo `json:"-"` + CfID string `json:"changefeed-id"` + CaptureID string `json:"capture-id"` + KeySpans map[KeySpanID]*KeySpanReplicaInfo `json:"-"` } diff --git a/cdc/cdc/model/owner_test.go b/cdc/cdc/model/owner_test.go index 24b2f0b6..2656eb89 100644 --- a/cdc/cdc/model/owner_test.go +++ b/cdc/cdc/model/owner_test.go @@ -102,7 +102,7 @@ func TestChangeFeedStatusMarshal(t *testing.T) { require.Equal(t, status, newStatus) } -func TestTableOperationState(t *testing.T) { +func TestKeySpanOperationState(t *testing.T) { t.Parallel() processedMap := map[uint64]bool{ @@ -115,20 +115,20 @@ func TestTableOperationState(t *testing.T) { OperProcessed: false, OperFinished: true, } - o := &TableOperation{} + o := &KeySpanOperation{} for status, processed := range processedMap { o.Status = status - require.Equal(t, processed, o.TableProcessed()) + require.Equal(t, processed, o.KeySpanProcessed()) } for status, applied := range appliedMap { o.Status = status - require.Equal(t, applied, o.TableApplied()) + require.Equal(t, applied, o.KeySpanApplied()) } // test clone nil operation. no-nil clone will be tested in `TestShouldBeDeepCopy` - var nilTableOper *TableOperation - require.Nil(t, nilTableOper.Clone()) + var nilKeySpanOper *KeySpanOperation + require.Nil(t, nilKeySpanOper.Clone()) } func TestTaskWorkloadMarshal(t *testing.T) { @@ -164,13 +164,13 @@ func TestShouldBeDeepCopy(t *testing.T) { info := TaskStatus{ - Tables: map[TableID]*TableReplicaInfo{ + KeySpans: map[KeySpanID]*KeySpanReplicaInfo{ 1: {StartTs: 100}, 2: {StartTs: 100}, 3: {StartTs: 100}, 4: {StartTs: 100}, }, - Operation: map[TableID]*TableOperation{ + Operation: map[KeySpanID]*KeySpanOperation{ 5: { Delete: true, BoundaryTs: 6, }, @@ -183,13 +183,13 @@ func TestShouldBeDeepCopy(t *testing.T) { clone := info.Clone() assertIsSnapshot := func() { - require.Equal(t, map[TableID]*TableReplicaInfo{ + require.Equal(t, map[KeySpanID]*KeySpanReplicaInfo{ 1: {StartTs: 100}, 2: {StartTs: 100}, 3: {StartTs: 100}, 4: {StartTs: 100}, - }, clone.Tables) - require.Equal(t, map[TableID]*TableOperation{ + }, clone.KeySpans) + require.Equal(t, map[KeySpanID]*KeySpanOperation{ 5: { Delete: true, BoundaryTs: 6, }, @@ -202,11 +202,11 @@ func TestShouldBeDeepCopy(t *testing.T) { assertIsSnapshot() - info.Tables[7] = &TableReplicaInfo{StartTs: 100} - info.Operation[7] = &TableOperation{Delete: true, BoundaryTs: 7} + info.KeySpans[7] = &KeySpanReplicaInfo{StartTs: 100} + info.Operation[7] = &KeySpanOperation{Delete: true, BoundaryTs: 7} info.Operation[5].BoundaryTs = 8 - info.Tables[1].StartTs = 200 + info.KeySpans[1].StartTs = 200 assertIsSnapshot() } @@ -215,7 +215,7 @@ func TestProcSnapshot(t *testing.T) { t.Parallel() info := TaskStatus{ - Tables: map[TableID]*TableReplicaInfo{ + KeySpans: map[KeySpanID]*KeySpanReplicaInfo{ 10: {StartTs: 100}, }, } @@ -224,19 +224,19 @@ func TestProcSnapshot(t *testing.T) { snap := info.Snapshot(cfID, captureID, 200) require.Equal(t, cfID, snap.CfID) require.Equal(t, captureID, snap.CaptureID) - require.Equal(t, 1, len(snap.Tables)) - require.Equal(t, &TableReplicaInfo{StartTs: 200}, snap.Tables[10]) + require.Equal(t, 1, len(snap.KeySpans)) + require.Equal(t, &KeySpanReplicaInfo{StartTs: 200}, snap.KeySpans[10]) } func TestTaskStatusMarshal(t *testing.T) { t.Parallel() status := &TaskStatus{ - Tables: map[TableID]*TableReplicaInfo{ + KeySpans: map[KeySpanID]*KeySpanReplicaInfo{ 1: {StartTs: 420875942036766723}, }, } - expected := `{"tables":{"1":{"start-ts":420875942036766723,"mark-table-id":0}},"operation":null,"admin-job-type":0}` + expected := `{"keyspans":{"1":{"start-ts":420875942036766723,"Start":null,"End":null}},"operation":null,"admin-job-type":0}` data, err := status.Marshal() require.Nil(t, err) @@ -249,15 +249,15 @@ func TestTaskStatusMarshal(t *testing.T) { require.Equal(t, status, newStatus) } -func TestAddTable(t *testing.T) { +func TestAddKeySpan(t *testing.T) { t.Parallel() ts := uint64(420875942036766723) expected := &TaskStatus{ - Tables: map[TableID]*TableReplicaInfo{ + KeySpans: map[KeySpanID]*KeySpanReplicaInfo{ 1: {StartTs: ts}, }, - Operation: map[TableID]*TableOperation{ + Operation: map[KeySpanID]*KeySpanOperation{ 1: { BoundaryTs: ts, Status: OperDispatched, @@ -265,11 +265,11 @@ func TestAddTable(t *testing.T) { }, } status := &TaskStatus{} - status.AddTable(1, &TableReplicaInfo{StartTs: ts}, ts) + status.AddKeySpan(1, &KeySpanReplicaInfo{StartTs: ts}, ts) require.Equal(t, expected, status) - // add existing table does nothing - status.AddTable(1, &TableReplicaInfo{StartTs: 1}, 1) + // add existing keyspan does nothing + status.AddKeySpan(1, &KeySpanReplicaInfo{StartTs: 1}, 1) require.Equal(t, expected, status) } @@ -279,8 +279,8 @@ func TestTaskStatusApplyState(t *testing.T) { ts1 := uint64(420875042036766723) ts2 := uint64(420876783269969921) status := &TaskStatus{} - status.AddTable(1, &TableReplicaInfo{StartTs: ts1}, ts1) - status.AddTable(2, &TableReplicaInfo{StartTs: ts2}, ts2) + status.AddKeySpan(1, &KeySpanReplicaInfo{StartTs: ts1}, ts1) + status.AddKeySpan(2, &KeySpanReplicaInfo{StartTs: ts2}, ts2) require.True(t, status.SomeOperationsUnapplied()) require.Equal(t, ts1, status.AppliedTs()) @@ -290,24 +290,24 @@ func TestTaskStatusApplyState(t *testing.T) { require.Equal(t, uint64(math.MaxUint64), status.AppliedTs()) } -func TestMoveTable(t *testing.T) { +func TestMoveKeySpan(t *testing.T) { t.Parallel() info := TaskStatus{ - Tables: map[TableID]*TableReplicaInfo{ + KeySpans: map[KeySpanID]*KeySpanReplicaInfo{ 1: {StartTs: 100}, 2: {StartTs: 200}, }, } - replicaInfo, found := info.RemoveTable(2, 300, true) + replicaInfo, found := info.RemoveKeySpan(2, 300, true) require.True(t, found) - require.Equal(t, &TableReplicaInfo{StartTs: 200}, replicaInfo) - require.NotNil(t, info.Tables[int64(1)]) - require.Nil(t, info.Tables[int64(2)]) - expectedFlag := uint64(1) // OperFlagMoveTable - require.Equal(t, map[int64]*TableOperation{ - 2: { + require.Equal(t, &KeySpanReplicaInfo{StartTs: 200}, replicaInfo) + require.NotNil(t, info.KeySpans[uint64(1)]) + require.Nil(t, info.KeySpans[uint64(2)]) + expectedFlag := uint64(1) // OperFlagMoveKeySpan + require.Equal(t, map[uint64]*KeySpanOperation{ + uint64(2): { Delete: true, Flag: expectedFlag, BoundaryTs: 300, @@ -316,11 +316,11 @@ func TestMoveTable(t *testing.T) { }, info.Operation) } -func TestShouldReturnRemovedTable(t *testing.T) { +func TestShouldReturnRemovedKeySpan(t *testing.T) { t.Parallel() info := TaskStatus{ - Tables: map[TableID]*TableReplicaInfo{ + KeySpans: map[KeySpanID]*KeySpanReplicaInfo{ 1: {StartTs: 100}, 2: {StartTs: 200}, 3: {StartTs: 300}, @@ -328,23 +328,23 @@ func TestShouldReturnRemovedTable(t *testing.T) { }, } - replicaInfo, found := info.RemoveTable(2, 666, false) + replicaInfo, found := info.RemoveKeySpan(2, 666, false) require.True(t, found) - require.Equal(t, &TableReplicaInfo{StartTs: 200}, replicaInfo) + require.Equal(t, &KeySpanReplicaInfo{StartTs: 200}, replicaInfo) } -func TestShouldHandleTableNotFound(t *testing.T) { +func TestShouldHandleKeySpanNotFound(t *testing.T) { t.Parallel() info := TaskStatus{} - _, found := info.RemoveTable(404, 666, false) + _, found := info.RemoveKeySpan(404, 666, false) require.False(t, found) info = TaskStatus{ - Tables: map[TableID]*TableReplicaInfo{ + KeySpans: map[KeySpanID]*KeySpanReplicaInfo{ 1: {StartTs: 100}, }, } - _, found = info.RemoveTable(404, 666, false) + _, found = info.RemoveKeySpan(404, 666, false) require.False(t, found) } diff --git a/cdc/cdc/model/protocol.go b/cdc/cdc/model/protocol.go index 5c32dfd1..66795148 100644 --- a/cdc/cdc/model/protocol.go +++ b/cdc/cdc/model/protocol.go @@ -24,27 +24,29 @@ import ( // This file contains a communication protocol between the Owner and the Processor. // FIXME a detailed documentation on the interaction will be added later in a separate file. -// DispatchTableTopic returns a message topic for dispatching a table. -func DispatchTableTopic(changefeedID ChangeFeedID) p2p.Topic { +// DispatchKeySpanTopic returns a message topic for dispatching a keyspan. +func DispatchKeySpanTopic(changefeedID ChangeFeedID) p2p.Topic { return fmt.Sprintf("dispatch/%s", changefeedID) } -// DispatchTableMessage is the message body for dispatching a table. -type DispatchTableMessage struct { - OwnerRev int64 `json:"owner-rev"` - ID TableID `json:"id"` - IsDelete bool `json:"is-delete"` +// DispatchKeySpanMessage is the message body for dispatching a keyspan. +type DispatchKeySpanMessage struct { + OwnerRev int64 `json:"owner-rev"` + ID KeySpanID `json:"id"` + IsDelete bool `json:"is-delete"` + Start []byte `json:"start"` + End []byte `json:"end"` } -// DispatchTableResponseTopic returns a message topic for the result of -// dispatching a table. It is sent from the Processor to the Owner. -func DispatchTableResponseTopic(changefeedID ChangeFeedID) p2p.Topic { +// DispatchKeySpanResponseTopic returns a message topic for the result of +// dispatching a keyspan. It is sent from the Processor to the Owner. +func DispatchKeySpanResponseTopic(changefeedID ChangeFeedID) p2p.Topic { return fmt.Sprintf("dispatch-resp/%s", changefeedID) } -// DispatchTableResponseMessage is the message body for the result of dispatching a table. -type DispatchTableResponseMessage struct { - ID TableID `json:"id"` +// DispatchKeySpanResponseMessage is the message body for the result of dispatching a keyspan. +type DispatchKeySpanResponseMessage struct { + ID KeySpanID `json:"id"` } // AnnounceTopic returns a message topic for announcing an ownership change. @@ -69,9 +71,9 @@ func SyncTopic(changefeedID ChangeFeedID) p2p.Topic { type SyncMessage struct { // Sends the processor's version for compatibility check ProcessorVersion string - Running []TableID - Adding []TableID - Removing []TableID + Running []KeySpanID + Adding []KeySpanID + Removing []KeySpanID } // Marshal serializes the message into MsgPack format. diff --git a/cdc/cdc/model/protocol_test.go b/cdc/cdc/model/protocol_test.go index ceccdf17..977f0bfe 100644 --- a/cdc/cdc/model/protocol_test.go +++ b/cdc/cdc/model/protocol_test.go @@ -50,9 +50,9 @@ func TestSerializeSyncMessage(t *testing.T) { } func makeVeryLargeSyncMessage() *SyncMessage { - largeSliceFn := func() (ret []TableID) { + largeSliceFn := func() (ret []KeySpanID) { for i := 0; i < 80000; i++ { - ret = append(ret, TableID(i)) + ret = append(ret, KeySpanID(i)) } return } @@ -63,20 +63,20 @@ func makeVeryLargeSyncMessage() *SyncMessage { } } -func TestMarshalDispatchTableMessage(t *testing.T) { - msg := &DispatchTableMessage{ +func TestMarshalDispatchKeySpanMessage(t *testing.T) { + msg := &DispatchKeySpanMessage{ OwnerRev: 1, - ID: TableID(1), + ID: KeySpanID(1), IsDelete: true, } bytes, err := json.Marshal(msg) require.NoError(t, err) - require.Equal(t, `{"owner-rev":1,"id":1,"is-delete":true}`, string(bytes)) + require.Equal(t, `{"owner-rev":1,"id":1,"is-delete":true,"start":null,"end":null}`, string(bytes)) } -func TestMarshalDispatchTableResponseMessage(t *testing.T) { - msg := &DispatchTableResponseMessage{ - ID: TableID(1), +func TestMarshalDispatchKeySpanResponseMessage(t *testing.T) { + msg := &DispatchKeySpanResponseMessage{ + ID: KeySpanID(1), } bytes, err := json.Marshal(msg) require.NoError(t, err) diff --git a/cdc/cdc/model/string_test.go b/cdc/cdc/model/string_test.go index 6cc1d83f..41575caa 100644 --- a/cdc/cdc/model/string_test.go +++ b/cdc/cdc/model/string_test.go @@ -47,10 +47,10 @@ func TestExtractKeySuffix(t *testing.T) { expect string hasErr bool }{ - {"/tidb/cdc/capture/info/6a6c6dd290bc8732", "6a6c6dd290bc8732", false}, - {"/tidb/cdc/capture/info/6a6c6dd290bc8732/", "", false}, - {"/tidb/cdc", "cdc", false}, - {"/tidb", "tidb", false}, + {"/tikv/cdc/capture/info/6a6c6dd290bc8732", "6a6c6dd290bc8732", false}, + {"/tikv/cdc/capture/info/6a6c6dd290bc8732/", "", false}, + {"/tikv/cdc", "cdc", false}, + {"/tikv", "tikv", false}, {"", "", true}, } for _, tc := range testCases { diff --git a/cdc/cdc/owner/barrier.go b/cdc/cdc/owner/barrier.go deleted file mode 100644 index 08db928c..00000000 --- a/cdc/cdc/owner/barrier.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package owner - -import ( - "math" - - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" -) - -type barrierType int - -const ( - // ddlJobBarrier denotes a replication barrier caused by a DDL. - ddlJobBarrier barrierType = iota - // syncPointBarrier denotes a barrier for snapshot replication. - syncPointBarrier - // finishBarrier denotes a barrier for changefeed finished. - finishBarrier -) - -// barriers stores some barrierType and barrierTs, and can calculate the min barrierTs -// barriers is NOT-THREAD-SAFE -type barriers struct { - inner map[barrierType]model.Ts - dirty bool - min barrierType -} - -func newBarriers() *barriers { - return &barriers{ - inner: make(map[barrierType]model.Ts), - dirty: true, - } -} - -func (b *barriers) Update(tp barrierType, barrierTs model.Ts) { - // the barriers structure was given the ability to handle a fallback barrierTs by design. - // but the barrierTs should never fallback in owner replication model - if !b.dirty && (tp == b.min || barrierTs <= b.inner[b.min]) { - b.dirty = true - } - b.inner[tp] = barrierTs -} - -func (b *barriers) Min() (tp barrierType, barrierTs model.Ts) { - if !b.dirty { - return b.min, b.inner[b.min] - } - tp, minTs := b.calcMin() - b.min = tp - b.dirty = false - return tp, minTs -} - -func (b *barriers) calcMin() (tp barrierType, barrierTs model.Ts) { - barrierTs = uint64(math.MaxUint64) - for br, ts := range b.inner { - if ts <= barrierTs { - tp = br - barrierTs = ts - } - } - if barrierTs == math.MaxUint64 { - log.Panic("the barriers is empty, please report a bug") - } - return -} - -func (b *barriers) Remove(tp barrierType) { - delete(b.inner, tp) - b.dirty = true -} diff --git a/cdc/cdc/owner/barrier_test.go b/cdc/cdc/owner/barrier_test.go deleted file mode 100644 index 73aff06c..00000000 --- a/cdc/cdc/owner/barrier_test.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package owner - -import ( - "math" - "math/rand" - "testing" - - "github.com/pingcap/check" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/util/testleak" -) - -func Test(t *testing.T) { check.TestingT(t) } - -var _ = check.Suite(&barrierSuite{}) - -type barrierSuite struct{} - -func (s *barrierSuite) TestBarrier(c *check.C) { - defer testleak.AfterTest(c)() - b := newBarriers() - b.Update(ddlJobBarrier, 2) - b.Update(syncPointBarrier, 3) - b.Update(finishBarrier, 1) - tp, ts := b.Min() - c.Assert(tp, check.Equals, finishBarrier) - c.Assert(ts, check.Equals, uint64(1)) - - b.Update(finishBarrier, 4) - tp, ts = b.Min() - c.Assert(tp, check.Equals, ddlJobBarrier) - c.Assert(ts, check.Equals, uint64(2)) - - b.Remove(ddlJobBarrier) - tp, ts = b.Min() - c.Assert(tp, check.Equals, syncPointBarrier) - c.Assert(ts, check.Equals, uint64(3)) - - b.Update(finishBarrier, 1) - tp, ts = b.Min() - c.Assert(tp, check.Equals, finishBarrier) - c.Assert(ts, check.Equals, uint64(1)) - - b.Update(ddlJobBarrier, 5) - tp, ts = b.Min() - c.Assert(tp, check.Equals, finishBarrier) - c.Assert(ts, check.Equals, uint64(1)) -} - -func (s *barrierSuite) TestBarrierRandom(c *check.C) { - defer testleak.AfterTest(c)() - maxBarrierType := 50 - maxBarrierTs := 1000000 - b := newBarriers() - expectedBarriers := make(map[barrierType]model.Ts) - - // set a barrier which can not be removed to avoid the barrier map is empty - b.Update(barrierType(maxBarrierType), model.Ts(maxBarrierTs)) - expectedBarriers[barrierType(maxBarrierType)] = model.Ts(maxBarrierTs) - - for i := 0; i < 100000; i++ { - switch rand.Intn(2) { - case 0: - tp := barrierType(rand.Intn(maxBarrierType)) - ts := model.Ts(rand.Intn(maxBarrierTs)) - b.Update(tp, ts) - expectedBarriers[tp] = ts - case 1: - tp := barrierType(rand.Intn(maxBarrierType)) - b.Remove(tp) - delete(expectedBarriers, tp) - } - expectedMinTs := uint64(math.MaxUint64) - for _, ts := range expectedBarriers { - if ts < expectedMinTs { - expectedMinTs = ts - } - } - tp, ts := b.Min() - c.Assert(ts, check.Equals, expectedMinTs) - c.Assert(expectedBarriers[tp], check.Equals, expectedMinTs) - } -} diff --git a/cdc/cdc/owner/changefeed.go b/cdc/cdc/owner/changefeed.go index 490f5b70..7ed144e5 100644 --- a/cdc/cdc/owner/changefeed.go +++ b/cdc/cdc/owner/changefeed.go @@ -15,19 +15,14 @@ package owner import ( "context" - "strings" "sync" "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/log" - "github.com/pingcap/tidb/parser" - "github.com/pingcap/tidb/parser/format" - timodel "github.com/pingcap/tidb/parser/model" "github.com/prometheus/client_golang/prometheus" "github.com/tikv/client-go/v2/oracle" "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/redo" schedulerv2 "github.com/tikv/migration/cdc/cdc/scheduler" cdcContext "github.com/tikv/migration/cdc/pkg/context" cerror "github.com/tikv/migration/cdc/pkg/errors" @@ -37,19 +32,25 @@ import ( "go.uber.org/zap" ) +const ( + defaultErrChSize = 1024 +) + type changefeed struct { id model.ChangeFeedID state *orchestrator.ChangefeedReactorState - scheduler scheduler - barriers *barriers + scheduler scheduler + // barriers *barriers feedStateManager *feedStateManager gcManager gc.Manager - redoManager redo.LogManager - schema *schemaWrap4Owner - sink DDLSink - ddlPuller DDLPuller + // TODO: Can we delete redoManager for tikv cdc? + // redoManager redo.LogManager + + // schema *schemaWrap4Owner + // sink DDLSink + // ddlPuller DDLPuller initialized bool // isRemoved is true if the changefeed is removed isRemoved bool @@ -57,7 +58,8 @@ type changefeed struct { // only used for asyncExecDDL function // ddlEventCache is not nil when the changefeed is executing a DDL event asynchronously // After the DDL event has been executed, ddlEventCache will be set to nil. - ddlEventCache *model.DDLEvent + + // ddlEventCache *model.DDLEvent errCh chan error // cancel the running goroutine start by `DDLPuller` @@ -73,8 +75,8 @@ type changefeed struct { metricsChangefeedResolvedTsGauge prometheus.Gauge metricsChangefeedResolvedTsLagGauge prometheus.Gauge - newDDLPuller func(ctx cdcContext.Context, startTs uint64) (DDLPuller, error) - newSink func() DDLSink + // newDDLPuller func(ctx cdcContext.Context, startTs uint64) (DDLPuller, error) + // newSink func() DDLSink newScheduler func(ctx cdcContext.Context, startTs uint64) (scheduler, error) } @@ -82,29 +84,34 @@ func newChangefeed(id model.ChangeFeedID, gcManager gc.Manager) *changefeed { c := &changefeed{ id: id, // The scheduler will be created lazily. - scheduler: nil, - barriers: newBarriers(), + scheduler: nil, + // barriers: newBarriers(), feedStateManager: new(feedStateManager), gcManager: gcManager, errCh: make(chan error, defaultErrChSize), cancel: func() {}, - newDDLPuller: newDDLPuller, - newSink: newDDLSink, + // newDDLPuller: newDDLPuller, + // newSink: newDDLSink, } c.newScheduler = newScheduler return c } +// TODO: modify for tikv cdc func newChangefeed4Test( id model.ChangeFeedID, gcManager gc.Manager, - newDDLPuller func(ctx cdcContext.Context, startTs uint64) (DDLPuller, error), - newSink func() DDLSink, + /* + newDDLPuller func(ctx cdcContext.Context, startTs uint64) (DDLPuller, error), + newSink func() DDLSink, + */ ) *changefeed { c := newChangefeed(id, gcManager) - c.newDDLPuller = newDDLPuller - c.newSink = newSink + /* + c.newDDLPuller = newDDLPuller + c.newSink = newSink + */ return c } @@ -174,18 +181,7 @@ func (c *changefeed) tick(ctx cdcContext.Context, state *orchestrator.Changefeed default: } - c.sink.emitCheckpointTs(ctx, checkpointTs) - barrierTs, err := c.handleBarrier(ctx) - if err != nil { - return errors.Trace(err) - } - if barrierTs < checkpointTs { - // This condition implies that the DDL resolved-ts has not yet reached checkpointTs, - // which implies that it would be premature to schedule tables or to update status. - // So we return here. - return nil - } - newCheckpointTs, newResolvedTs, err := c.scheduler.Tick(ctx, c.state, c.schema.AllPhysicalTables(), captures) + newCheckpointTs, newResolvedTs, err := c.scheduler.Tick(ctx, c.state, captures) if err != nil { return errors.Trace(err) } @@ -194,12 +190,6 @@ func (c *changefeed) tick(ctx cdcContext.Context, state *orchestrator.Changefeed if newCheckpointTs != schedulerv2.CheckpointCannotProceed { pdTime, _ := ctx.GlobalVars().TimeAcquirer.CurrentTimeFromCached() currentTs := oracle.GetPhysical(pdTime) - if newResolvedTs > barrierTs { - newResolvedTs = barrierTs - } - if newCheckpointTs > barrierTs { - newCheckpointTs = barrierTs - } c.updateStatus(currentTs, newCheckpointTs, newResolvedTs) } return nil @@ -248,48 +238,62 @@ LOOP: return errors.Trace(err) } } - if c.state.Info.SyncPointEnabled { - c.barriers.Update(syncPointBarrier, checkpointTs) - } + + /* + if c.state.Info.SyncPointEnabled { + c.barriers.Update(syncPointBarrier, checkpointTs) + } + */ + // Since we are starting DDL puller from (checkpointTs-1) to make // the DDL committed at checkpointTs executable by CDC, we need to set // the DDL barrier to the correct start point. - c.barriers.Update(ddlJobBarrier, checkpointTs-1) - c.barriers.Update(finishBarrier, c.state.Info.GetTargetTs()) + + /* + c.barriers.Update(ddlJobBarrier, checkpointTs-1) + c.barriers.Update(finishBarrier, c.state.Info.GetTargetTs()) + */ + var err error // Note that (checkpointTs == ddl.FinishedTs) DOES NOT imply that the DDL has been completed executed. // So we need to process all DDLs from the range [checkpointTs, ...), but since the semantics of start-ts requires // the lower bound of an open interval, i.e. (startTs, ...), we pass checkpointTs-1 as the start-ts to initialize // the schema cache. - c.schema, err = newSchemaWrap4Owner(ctx.GlobalVars().KVStorage, checkpointTs-1, c.state.Info.Config) - if err != nil { - return errors.Trace(err) - } - - cancelCtx, cancel := cdcContext.WithCancel(ctx) - c.cancel = cancel + /* + c.schema, err = newSchemaWrap4Owner(ctx.GlobalVars().KVStorage, checkpointTs-1, c.state.Info.Config) + if err != nil { + return errors.Trace(err) + } + */ - c.sink = c.newSink() - c.sink.run(cancelCtx, cancelCtx.ChangefeedVars().ID, cancelCtx.ChangefeedVars().Info) + /* + cancelCtx, cancel := cdcContext.WithCancel(ctx) + c.cancel = cancel + c.sink = c.newSink() + c.sink.run(cancelCtx, cancelCtx.ChangefeedVars().ID, cancelCtx.ChangefeedVars().Info) + */ // Refer to the previous comment on why we use (checkpointTs-1). - c.ddlPuller, err = c.newDDLPuller(cancelCtx, checkpointTs-1) - if err != nil { - return errors.Trace(err) - } - c.wg.Add(1) - go func() { - defer c.wg.Done() - ctx.Throw(c.ddlPuller.Run(cancelCtx)) - }() - stdCtx := util.PutChangefeedIDInCtx(cancelCtx, c.id) - redoManagerOpts := &redo.ManagerOptions{EnableBgRunner: false} - redoManager, err := redo.NewManager(stdCtx, c.state.Info.Config.Consistent, redoManagerOpts) - if err != nil { - return err - } - c.redoManager = redoManager + /* + c.ddlPuller, err = c.newDDLPuller(cancelCtx, checkpointTs-1) + if err != nil { + return errors.Trace(err) + } + c.wg.Add(1) + go func() { + defer c.wg.Done() + ctx.Throw(c.ddlPuller.Run(cancelCtx)) + }() + + stdCtx := util.PutChangefeedIDInCtx(cancelCtx, c.id) + redoManagerOpts := &redo.ManagerOptions{EnableBgRunner: false} + redoManager, err := redo.NewManager(stdCtx, c.state.Info.Config.Consistent, redoManagerOpts) + if err != nil { + return err + } + c.redoManager = redoManager + */ // init metrics c.metricsChangefeedCheckpointTsGauge = changefeedCheckpointTsGauge.WithLabelValues(c.id) @@ -309,22 +313,27 @@ LOOP: func (c *changefeed) releaseResources(ctx cdcContext.Context) { if !c.initialized { - c.redoManagerCleanup(ctx) + // c.redoManagerCleanup(ctx) return } log.Info("close changefeed", zap.String("changefeed", c.state.ID), zap.Stringer("info", c.state.Info), zap.Bool("isRemoved", c.isRemoved)) c.cancel() c.cancel = func() {} - c.ddlPuller.Close() - c.schema = nil - c.redoManagerCleanup(ctx) - canceledCtx, cancel := context.WithCancel(context.Background()) + /* + c.schema = nil + c.ddlPuller.Close() + c.redoManagerCleanup(ctx) + */ + _, cancel := context.WithCancel(context.Background()) cancel() // We don't need to wait sink Close, pass a canceled context is ok - if err := c.sink.close(canceledCtx); err != nil { - log.Warn("Closing sink failed in Owner", zap.String("changefeedID", c.state.ID), zap.Error(err)) - } + + /* + if err := c.sink.close(canceledCtx); err != nil { + log.Warn("Closing sink failed in Owner", zap.String("changefeedID", c.state.ID), zap.Error(err)) + } + */ c.wg.Wait() c.scheduler.Close(ctx) @@ -341,6 +350,7 @@ func (c *changefeed) releaseResources(ctx cdcContext.Context) { c.initialized = false } +/* // redoManagerCleanup cleanups redo logs if changefeed is removed and redo log is enabled func (c *changefeed) redoManagerCleanup(ctx context.Context) { if c.isRemoved { @@ -368,6 +378,7 @@ func (c *changefeed) redoManagerCleanup(ctx context.Context) { } } } +*/ // preflightCheck makes sure that the metadata in Etcd is complete enough to run the tick. // If the metadata is not complete, such as when the ChangeFeedStatus is nil, @@ -429,6 +440,7 @@ func (c *changefeed) preflightCheck(captures map[model.CaptureID]*model.CaptureI return } +/* func (c *changefeed) handleBarrier(ctx cdcContext.Context) (uint64, error) { barrierTp, barrierTs := c.barriers.Min() blocked := (barrierTs == c.state.Status.CheckpointTs) && (barrierTs == c.state.Status.ResolvedTs) @@ -521,6 +533,7 @@ func (c *changefeed) asyncExecDDL(ctx cdcContext.Context, job *timodel.Job) (don } return done, nil } +*/ func (c *changefeed) updateStatus(currentTs int64, checkpointTs, resolvedTs model.Ts) { c.state.PatchStatus(func(status *model.ChangeFeedStatus) (*model.ChangeFeedStatus, bool, error) { @@ -558,7 +571,9 @@ func (c *changefeed) GetInfoProvider() schedulerv2.InfoProvider { return nil } +/* // addSpecialComment translate tidb feature to comment +// TODO: Maybe need delete this codes for tikv cdc. func addSpecialComment(ddlQuery string) (string, error) { stms, _, err := parser.New().ParseSQL(ddlQuery) if err != nil { @@ -581,3 +596,4 @@ func addSpecialComment(ddlQuery string) (string, error) { } return sb.String(), nil } +*/ diff --git a/cdc/cdc/owner/changefeed_test.go b/cdc/cdc/owner/changefeed_test.go index 0e3cb072..290cd89c 100644 --- a/cdc/cdc/owner/changefeed_test.go +++ b/cdc/cdc/owner/changefeed_test.go @@ -17,101 +17,17 @@ import ( "context" "os" "path/filepath" - "sync" - "sync/atomic" - "time" "github.com/pingcap/check" "github.com/pingcap/errors" - timodel "github.com/pingcap/tidb/parser/model" - "github.com/tikv/client-go/v2/oracle" - "github.com/tikv/migration/cdc/cdc/entry" "github.com/tikv/migration/cdc/cdc/model" "github.com/tikv/migration/cdc/pkg/config" cdcContext "github.com/tikv/migration/cdc/pkg/context" "github.com/tikv/migration/cdc/pkg/orchestrator" - "github.com/tikv/migration/cdc/pkg/pdtime" "github.com/tikv/migration/cdc/pkg/txnutil/gc" "github.com/tikv/migration/cdc/pkg/util/testleak" - "github.com/tikv/migration/cdc/pkg/version" ) -type mockDDLPuller struct { - // DDLPuller - resolvedTs model.Ts - ddlQueue []*timodel.Job -} - -func (m *mockDDLPuller) FrontDDL() (uint64, *timodel.Job) { - if len(m.ddlQueue) > 0 { - return m.ddlQueue[0].BinlogInfo.FinishedTS, m.ddlQueue[0] - } - return m.resolvedTs, nil -} - -func (m *mockDDLPuller) PopFrontDDL() (uint64, *timodel.Job) { - if len(m.ddlQueue) > 0 { - job := m.ddlQueue[0] - m.ddlQueue = m.ddlQueue[1:] - return job.BinlogInfo.FinishedTS, job - } - return m.resolvedTs, nil -} - -func (m *mockDDLPuller) Close() {} - -func (m *mockDDLPuller) Run(ctx cdcContext.Context) error { - <-ctx.Done() - return nil -} - -type mockDDLSink struct { - // DDLSink - ddlExecuting *model.DDLEvent - ddlDone bool - checkpointTs model.Ts - syncPoint model.Ts - syncPointHis []model.Ts - - wg sync.WaitGroup -} - -func (m *mockDDLSink) run(ctx cdcContext.Context, _ model.ChangeFeedID, _ *model.ChangeFeedInfo) { - m.wg.Add(1) - go func() { - <-ctx.Done() - m.wg.Done() - }() -} - -func (m *mockDDLSink) emitDDLEvent(ctx cdcContext.Context, ddl *model.DDLEvent) (bool, error) { - m.ddlExecuting = ddl - defer func() { m.ddlDone = false }() - return m.ddlDone, nil -} - -func (m *mockDDLSink) emitSyncPoint(ctx cdcContext.Context, checkpointTs uint64) error { - if checkpointTs == m.syncPoint { - return nil - } - m.syncPoint = checkpointTs - m.syncPointHis = append(m.syncPointHis, checkpointTs) - return nil -} - -func (m *mockDDLSink) emitCheckpointTs(ctx cdcContext.Context, ts uint64) { - atomic.StoreUint64(&m.checkpointTs, ts) -} - -func (m *mockDDLSink) close(ctx context.Context) error { - m.wg.Wait() - return nil -} - -func (m *mockDDLSink) Barrier(ctx context.Context) error { - return nil -} - var _ = check.Suite(&changefeedSuite{}) type changefeedSuite struct{} @@ -124,11 +40,7 @@ func createChangefeed4Test(ctx cdcContext.Context, c *check.C) (*changefeed, *or }, } gcManager := gc.NewManager(ctx.GlobalVars().PDClient) - cf := newChangefeed4Test(ctx.ChangefeedVars().ID, gcManager, func(ctx cdcContext.Context, startTs uint64) (DDLPuller, error) { - return &mockDDLPuller{resolvedTs: startTs - 1}, nil - }, func() DDLSink { - return &mockDDLSink{} - }) + cf := newChangefeed4Test(ctx.ChangefeedVars().ID, gcManager) state := orchestrator.NewChangefeedReactorState(ctx.ChangefeedVars().ID) tester := orchestrator.NewReactorStateTester(c, state, nil) state.PatchInfo(func(info *model.ChangeFeedInfo) (*model.ChangeFeedInfo, bool, error) { @@ -136,7 +48,7 @@ func createChangefeed4Test(ctx cdcContext.Context, c *check.C) (*changefeed, *or info = ctx.ChangefeedVars().Info return info, true, nil }) - tester.MustUpdate("/tidb/cdc/capture/"+ctx.GlobalVars().CaptureInfo.ID, []byte(`{"id":"`+ctx.GlobalVars().CaptureInfo.ID+`","address":"127.0.0.1:8300"}`)) + tester.MustUpdate("/tikv/cdc/capture/"+ctx.GlobalVars().CaptureInfo.ID, []byte(`{"id":"`+ctx.GlobalVars().CaptureInfo.ID+`","address":"127.0.0.1:8300"}`)) tester.MustApplyPatches() captures := map[model.CaptureID]*model.CaptureInfo{ctx.GlobalVars().CaptureInfo.ID: ctx.GlobalVars().CaptureInfo} return cf, state, captures, tester @@ -209,160 +121,6 @@ func (s *changefeedSuite) TestHandleError(c *check.C) { c.Assert(state.Info.Error.Message, check.Equals, "fake error") } -func (s *changefeedSuite) TestExecDDL(c *check.C) { - defer testleak.AfterTest(c)() - - helper := entry.NewSchemaTestHelper(c) - defer helper.Close() - // Creates a table, which will be deleted at the start-ts of the changefeed. - // It is expected that the changefeed DOES NOT replicate this table. - helper.DDL2Job("create database test0") - job := helper.DDL2Job("create table test0.table0(id int primary key)") - startTs := job.BinlogInfo.FinishedTS + 1000 - - ctx := cdcContext.NewContext(context.Background(), &cdcContext.GlobalVars{ - KVStorage: helper.Storage(), - CaptureInfo: &model.CaptureInfo{ - ID: "capture-id-test", - AdvertiseAddr: "127.0.0.1:0000", - Version: version.ReleaseVersion, - }, - TimeAcquirer: pdtime.NewTimeAcquirer4Test(), - }) - ctx = cdcContext.WithChangefeedVars(ctx, &cdcContext.ChangefeedVars{ - ID: "changefeed-id-test", - Info: &model.ChangeFeedInfo{ - StartTs: startTs, - Config: config.GetDefaultReplicaConfig(), - }, - }) - - cf, state, captures, tester := createChangefeed4Test(ctx, c) - defer cf.Close(ctx) - tickThreeTime := func() { - cf.Tick(ctx, state, captures) - tester.MustApplyPatches() - cf.Tick(ctx, state, captures) - tester.MustApplyPatches() - cf.Tick(ctx, state, captures) - tester.MustApplyPatches() - } - // pre check and initialize - tickThreeTime() - - c.Assert(cf.schema.AllPhysicalTables(), check.HasLen, 1) - c.Assert(state.TaskStatuses[ctx.GlobalVars().CaptureInfo.ID].Operation, check.HasLen, 0) - c.Assert(state.TaskStatuses[ctx.GlobalVars().CaptureInfo.ID].Tables, check.HasLen, 0) - - job = helper.DDL2Job("drop table test0.table0") - // ddl puller resolved ts grow uo - mockDDLPuller := cf.ddlPuller.(*mockDDLPuller) - mockDDLPuller.resolvedTs = startTs - mockDDLSink := cf.sink.(*mockDDLSink) - job.BinlogInfo.FinishedTS = mockDDLPuller.resolvedTs - mockDDLPuller.ddlQueue = append(mockDDLPuller.ddlQueue, job) - // three tick to make sure all barriers set in initialize is handled - tickThreeTime() - c.Assert(state.Status.CheckpointTs, check.Equals, mockDDLPuller.resolvedTs) - // The ephemeral table should have left no trace in the schema cache - c.Assert(cf.schema.AllPhysicalTables(), check.HasLen, 0) - - // executing the ddl finished - mockDDLSink.ddlDone = true - mockDDLPuller.resolvedTs += 1000 - tickThreeTime() - c.Assert(state.Status.CheckpointTs, check.Equals, mockDDLPuller.resolvedTs) - - // handle create database - job = helper.DDL2Job("create database test1") - mockDDLPuller.resolvedTs += 1000 - job.BinlogInfo.FinishedTS = mockDDLPuller.resolvedTs - mockDDLPuller.ddlQueue = append(mockDDLPuller.ddlQueue, job) - tickThreeTime() - c.Assert(state.Status.CheckpointTs, check.Equals, mockDDLPuller.resolvedTs) - c.Assert(mockDDLSink.ddlExecuting.Query, check.Equals, "CREATE DATABASE `test1`") - - // executing the ddl finished - mockDDLSink.ddlDone = true - mockDDLPuller.resolvedTs += 1000 - tickThreeTime() - c.Assert(state.Status.CheckpointTs, check.Equals, mockDDLPuller.resolvedTs) - - // handle create table - job = helper.DDL2Job("create table test1.test1(id int primary key)") - mockDDLPuller.resolvedTs += 1000 - job.BinlogInfo.FinishedTS = mockDDLPuller.resolvedTs - mockDDLPuller.ddlQueue = append(mockDDLPuller.ddlQueue, job) - tickThreeTime() - c.Assert(state.Status.CheckpointTs, check.Equals, mockDDLPuller.resolvedTs) - c.Assert(mockDDLSink.ddlExecuting.Query, check.Equals, "CREATE TABLE `test1`.`test1` (`id` INT PRIMARY KEY)") - - // executing the ddl finished - mockDDLSink.ddlDone = true - mockDDLPuller.resolvedTs += 1000 - tickThreeTime() - c.Assert(state.TaskStatuses[ctx.GlobalVars().CaptureInfo.ID].Tables, check.HasKey, job.TableID) -} - -func (s *changefeedSuite) TestSyncPoint(c *check.C) { - defer testleak.AfterTest(c)() - ctx := cdcContext.NewBackendContext4Test(true) - ctx.ChangefeedVars().Info.SyncPointEnabled = true - ctx.ChangefeedVars().Info.SyncPointInterval = 1 * time.Second - cf, state, captures, tester := createChangefeed4Test(ctx, c) - defer cf.Close(ctx) - - // pre check - cf.Tick(ctx, state, captures) - tester.MustApplyPatches() - - // initialize - cf.Tick(ctx, state, captures) - tester.MustApplyPatches() - - mockDDLPuller := cf.ddlPuller.(*mockDDLPuller) - mockDDLSink := cf.sink.(*mockDDLSink) - // add 5s to resolvedTs - mockDDLPuller.resolvedTs = oracle.GoTimeToTS(oracle.GetTimeFromTS(mockDDLPuller.resolvedTs).Add(5 * time.Second)) - // tick 20 times - for i := 0; i <= 20; i++ { - cf.Tick(ctx, state, captures) - tester.MustApplyPatches() - } - for i := 1; i < len(mockDDLSink.syncPointHis); i++ { - // check the time interval between adjacent sync points is less or equal than one second - c.Assert(mockDDLSink.syncPointHis[i]-mockDDLSink.syncPointHis[i-1], check.LessEqual, uint64(1000<<18)) - } - c.Assert(len(mockDDLSink.syncPointHis), check.GreaterEqual, 5) -} - -func (s *changefeedSuite) TestFinished(c *check.C) { - defer testleak.AfterTest(c)() - ctx := cdcContext.NewBackendContext4Test(true) - ctx.ChangefeedVars().Info.TargetTs = ctx.ChangefeedVars().Info.StartTs + 1000 - cf, state, captures, tester := createChangefeed4Test(ctx, c) - defer cf.Close(ctx) - - // pre check - cf.Tick(ctx, state, captures) - tester.MustApplyPatches() - - // initialize - cf.Tick(ctx, state, captures) - tester.MustApplyPatches() - - mockDDLPuller := cf.ddlPuller.(*mockDDLPuller) - mockDDLPuller.resolvedTs += 2000 - // tick many times to make sure the change feed is stopped - for i := 0; i <= 10; i++ { - cf.Tick(ctx, state, captures) - tester.MustApplyPatches() - } - - c.Assert(state.Status.CheckpointTs, check.Equals, state.Info.TargetTs) - c.Assert(state.Info.State, check.Equals, model.StateFinished) -} - func (s *changefeedSuite) TestRemoveChangefeed(c *check.C) { defer testleak.AfterTest(c)() @@ -431,144 +189,3 @@ func testChangefeedReleaseResource( _, err = os.Stat(redoLogDir) c.Assert(os.IsNotExist(err), check.IsTrue) } - -func (s *changefeedSuite) TestAddSpecialComment(c *check.C) { - defer testleak.AfterTest(c)() - testCase := []struct { - input string - result string - }{ - { - "create table t1 (id int ) shard_row_id_bits=2;", - "CREATE TABLE `t1` (`id` INT) /*T! SHARD_ROW_ID_BITS = 2 */", - }, - { - "create table t1 (id int ) shard_row_id_bits=2 pre_split_regions=2;", - "CREATE TABLE `t1` (`id` INT) /*T! SHARD_ROW_ID_BITS = 2 */ /*T! PRE_SPLIT_REGIONS = 2 */", - }, - { - "create table t1 (id int ) shard_row_id_bits=2 pre_split_regions=2;", - "CREATE TABLE `t1` (`id` INT) /*T! SHARD_ROW_ID_BITS = 2 */ /*T! PRE_SPLIT_REGIONS = 2 */", - }, - { - "create table t1 (id int ) shard_row_id_bits=2 engine=innodb pre_split_regions=2;", - "CREATE TABLE `t1` (`id` INT) /*T! SHARD_ROW_ID_BITS = 2 */ ENGINE = innodb /*T! PRE_SPLIT_REGIONS = 2 */", - }, - { - "create table t1 (id int ) pre_split_regions=2 shard_row_id_bits=2;", - "CREATE TABLE `t1` (`id` INT) /*T! PRE_SPLIT_REGIONS = 2 */ /*T! SHARD_ROW_ID_BITS = 2 */", - }, - { - "create table t6 (id int ) shard_row_id_bits=2 shard_row_id_bits=3 pre_split_regions=2;", - "CREATE TABLE `t6` (`id` INT) /*T! SHARD_ROW_ID_BITS = 2 */ /*T! SHARD_ROW_ID_BITS = 3 */ /*T! PRE_SPLIT_REGIONS = 2 */", - }, - { - "create table t1 (id int primary key auto_random(2));", - "CREATE TABLE `t1` (`id` INT PRIMARY KEY /*T![auto_rand] AUTO_RANDOM(2) */)", - }, - { - "create table t1 (id int primary key auto_random);", - "CREATE TABLE `t1` (`id` INT PRIMARY KEY /*T![auto_rand] AUTO_RANDOM */)", - }, - { - "create table t1 (id int auto_random ( 4 ) primary key);", - "CREATE TABLE `t1` (`id` INT /*T![auto_rand] AUTO_RANDOM(4) */ PRIMARY KEY)", - }, - { - "create table t1 (id int auto_random ( 4 ) primary key);", - "CREATE TABLE `t1` (`id` INT /*T![auto_rand] AUTO_RANDOM(4) */ PRIMARY KEY)", - }, - { - "create table t1 (id int auto_random ( 3 ) primary key) auto_random_base = 100;", - "CREATE TABLE `t1` (`id` INT /*T![auto_rand] AUTO_RANDOM(3) */ PRIMARY KEY) /*T![auto_rand_base] AUTO_RANDOM_BASE = 100 */", - }, - { - "create table t1 (id int auto_random primary key) auto_random_base = 50;", - "CREATE TABLE `t1` (`id` INT /*T![auto_rand] AUTO_RANDOM */ PRIMARY KEY) /*T![auto_rand_base] AUTO_RANDOM_BASE = 50 */", - }, - { - "create table t1 (id int auto_increment key) auto_id_cache 100;", - "CREATE TABLE `t1` (`id` INT AUTO_INCREMENT PRIMARY KEY) /*T![auto_id_cache] AUTO_ID_CACHE = 100 */", - }, - { - "create table t1 (id int auto_increment unique) auto_id_cache 10;", - "CREATE TABLE `t1` (`id` INT AUTO_INCREMENT UNIQUE KEY) /*T![auto_id_cache] AUTO_ID_CACHE = 10 */", - }, - { - "create table t1 (id int) auto_id_cache = 5;", - "CREATE TABLE `t1` (`id` INT) /*T![auto_id_cache] AUTO_ID_CACHE = 5 */", - }, - { - "create table t1 (id int) auto_id_cache=5;", - "CREATE TABLE `t1` (`id` INT) /*T![auto_id_cache] AUTO_ID_CACHE = 5 */", - }, - { - "create table t1 (id int) /*T![auto_id_cache] auto_id_cache=5 */ ;", - "CREATE TABLE `t1` (`id` INT) /*T![auto_id_cache] AUTO_ID_CACHE = 5 */", - }, - { - "create table t1 (id int, a varchar(255), primary key (a, b) clustered);", - "CREATE TABLE `t1` (`id` INT,`a` VARCHAR(255),PRIMARY KEY(`a`, `b`) /*T![clustered_index] CLUSTERED */)", - }, - { - "create table t1(id int, v int, primary key(a) clustered);", - "CREATE TABLE `t1` (`id` INT,`v` INT,PRIMARY KEY(`a`) /*T![clustered_index] CLUSTERED */)", - }, - { - "create table t1(id int primary key clustered, v int);", - "CREATE TABLE `t1` (`id` INT PRIMARY KEY /*T![clustered_index] CLUSTERED */,`v` INT)", - }, - { - "alter table t add primary key(a) clustered;", - "ALTER TABLE `t` ADD PRIMARY KEY(`a`) /*T![clustered_index] CLUSTERED */", - }, - { - "create table t1 (id int, a varchar(255), primary key (a, b) nonclustered);", - "CREATE TABLE `t1` (`id` INT,`a` VARCHAR(255),PRIMARY KEY(`a`, `b`) /*T![clustered_index] NONCLUSTERED */)", - }, - { - "create table t1 (id int, a varchar(255), primary key (a, b) /*T![clustered_index] nonclustered */);", - "CREATE TABLE `t1` (`id` INT,`a` VARCHAR(255),PRIMARY KEY(`a`, `b`) /*T![clustered_index] NONCLUSTERED */)", - }, - { - "create table clustered_test(id int)", - "CREATE TABLE `clustered_test` (`id` INT)", - }, - { - "create database clustered_test", - "CREATE DATABASE `clustered_test`", - }, - { - "create database clustered", - "CREATE DATABASE `clustered`", - }, - { - "create table clustered (id int)", - "CREATE TABLE `clustered` (`id` INT)", - }, - { - "create table t1 (id int, a varchar(255) key clustered);", - "CREATE TABLE `t1` (`id` INT,`a` VARCHAR(255) PRIMARY KEY /*T![clustered_index] CLUSTERED */)", - }, - { - "alter table t force auto_increment = 12;", - "ALTER TABLE `t` /*T![force_inc] FORCE */ AUTO_INCREMENT = 12", - }, - { - "alter table t force, auto_increment = 12;", - "ALTER TABLE `t` FORCE /* AlterTableForce is not supported */ , AUTO_INCREMENT = 12", - }, - { - "create table cdc_test (id varchar(10) primary key ,c1 varchar(10)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin/*!90000 SHARD_ROW_ID_BITS=4 PRE_SPLIT_REGIONS=3 */", - "CREATE TABLE `cdc_test` (`id` VARCHAR(10) PRIMARY KEY,`c1` VARCHAR(10)) ENGINE = InnoDB DEFAULT CHARACTER SET = UTF8MB4 DEFAULT COLLATE = UTF8MB4_BIN /*T! SHARD_ROW_ID_BITS = 4 */ /*T! PRE_SPLIT_REGIONS = 3 */", - }, - } - for _, ca := range testCase { - re, err := addSpecialComment(ca.input) - c.Check(err, check.IsNil) - c.Check(re, check.Equals, ca.result) - } - c.Assert(func() { - _, _ = addSpecialComment("alter table t force, auto_increment = 12;alter table t force, auto_increment = 12;") - }, check.Panics, "invalid ddlQuery statement size") -} diff --git a/cdc/cdc/owner/ddl_puller.go b/cdc/cdc/owner/ddl_puller.go deleted file mode 100644 index fc207c57..00000000 --- a/cdc/cdc/owner/ddl_puller.go +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package owner - -import ( - "context" - "sync" - "time" - - "github.com/benbjohnson/clock" - "github.com/pingcap/errors" - "github.com/pingcap/log" - timodel "github.com/pingcap/tidb/parser/model" - "github.com/tikv/migration/cdc/cdc/entry" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/puller" - "github.com/tikv/migration/cdc/cdc/sorter/memory" - cdcContext "github.com/tikv/migration/cdc/pkg/context" - "github.com/tikv/migration/cdc/pkg/filter" - "github.com/tikv/migration/cdc/pkg/regionspan" - "github.com/tikv/migration/cdc/pkg/util" - "go.uber.org/zap" - "golang.org/x/sync/errgroup" -) - -const ( - ownerDDLPullerStuckWarnTimeout = 30 * time.Second -) - -// DDLPuller is a wrapper of the Puller interface for the owner -// DDLPuller starts a puller, listens to the DDL range, adds the received DDLs into an internal queue -type DDLPuller interface { - // Run runs the DDLPuller - Run(ctx cdcContext.Context) error - // FrontDDL returns the first DDL job in the internal queue - FrontDDL() (uint64, *timodel.Job) - // PopFrontDDL returns and pops the first DDL job in the internal queue - PopFrontDDL() (uint64, *timodel.Job) - // Close closes the DDLPuller - Close() -} - -type ddlPullerImpl struct { - puller puller.Puller - filter *filter.Filter - - mu sync.Mutex - resolvedTS uint64 - pendingDDLJobs []*timodel.Job - lastDDLJobID int64 - cancel context.CancelFunc - - clock clock.Clock -} - -func newDDLPuller(ctx cdcContext.Context, startTs uint64) (DDLPuller, error) { - pdCli := ctx.GlobalVars().PDClient - f, err := filter.NewFilter(ctx.ChangefeedVars().Info.Config) - if err != nil { - return nil, errors.Trace(err) - } - var plr puller.Puller - kvStorage := ctx.GlobalVars().KVStorage - // kvStorage can be nil only in the test - if kvStorage != nil { - plr = puller.NewPuller(ctx, pdCli, ctx.GlobalVars().GrpcPool, ctx.GlobalVars().RegionCache, kvStorage, startTs, - []regionspan.Span{regionspan.GetDDLSpan(), regionspan.GetAddIndexDDLSpan()}, false) - } - - return &ddlPullerImpl{ - puller: plr, - resolvedTS: startTs, - filter: f, - cancel: func() {}, - clock: clock.New(), - }, nil -} - -func (h *ddlPullerImpl) Run(ctx cdcContext.Context) error { - ctx, cancel := cdcContext.WithCancel(ctx) - h.cancel = cancel - log.Debug("DDL puller started", zap.String("changefeed-id", ctx.ChangefeedVars().ID)) - stdCtx := util.PutTableInfoInCtx(ctx, -1, puller.DDLPullerTableName) - stdCtx = util.PutChangefeedIDInCtx(stdCtx, ctx.ChangefeedVars().ID) - errg, stdCtx := errgroup.WithContext(stdCtx) - lastResolvedTsAdanvcedTime := h.clock.Now() - - errg.Go(func() error { - return h.puller.Run(stdCtx) - }) - - rawDDLCh := memory.SortOutput(stdCtx, h.puller.Output()) - - receiveDDL := func(rawDDL *model.RawKVEntry) error { - if rawDDL == nil { - return nil - } - if rawDDL.OpType == model.OpTypeResolved { - h.mu.Lock() - defer h.mu.Unlock() - if rawDDL.CRTs > h.resolvedTS { - lastResolvedTsAdanvcedTime = h.clock.Now() - h.resolvedTS = rawDDL.CRTs - } - return nil - } - job, err := entry.UnmarshalDDL(rawDDL) - if err != nil { - return errors.Trace(err) - } - if job == nil { - return nil - } - if h.filter.ShouldDiscardDDL(job.Type) { - log.Info("discard the ddl job", zap.Int64("jobID", job.ID), zap.String("query", job.Query)) - return nil - } - if job.ID == h.lastDDLJobID { - log.Warn("ignore duplicated DDL job", zap.Any("job", job)) - return nil - } - h.mu.Lock() - defer h.mu.Unlock() - h.pendingDDLJobs = append(h.pendingDDLJobs, job) - h.lastDDLJobID = job.ID - return nil - } - - ticker := h.clock.Ticker(ownerDDLPullerStuckWarnTimeout) - defer ticker.Stop() - - errg.Go(func() error { - for { - select { - case <-stdCtx.Done(): - return stdCtx.Err() - case <-ticker.C: - duration := h.clock.Since(lastResolvedTsAdanvcedTime) - if duration > ownerDDLPullerStuckWarnTimeout { - log.Warn("ddl puller resolved ts has not advanced", - zap.String("changefeed-id", ctx.ChangefeedVars().ID), - zap.Duration("duration", duration), - zap.Uint64("resolved-ts", h.resolvedTS)) - } - case e := <-rawDDLCh: - if err := receiveDDL(e); err != nil { - return errors.Trace(err) - } - } - } - }) - - return errg.Wait() -} - -func (h *ddlPullerImpl) FrontDDL() (uint64, *timodel.Job) { - h.mu.Lock() - defer h.mu.Unlock() - if len(h.pendingDDLJobs) == 0 { - return h.resolvedTS, nil - } - job := h.pendingDDLJobs[0] - return job.BinlogInfo.FinishedTS, job -} - -func (h *ddlPullerImpl) PopFrontDDL() (uint64, *timodel.Job) { - h.mu.Lock() - defer h.mu.Unlock() - if len(h.pendingDDLJobs) == 0 { - return h.resolvedTS, nil - } - job := h.pendingDDLJobs[0] - h.pendingDDLJobs = h.pendingDDLJobs[1:] - return job.BinlogInfo.FinishedTS, job -} - -func (h *ddlPullerImpl) Close() { - log.Info("Close the ddl puller") - h.cancel() -} diff --git a/cdc/cdc/owner/ddl_puller_test.go b/cdc/cdc/owner/ddl_puller_test.go deleted file mode 100644 index 5eb3bbde..00000000 --- a/cdc/cdc/owner/ddl_puller_test.go +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package owner - -import ( - "context" - "encoding/json" - "sync" - "sync/atomic" - "time" - - "github.com/benbjohnson/clock" - "github.com/pingcap/check" - "github.com/pingcap/errors" - "github.com/pingcap/log" - timodel "github.com/pingcap/tidb/parser/model" - "github.com/pingcap/tidb/util/codec" - "github.com/tikv/migration/cdc/cdc/model" - cdcContext "github.com/tikv/migration/cdc/pkg/context" - "github.com/tikv/migration/cdc/pkg/retry" - "github.com/tikv/migration/cdc/pkg/util/testleak" - "go.uber.org/zap" - "go.uber.org/zap/zaptest/observer" -) - -var _ = check.Suite(&ddlPullerSuite{}) - -type ddlPullerSuite struct{} - -type mockPuller struct { - c *check.C - inCh chan *model.RawKVEntry - outCh chan *model.RawKVEntry - resolvedTs model.Ts -} - -func newMockPuller(c *check.C, startTs model.Ts) *mockPuller { - return &mockPuller{ - c: c, - inCh: make(chan *model.RawKVEntry), - outCh: make(chan *model.RawKVEntry), - resolvedTs: startTs - 1, - } -} - -func (m *mockPuller) Run(ctx context.Context) error { - for { - select { - case <-ctx.Done(): - return ctx.Err() - case e := <-m.inCh: - m.outCh <- e - atomic.StoreUint64(&m.resolvedTs, e.CRTs) - } - } -} - -func (m *mockPuller) GetResolvedTs() uint64 { - return atomic.LoadUint64(&m.resolvedTs) -} - -func (m *mockPuller) Output() <-chan *model.RawKVEntry { - return m.outCh -} - -func (m *mockPuller) IsInitialized() bool { - return true -} - -func (m *mockPuller) append(e *model.RawKVEntry) { - m.inCh <- e -} - -func (m *mockPuller) appendDDL(job *timodel.Job) { - b, err := json.Marshal(job) - m.c.Assert(err, check.IsNil) - ek := []byte("m") - ek = codec.EncodeBytes(ek, []byte("DDLJobList")) - ek = codec.EncodeUint(ek, uint64('l')) - ek = codec.EncodeInt(ek, 1) - m.append(&model.RawKVEntry{ - OpType: model.OpTypePut, - Key: ek, - Value: b, - StartTs: job.StartTS, - CRTs: job.BinlogInfo.FinishedTS, - }) -} - -func (m *mockPuller) appendResolvedTs(ts model.Ts) { - m.append(&model.RawKVEntry{ - OpType: model.OpTypeResolved, - CRTs: ts, - StartTs: ts, - }) -} - -func (s *ddlPullerSuite) TestPuller(c *check.C) { - defer testleak.AfterTest(c)() - startTs := uint64(10) - mockPuller := newMockPuller(c, startTs) - ctx := cdcContext.NewBackendContext4Test(true) - p, err := newDDLPuller(ctx, startTs) - c.Assert(err, check.IsNil) - p.(*ddlPullerImpl).puller = mockPuller - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - err := p.Run(ctx) - if errors.Cause(err) == context.Canceled { - err = nil - } - c.Assert(err, check.IsNil) - }() - defer wg.Wait() - defer p.Close() - - // test initialize state - resolvedTs, ddl := p.FrontDDL() - c.Assert(resolvedTs, check.Equals, startTs) - c.Assert(ddl, check.IsNil) - resolvedTs, ddl = p.PopFrontDDL() - c.Assert(resolvedTs, check.Equals, startTs) - c.Assert(ddl, check.IsNil) - - // test send resolvedTs - mockPuller.appendResolvedTs(15) - waitResolvedTsGrowing(c, p, 15) - - // test send ddl job out of order - mockPuller.appendDDL(&timodel.Job{ - ID: 2, - Type: timodel.ActionCreateTable, - StartTS: 5, - State: timodel.JobStateDone, - BinlogInfo: &timodel.HistoryInfo{FinishedTS: 18}, - }) - mockPuller.appendDDL(&timodel.Job{ - ID: 1, - Type: timodel.ActionCreateTable, - StartTS: 5, - State: timodel.JobStateDone, - BinlogInfo: &timodel.HistoryInfo{FinishedTS: 16}, - }) - resolvedTs, ddl = p.FrontDDL() - c.Assert(resolvedTs, check.Equals, uint64(15)) - c.Assert(ddl, check.IsNil) - - mockPuller.appendResolvedTs(20) - waitResolvedTsGrowing(c, p, 16) - resolvedTs, ddl = p.FrontDDL() - c.Assert(resolvedTs, check.Equals, uint64(16)) - c.Assert(ddl.ID, check.Equals, int64(1)) - resolvedTs, ddl = p.PopFrontDDL() - c.Assert(resolvedTs, check.Equals, uint64(16)) - c.Assert(ddl.ID, check.Equals, int64(1)) - - // DDL could be processed with a delay, wait here for a pending DDL job is added - waitResolvedTsGrowing(c, p, 18) - resolvedTs, ddl = p.PopFrontDDL() - c.Assert(resolvedTs, check.Equals, uint64(18)) - c.Assert(ddl.ID, check.Equals, int64(2)) - - // test add ddl job repeated - mockPuller.appendDDL(&timodel.Job{ - ID: 3, - Type: timodel.ActionCreateTable, - StartTS: 20, - State: timodel.JobStateDone, - BinlogInfo: &timodel.HistoryInfo{FinishedTS: 25}, - }) - mockPuller.appendDDL(&timodel.Job{ - ID: 3, - Type: timodel.ActionCreateTable, - StartTS: 20, - State: timodel.JobStateDone, - BinlogInfo: &timodel.HistoryInfo{FinishedTS: 25}, - }) - mockPuller.appendResolvedTs(30) - waitResolvedTsGrowing(c, p, 25) - - resolvedTs, ddl = p.PopFrontDDL() - c.Assert(resolvedTs, check.Equals, uint64(25)) - c.Assert(ddl.ID, check.Equals, int64(3)) - _, ddl = p.PopFrontDDL() - c.Assert(ddl, check.IsNil) - - waitResolvedTsGrowing(c, p, 30) - resolvedTs, ddl = p.PopFrontDDL() - c.Assert(resolvedTs, check.Equals, uint64(30)) - c.Assert(ddl, check.IsNil) - - // test add invalid ddl job - mockPuller.appendDDL(&timodel.Job{ - ID: 4, - Type: timodel.ActionLockTable, - StartTS: 20, - State: timodel.JobStateDone, - BinlogInfo: &timodel.HistoryInfo{FinishedTS: 35}, - }) - mockPuller.appendDDL(&timodel.Job{ - ID: 5, - Type: timodel.ActionCreateTable, - StartTS: 20, - State: timodel.JobStateCancelled, - BinlogInfo: &timodel.HistoryInfo{FinishedTS: 36}, - }) - mockPuller.appendResolvedTs(40) - waitResolvedTsGrowing(c, p, 40) - resolvedTs, ddl = p.PopFrontDDL() - // no ddl should be received - c.Assert(resolvedTs, check.Equals, uint64(40)) - c.Assert(ddl, check.IsNil) -} - -func (*ddlPullerSuite) TestResolvedTsStuck(c *check.C) { - defer testleak.AfterTest(c)() - // For observing the logs - zapcore, logs := observer.New(zap.WarnLevel) - conf := &log.Config{Level: "warn", File: log.FileLogConfig{}} - _, r, _ := log.InitLogger(conf) - logger := zap.New(zapcore) - restoreFn := log.ReplaceGlobals(logger, r) - defer restoreFn() - - startTs := uint64(10) - mockPuller := newMockPuller(c, startTs) - ctx := cdcContext.NewBackendContext4Test(true) - p, err := newDDLPuller(ctx, startTs) - c.Assert(err, check.IsNil) - - mockClock := clock.NewMock() - p.(*ddlPullerImpl).clock = mockClock - - p.(*ddlPullerImpl).puller = mockPuller - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - err := p.Run(ctx) - if errors.Cause(err) == context.Canceled { - err = nil - } - c.Assert(err, check.IsNil) - }() - defer wg.Wait() - defer p.Close() - - // test initialize state - resolvedTs, ddl := p.FrontDDL() - c.Assert(resolvedTs, check.Equals, startTs) - c.Assert(ddl, check.IsNil) - resolvedTs, ddl = p.PopFrontDDL() - c.Assert(resolvedTs, check.Equals, startTs) - c.Assert(ddl, check.IsNil) - - mockPuller.appendResolvedTs(30) - waitResolvedTsGrowing(c, p, 30) - c.Assert(logs.Len(), check.Equals, 0) - - mockClock.Add(2 * ownerDDLPullerStuckWarnTimeout) - for i := 0; i < 20; i++ { - mockClock.Add(time.Second) - if logs.Len() > 0 { - break - } - time.Sleep(10 * time.Millisecond) - if i == 19 { - c.Fatal("warning log not printed") - } - } - - mockPuller.appendResolvedTs(40) - waitResolvedTsGrowing(c, p, 40) -} - -// waitResolvedTsGrowing can wait the first DDL reaches targetTs or if no pending -// DDL, DDL resolved ts reaches targetTs. -func waitResolvedTsGrowing(c *check.C, p DDLPuller, targetTs model.Ts) { - err := retry.Do(context.Background(), func() error { - resolvedTs, _ := p.FrontDDL() - if resolvedTs < targetTs { - return errors.New("resolvedTs < targetTs") - } - return nil - }, retry.WithBackoffBaseDelay(20), retry.WithMaxTries(100)) - c.Assert(err, check.IsNil) -} diff --git a/cdc/cdc/owner/ddl_sink.go b/cdc/cdc/owner/ddl_sink.go deleted file mode 100644 index ead1bd77..00000000 --- a/cdc/cdc/owner/ddl_sink.go +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package owner - -import ( - "context" - "sync" - "sync/atomic" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/sink" - cdcContext "github.com/tikv/migration/cdc/pkg/context" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/filter" - "go.uber.org/zap" -) - -const ( - defaultErrChSize = 1024 -) - -// DDLSink is a wrapper of the `Sink` interface for the owner -// DDLSink should send `DDLEvent` and `CheckpointTs` to downstream sink, -// If `SyncPointEnabled`, also send `syncPoint` to downstream. -type DDLSink interface { - // run the DDLSink - run(ctx cdcContext.Context, id model.ChangeFeedID, info *model.ChangeFeedInfo) - // emitCheckpointTs emits the checkpoint Ts to downstream data source - // this function will return after recording the checkpointTs specified in memory immediately - // and the recorded checkpointTs will be sent and updated to downstream data source every second - emitCheckpointTs(ctx cdcContext.Context, ts uint64) - // emitDDLEvent emits DDL event and return true if the DDL is executed - // the DDL event will be sent to another goroutine and execute to downstream - // the caller of this function can call again and again until a true returned - emitDDLEvent(ctx cdcContext.Context, ddl *model.DDLEvent) (bool, error) - emitSyncPoint(ctx cdcContext.Context, checkpointTs uint64) error - // close the sink, cancel running goroutine. - close(ctx context.Context) error -} - -type ddlSinkImpl struct { - lastSyncPoint model.Ts - syncPointStore sink.SyncpointStore - - checkpointTs model.Ts - ddlFinishedTs model.Ts - ddlSentTs model.Ts - - ddlCh chan *model.DDLEvent - errCh chan error - - sink sink.Sink - // `sinkInitHandler` can be helpful in unit testing. - sinkInitHandler ddlSinkInitHandler - - // cancel would be used to cancel the goroutine start by `run` - cancel context.CancelFunc - wg sync.WaitGroup -} - -func newDDLSink() DDLSink { - return &ddlSinkImpl{ - ddlCh: make(chan *model.DDLEvent, 1), - errCh: make(chan error, defaultErrChSize), - sinkInitHandler: ddlSinkInitializer, - cancel: func() {}, - } -} - -type ddlSinkInitHandler func(ctx cdcContext.Context, a *ddlSinkImpl, id model.ChangeFeedID, info *model.ChangeFeedInfo) error - -func ddlSinkInitializer(ctx cdcContext.Context, a *ddlSinkImpl, id model.ChangeFeedID, info *model.ChangeFeedInfo) error { - filter, err := filter.NewFilter(info.Config) - if err != nil { - return errors.Trace(err) - } - - s, err := sink.New(ctx, id, info.SinkURI, filter, info.Config, info.Opts, a.errCh) - if err != nil { - return errors.Trace(err) - } - a.sink = s - - if !info.SyncPointEnabled { - return nil - } - syncPointStore, err := sink.NewSyncpointStore(ctx, id, info.SinkURI) - if err != nil { - return errors.Trace(err) - } - a.syncPointStore = syncPointStore - - if err := a.syncPointStore.CreateSynctable(ctx); err != nil { - return errors.Trace(err) - } - return nil -} - -func (s *ddlSinkImpl) run(ctx cdcContext.Context, id model.ChangeFeedID, info *model.ChangeFeedInfo) { - ctx, cancel := cdcContext.WithCancel(ctx) - s.cancel = cancel - - s.wg.Add(1) - go func() { - defer s.wg.Done() - - start := time.Now() - if err := s.sinkInitHandler(ctx, s, id, info); err != nil { - log.Warn("ddl sink initialize failed", zap.Duration("elapsed", time.Since(start))) - ctx.Throw(err) - return - } - log.Info("ddl sink initialized, start processing...", zap.Duration("elapsed", time.Since(start))) - - // TODO make the tick duration configurable - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - var lastCheckpointTs model.Ts - for { - select { - case <-ctx.Done(): - return - case err := <-s.errCh: - ctx.Throw(err) - return - case <-ticker.C: - checkpointTs := atomic.LoadUint64(&s.checkpointTs) - if checkpointTs == 0 || checkpointTs <= lastCheckpointTs { - continue - } - lastCheckpointTs = checkpointTs - if err := s.sink.EmitCheckpointTs(ctx, checkpointTs); err != nil { - ctx.Throw(errors.Trace(err)) - return - } - case ddl := <-s.ddlCh: - err := s.sink.EmitDDLEvent(ctx, ddl) - failpoint.Inject("InjectChangefeedDDLError", func() { - err = cerror.ErrExecDDLFailed.GenWithStackByArgs() - }) - if err == nil || cerror.ErrDDLEventIgnored.Equal(errors.Cause(err)) { - log.Info("Execute DDL succeeded", zap.String("changefeed", ctx.ChangefeedVars().ID), zap.Bool("ignored", err != nil), zap.Reflect("ddl", ddl)) - atomic.StoreUint64(&s.ddlFinishedTs, ddl.CommitTs) - continue - } - // If DDL executing failed, and the error can not be ignored, throw an error and pause the changefeed - log.Error("Execute DDL failed", - zap.String("ChangeFeedID", ctx.ChangefeedVars().ID), - zap.Error(err), - zap.Reflect("ddl", ddl)) - ctx.Throw(errors.Trace(err)) - return - } - } - }() -} - -func (s *ddlSinkImpl) emitCheckpointTs(ctx cdcContext.Context, ts uint64) { - atomic.StoreUint64(&s.checkpointTs, ts) -} - -func (s *ddlSinkImpl) emitDDLEvent(ctx cdcContext.Context, ddl *model.DDLEvent) (bool, error) { - ddlFinishedTs := atomic.LoadUint64(&s.ddlFinishedTs) - if ddl.CommitTs <= ddlFinishedTs { - // the DDL event is executed successfully, and done is true - return true, nil - } - if ddl.CommitTs <= s.ddlSentTs { - // the DDL event is executing and not finished yet, return false - return false, nil - } - select { - case <-ctx.Done(): - return false, errors.Trace(ctx.Err()) - case s.ddlCh <- ddl: - s.ddlSentTs = ddl.CommitTs - default: - // if this hit, we think that ddlCh is full, - // just return false and send the ddl in the next round. - } - return false, nil -} - -func (s *ddlSinkImpl) emitSyncPoint(ctx cdcContext.Context, checkpointTs uint64) error { - if checkpointTs == s.lastSyncPoint { - return nil - } - s.lastSyncPoint = checkpointTs - // TODO implement async sink syncPoint - return s.syncPointStore.SinkSyncpoint(ctx, ctx.ChangefeedVars().ID, checkpointTs) -} - -func (s *ddlSinkImpl) close(ctx context.Context) (err error) { - s.cancel() - if s.sink != nil { - err = s.sink.Close(ctx) - } - if s.syncPointStore != nil { - err = s.syncPointStore.Close() - } - s.wg.Wait() - return err -} diff --git a/cdc/cdc/owner/ddl_sink_test.go b/cdc/cdc/owner/ddl_sink_test.go deleted file mode 100644 index bb31d029..00000000 --- a/cdc/cdc/owner/ddl_sink_test.go +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package owner - -import ( - "context" - "sync" - "sync/atomic" - "time" - - "github.com/pingcap/check" - "github.com/pingcap/errors" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/sink" - cdcContext "github.com/tikv/migration/cdc/pkg/context" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/retry" - "github.com/tikv/migration/cdc/pkg/util/testleak" -) - -var _ = check.Suite(&ddlSinkSuite{}) - -type ddlSinkSuite struct{} - -type mockSink struct { - sink.Sink - checkpointTs model.Ts - ddl *model.DDLEvent - ddlMu sync.Mutex - ddlError error -} - -func (m *mockSink) EmitCheckpointTs(ctx context.Context, ts uint64) error { - atomic.StoreUint64(&m.checkpointTs, ts) - return nil -} - -func (m *mockSink) EmitDDLEvent(ctx context.Context, ddl *model.DDLEvent) error { - m.ddlMu.Lock() - defer m.ddlMu.Unlock() - time.Sleep(1 * time.Second) - m.ddl = ddl - return m.ddlError -} - -func (m *mockSink) Close(ctx context.Context) error { - return nil -} - -func (m *mockSink) Barrier(ctx context.Context, tableID model.TableID) error { - return nil -} - -func (m *mockSink) GetDDL() *model.DDLEvent { - m.ddlMu.Lock() - defer m.ddlMu.Unlock() - return m.ddl -} - -func newDDLSink4Test() (DDLSink, *mockSink) { - mockSink := &mockSink{} - ddlSink := newDDLSink() - ddlSink.(*ddlSinkImpl).sinkInitHandler = func(ctx cdcContext.Context, a *ddlSinkImpl, _ model.ChangeFeedID, _ *model.ChangeFeedInfo) error { - a.sink = mockSink - return nil - } - return ddlSink, mockSink -} - -func (s *ddlSinkSuite) TestCheckpoint(c *check.C) { - defer testleak.AfterTest(c)() - ddlSink, mSink := newDDLSink4Test() - ctx := cdcContext.NewBackendContext4Test(true) - ctx, cancel := cdcContext.WithCancel(ctx) - defer func() { - cancel() - ddlSink.close(ctx) - }() - ddlSink.run(ctx, ctx.ChangefeedVars().ID, ctx.ChangefeedVars().Info) - - waitCheckpointGrowingUp := func(m *mockSink, targetTs model.Ts) error { - return retry.Do(context.Background(), func() error { - if targetTs != atomic.LoadUint64(&m.checkpointTs) { - return errors.New("targetTs!=checkpointTs") - } - return nil - }, retry.WithBackoffBaseDelay(100), retry.WithMaxTries(30)) - } - ddlSink.emitCheckpointTs(ctx, 1) - c.Assert(waitCheckpointGrowingUp(mSink, 1), check.IsNil) - ddlSink.emitCheckpointTs(ctx, 10) - c.Assert(waitCheckpointGrowingUp(mSink, 10), check.IsNil) -} - -func (s *ddlSinkSuite) TestExecDDL(c *check.C) { - defer testleak.AfterTest(c)() - ddlSink, mSink := newDDLSink4Test() - ctx := cdcContext.NewBackendContext4Test(true) - ctx, cancel := cdcContext.WithCancel(ctx) - defer func() { - cancel() - ddlSink.close(ctx) - }() - ddlSink.run(ctx, ctx.ChangefeedVars().ID, ctx.ChangefeedVars().Info) - - ddlEvents := []*model.DDLEvent{ - {CommitTs: 1}, - {CommitTs: 2}, - {CommitTs: 3}, - } - - for _, event := range ddlEvents { - for { - done, err := ddlSink.emitDDLEvent(ctx, event) - c.Assert(err, check.IsNil) - if done { - c.Assert(mSink.GetDDL(), check.DeepEquals, event) - break - } - } - } -} - -func (s *ddlSinkSuite) TestExecDDLError(c *check.C) { - defer testleak.AfterTest(c)() - ctx := cdcContext.NewBackendContext4Test(true) - - var ( - resultErr error - resultErrMu sync.Mutex - ) - readResultErr := func() error { - resultErrMu.Lock() - defer resultErrMu.Unlock() - return resultErr - } - - ddlSink, mSink := newDDLSink4Test() - ctx = cdcContext.WithErrorHandler(ctx, func(err error) error { - resultErrMu.Lock() - defer resultErrMu.Unlock() - resultErr = err - return nil - }) - ctx, cancel := cdcContext.WithCancel(ctx) - defer func() { - cancel() - ddlSink.close(ctx) - }() - - ddlSink.run(ctx, ctx.ChangefeedVars().ID, ctx.ChangefeedVars().Info) - - mSink.ddlError = cerror.ErrDDLEventIgnored.GenWithStackByArgs() - ddl1 := &model.DDLEvent{CommitTs: 1} - for { - done, err := ddlSink.emitDDLEvent(ctx, ddl1) - c.Assert(err, check.IsNil) - if done { - c.Assert(mSink.GetDDL(), check.DeepEquals, ddl1) - break - } - } - c.Assert(resultErr, check.IsNil) - - mSink.ddlError = cerror.ErrExecDDLFailed.GenWithStackByArgs() - ddl2 := &model.DDLEvent{CommitTs: 2} - for { - done, err := ddlSink.emitDDLEvent(ctx, ddl2) - c.Assert(err, check.IsNil) - - if done || readResultErr() != nil { - c.Assert(mSink.GetDDL(), check.DeepEquals, ddl2) - break - } - } - c.Assert(cerror.ErrExecDDLFailed.Equal(readResultErr()), check.IsTrue) -} diff --git a/cdc/cdc/owner/feed_state_manager_test.go b/cdc/cdc/owner/feed_state_manager_test.go index 7dcee33e..b598cdf0 100644 --- a/cdc/cdc/owner/feed_state_manager_test.go +++ b/cdc/cdc/owner/feed_state_manager_test.go @@ -232,9 +232,9 @@ func (s *feedStateManagerSuite) TestChangefeedStatusNotExist(c *check.C) { manager := new(feedStateManager) state := orchestrator.NewChangefeedReactorState(ctx.ChangefeedVars().ID) tester := orchestrator.NewReactorStateTester(c, state, map[string]string{ - "/tidb/cdc/capture/d563bfc0-f406-4f34-bc7d-6dc2e35a44e5": `{"id":"d563bfc0-f406-4f34-bc7d-6dc2e35a44e5","address":"172.16.6.147:8300","version":"v5.0.0-master-dirty"}`, - "/tidb/cdc/changefeed/info/" + ctx.ChangefeedVars().ID: `{"sink-uri":"blackhole:///","opts":{},"create-time":"2021-06-05T00:44:15.065939487+08:00","start-ts":425381670108266496,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","config":{"case-sensitive":true,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"open-protocol"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"table-number","polling-time":-1}},"state":"failed","history":[],"error":{"addr":"172.16.6.147:8300","code":"CDC:ErrSnapshotLostByGC","message":"[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 425381670108266496 is earlier than GC safepoint at 0"},"sync-point-enabled":false,"sync-point-interval":600000000000,"creator-version":"v5.0.0-master-dirty"}`, - "/tidb/cdc/owner/156579d017f84a68": "d563bfc0-f406-4f34-bc7d-6dc2e35a44e5", + "/tikv/cdc/capture/d563bfc0-f406-4f34-bc7d-6dc2e35a44e5": `{"id":"d563bfc0-f406-4f34-bc7d-6dc2e35a44e5","address":"172.16.6.147:8300","version":"v5.0.0-master-dirty"}`, + "/tikv/cdc/changefeed/info/" + ctx.ChangefeedVars().ID: `{"sink-uri":"blackhole:///","opts":{},"create-time":"2021-06-05T00:44:15.065939487+08:00","start-ts":425381670108266496,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","config":{"case-sensitive":true,"enable-old-value":true,"force-replicate":false,"check-gc-safe-point":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"open-protocol"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"keyspan-number","polling-time":-1}},"state":"failed","history":[],"error":{"addr":"172.16.6.147:8300","code":"CDC:ErrSnapshotLostByGC","message":"[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 425381670108266496 is earlier than GC safepoint at 0"},"sync-point-enabled":false,"sync-point-interval":600000000000,"creator-version":"v5.0.0-master-dirty"}`, + "/tikv/cdc/owner/156579d017f84a68": "d563bfc0-f406-4f34-bc7d-6dc2e35a44e5", }) manager.Tick(state) c.Assert(manager.ShouldRunning(), check.IsFalse) diff --git a/cdc/cdc/owner/metrics.go b/cdc/cdc/owner/metrics.go index bc1bff8d..8297eb59 100644 --- a/cdc/cdc/owner/metrics.go +++ b/cdc/cdc/owner/metrics.go @@ -18,49 +18,49 @@ import "github.com/prometheus/client_golang/prometheus" var ( changefeedCheckpointTsGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "owner", Name: "checkpoint_ts", Help: "checkpoint ts of changefeeds", }, []string{"changefeed"}) changefeedCheckpointTsLagGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "owner", Name: "checkpoint_ts_lag", Help: "checkpoint ts lag of changefeeds in seconds", }, []string{"changefeed"}) changefeedResolvedTsGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "owner", Name: "resolved_ts", Help: "resolved ts of changefeeds", }, []string{"changefeed"}) changefeedResolvedTsLagGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "owner", Name: "resolved_ts_lag", Help: "resolved ts lag of changefeeds in seconds", }, []string{"changefeed"}) ownershipCounter = prometheus.NewCounter( prometheus.CounterOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "owner", Name: "ownership_counter", Help: "The counter of ownership increases every 5 seconds on a owner capture", }) - ownerMaintainTableNumGauge = prometheus.NewGaugeVec( + ownerMaintainKeySpanNumGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "owner", - Name: "maintain_table_num", - Help: "number of replicated tables maintained in owner", + Name: "maintain_keyspan_num", + Help: "number of replicated keyspans maintained in owner", }, []string{"changefeed", "capture", "type"}) changefeedStatusGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "owner", Name: "status", Help: "The status of changefeeds", @@ -68,10 +68,10 @@ var ( ) const ( - // total tables that have been dispatched to a single processor - maintainTableTypeTotal string = "total" - // tables that are dispatched to a processor and have not been finished yet - maintainTableTypeWip string = "wip" + // total keyspans that have been dispatched to a single processor + maintainKeySpanTypeTotal string = "total" + // keyspans that are dispatched to a processor and have not been finished yet + maintainKeySpanTypeWip string = "wip" ) // InitMetrics registers all metrics used in owner @@ -81,6 +81,6 @@ func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(changefeedCheckpointTsLagGauge) registry.MustRegister(changefeedResolvedTsLagGauge) registry.MustRegister(ownershipCounter) - registry.MustRegister(ownerMaintainTableNumGauge) + registry.MustRegister(ownerMaintainKeySpanNumGauge) registry.MustRegister(changefeedStatusGauge) } diff --git a/cdc/cdc/owner/owner.go b/cdc/cdc/owner/owner.go index 100537e9..cd5d0ca7 100644 --- a/cdc/cdc/owner/owner.go +++ b/cdc/cdc/owner/owner.go @@ -58,7 +58,7 @@ type ownerJob struct { // for ManualSchedule only targetCaptureID model.CaptureID // for ManualSchedule only - tableID model.TableID + keyspanID model.KeySpanID // for Admin Job only adminJob *model.AdminJob @@ -106,16 +106,15 @@ func NewOwner(pdClient pd.Client) *Owner { } // NewOwner4Test creates a new Owner for test +// TODO: modify for tikv cdc func NewOwner4Test( - newDDLPuller func(ctx cdcContext.Context, startTs uint64) (DDLPuller, error), - newSink func() DDLSink, pdClient pd.Client, ) *Owner { o := NewOwner(pdClient) // Most tests do not need to test bootstrap. o.bootstrapped = true o.newChangefeed = func(id model.ChangeFeedID, gcManager gc.Manager) *changefeed { - return newChangefeed4Test(id, gcManager, newDDLPuller, newSink) + return newChangefeed4Test(id, gcManager) } return o } @@ -219,13 +218,13 @@ func (o *Owner) TriggerRebalance(cfID model.ChangeFeedID) { }) } -// ManualSchedule moves a table from a capture to another capture -func (o *Owner) ManualSchedule(cfID model.ChangeFeedID, toCapture model.CaptureID, tableID model.TableID) { +// ManualSchedule moves a keyspan from a capture to another capture +func (o *Owner) ManualSchedule(cfID model.ChangeFeedID, toCapture model.CaptureID, keyspanID model.KeySpanID) { o.pushOwnerJob(&ownerJob{ tp: ownerJobTypeManualSchedule, changefeedID: cfID, targetCaptureID: toCapture, - tableID: tableID, + keyspanID: keyspanID, done: make(chan struct{}), }) } @@ -304,7 +303,7 @@ func (o *Owner) updateMetrics(state *orchestrator.GlobalReactorState) { ownershipCounter.Add(float64(now.Sub(o.lastTickTime)) / float64(time.Second)) o.lastTickTime = now - ownerMaintainTableNumGauge.Reset() + ownerMaintainKeySpanNumGauge.Reset() changefeedStatusGauge.Reset() for changefeedID, changefeedState := range state.Changefeeds { for captureID, captureInfo := range state.Captures { @@ -312,8 +311,8 @@ func (o *Owner) updateMetrics(state *orchestrator.GlobalReactorState) { if !exist { continue } - ownerMaintainTableNumGauge.WithLabelValues(changefeedID, captureInfo.AdvertiseAddr, maintainTableTypeTotal).Set(float64(len(taskStatus.Tables))) - ownerMaintainTableNumGauge.WithLabelValues(changefeedID, captureInfo.AdvertiseAddr, maintainTableTypeWip).Set(float64(len(taskStatus.Operation))) + ownerMaintainKeySpanNumGauge.WithLabelValues(changefeedID, captureInfo.AdvertiseAddr, maintainKeySpanTypeTotal).Set(float64(len(taskStatus.KeySpans))) + ownerMaintainKeySpanNumGauge.WithLabelValues(changefeedID, captureInfo.AdvertiseAddr, maintainKeySpanTypeWip).Set(float64(len(taskStatus.Operation))) if changefeedState.Info != nil { changefeedStatusGauge.WithLabelValues(changefeedID).Set(float64(changefeedState.Info.State.ToInt())) } @@ -347,7 +346,7 @@ func (o *Owner) handleJobs() { case ownerJobTypeAdminJob: cfReactor.feedStateManager.PushAdminJob(job.adminJob) case ownerJobTypeManualSchedule: - cfReactor.scheduler.MoveTable(job.tableID, job.targetCaptureID) + cfReactor.scheduler.MoveKeySpan(job.keyspanID, job.targetCaptureID) case ownerJobTypeRebalance: cfReactor.scheduler.Rebalance() case ownerJobTypeQuery: diff --git a/cdc/cdc/owner/owner_test.go b/cdc/cdc/owner/owner_test.go index 5ca5b6dc..aac45fa3 100644 --- a/cdc/cdc/owner/owner_test.go +++ b/cdc/cdc/owner/owner_test.go @@ -54,13 +54,7 @@ func createOwner4Test(ctx cdcContext.Context, c *check.C) (*Owner, *orchestrator return safePoint, nil }, } - owner := NewOwner4Test(func(ctx cdcContext.Context, startTs uint64) (DDLPuller, error) { - return &mockDDLPuller{resolvedTs: startTs - 1}, nil - }, func() DDLSink { - return &mockDDLSink{} - }, - ctx.GlobalVars().PDClient, - ) + owner := NewOwner4Test(ctx.GlobalVars().PDClient) state := orchestrator.NewGlobalState() tester := orchestrator.NewReactorStateTester(c, state, nil) @@ -230,49 +224,6 @@ func (s *ownerSuite) TestFixChangefeedState(c *check.C) { c.Assert(owner.changefeeds[changefeedID].state.Info.State, check.Equals, model.StateStopped) } -func (s *ownerSuite) TestFixChangefeedSinkProtocol(c *check.C) { - defer testleak.AfterTest(c)() - ctx := cdcContext.NewBackendContext4Test(false) - owner, state, tester := createOwner4Test(ctx, c) - // We need to do bootstrap. - owner.bootstrapped = false - changefeedID := "test-changefeed" - // Unknown protocol. - changefeedInfo := &model.ChangeFeedInfo{ - State: model.StateNormal, - AdminJobType: model.AdminStop, - StartTs: oracle.GoTimeToTS(time.Now()), - CreatorVersion: "5.3.0", - SinkURI: "kafka://127.0.0.1:9092/ticdc-test2?protocol=random", - Config: &config.ReplicaConfig{ - Sink: &config.SinkConfig{Protocol: config.ProtocolDefault.String()}, - }, - } - changefeedStr, err := changefeedInfo.Marshal() - c.Assert(err, check.IsNil) - cdcKey := etcd.CDCKey{ - Tp: etcd.CDCKeyTypeChangefeedInfo, - ChangefeedID: changefeedID, - } - tester.MustUpdate(cdcKey.String(), []byte(changefeedStr)) - // For the first tick, we do a bootstrap, and it tries to fix the meta information. - _, err = owner.Tick(ctx, state) - tester.MustApplyPatches() - c.Assert(err, check.IsNil) - c.Assert(owner.bootstrapped, check.IsTrue) - c.Assert(owner.changefeeds, check.Not(check.HasKey), changefeedID) - - // Start tick normally. - _, err = owner.Tick(ctx, state) - tester.MustApplyPatches() - c.Assert(err, check.IsNil) - c.Assert(owner.changefeeds, check.HasKey, changefeedID) - // The meta information is fixed correctly. - c.Assert(owner.changefeeds[changefeedID].state.Info.SinkURI, - check.Equals, - "kafka://127.0.0.1:9092/ticdc-test2?protocol=open-protocol") -} - func (s *ownerSuite) TestCheckClusterVersion(c *check.C) { defer testleak.AfterTest(c)() ctx := cdcContext.NewBackendContext4Test(false) @@ -280,7 +231,7 @@ func (s *ownerSuite) TestCheckClusterVersion(c *check.C) { ctx, cancel := cdcContext.WithCancel(ctx) defer cancel() - tester.MustUpdate("/tidb/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", []byte(`{"id":"6bbc01c8-0605-4f86-a0f9-b3119109b225","address":"127.0.0.1:8300","version":"v6.0.0"}`)) + tester.MustUpdate("/tikv/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", []byte(`{"id":"6bbc01c8-0605-4f86-a0f9-b3119109b225","address":"127.0.0.1:8300","version":"v6.0.0"}`)) changefeedID := "test-changefeed" changefeedInfo := &model.ChangeFeedInfo{ @@ -301,7 +252,7 @@ func (s *ownerSuite) TestCheckClusterVersion(c *check.C) { c.Assert(err, check.IsNil) c.Assert(owner.changefeeds, check.Not(check.HasKey), changefeedID) - tester.MustUpdate("/tidb/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", + tester.MustUpdate("/tikv/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", []byte(`{"id":"6bbc01c8-0605-4f86-a0f9-b3119109b225","address":"127.0.0.1:8300","version":"`+ctx.GlobalVars().CaptureInfo.Version+`"}`)) // check the tick is not skipped and the changefeed will be handled normally @@ -349,7 +300,7 @@ func (s *ownerSuite) TestAdminJob(c *check.C) { tp: ownerJobTypeManualSchedule, changefeedID: "test-changefeed3", targetCaptureID: "test-caputre1", - tableID: 10, + keyspanID: 10, }, { tp: ownerJobTypeDebugInfo, debugInfoWriter: &buf, @@ -388,7 +339,7 @@ func (s *ownerSuite) TestUpdateGCSafePoint(c *check.C) { } changefeedID1 := "changefeed-test1" tester.MustUpdate( - fmt.Sprintf("/tidb/cdc/changefeed/info/%s", changefeedID1), + fmt.Sprintf("/tikv/cdc/changefeed/info/%s", changefeedID1), []byte(`{"config":{"cyclic-replication":{}},"state":"failed"}`)) tester.MustApplyPatches() state.Changefeeds[changefeedID1].PatchStatus( @@ -428,7 +379,7 @@ func (s *ownerSuite) TestUpdateGCSafePoint(c *check.C) { // add another changefeed, it must update GC safepoint. changefeedID2 := "changefeed-test2" tester.MustUpdate( - fmt.Sprintf("/tidb/cdc/changefeed/info/%s", changefeedID2), + fmt.Sprintf("/tikv/cdc/changefeed/info/%s", changefeedID2), []byte(`{"config":{"cyclic-replication":{}},"state":"normal"}`)) tester.MustApplyPatches() state.Changefeeds[changefeedID1].PatchStatus( @@ -553,3 +504,46 @@ WorkLoop: c.Assert(infos[cf1], check.NotNil) c.Assert(infos[cf2], check.IsNil) } + +func (s *ownerSuite) TestFixChangefeedSinkProtocol(c *check.C) { + defer testleak.AfterTest(c)() + ctx := cdcContext.NewBackendContext4Test(false) + owner, state, tester := createOwner4Test(ctx, c) + // We need to do bootstrap. + owner.bootstrapped = false + changefeedID := "test-changefeed" + // Unknown protocol. + changefeedInfo := &model.ChangeFeedInfo{ + State: model.StateNormal, + AdminJobType: model.AdminStop, + StartTs: oracle.GoTimeToTS(time.Now()), + CreatorVersion: "5.3.0", + SinkURI: "tikv://127.0.0.1:1234", + Config: &config.ReplicaConfig{ + Sink: &config.SinkConfig{Protocol: config.ProtocolDefault.String()}, + }, + } + changefeedStr, err := changefeedInfo.Marshal() + c.Assert(err, check.IsNil) + cdcKey := etcd.CDCKey{ + Tp: etcd.CDCKeyTypeChangefeedInfo, + ChangefeedID: changefeedID, + } + tester.MustUpdate(cdcKey.String(), []byte(changefeedStr)) + // For the first tick, we do a bootstrap, and it tries to fix the meta information. + _, err = owner.Tick(ctx, state) + tester.MustApplyPatches() + c.Assert(err, check.IsNil) + c.Assert(owner.bootstrapped, check.IsTrue) + c.Assert(owner.changefeeds, check.Not(check.HasKey), changefeedID) + + // Start tick normally. + _, err = owner.Tick(ctx, state) + tester.MustApplyPatches() + c.Assert(err, check.IsNil) + c.Assert(owner.changefeeds, check.HasKey, changefeedID) + // The meta information is fixed correctly. + c.Assert(owner.changefeeds[changefeedID].state.Info.SinkURI, + check.Equals, + "tikv://127.0.0.1:1234") +} diff --git a/cdc/cdc/owner/scheduler.go b/cdc/cdc/owner/scheduler.go index 459ac9f2..21389a03 100644 --- a/cdc/cdc/owner/scheduler.go +++ b/cdc/cdc/owner/scheduler.go @@ -14,6 +14,7 @@ package owner import ( + "fmt" "sync/atomic" "github.com/pingcap/errors" @@ -22,17 +23,19 @@ import ( pscheduler "github.com/tikv/migration/cdc/cdc/scheduler" "github.com/tikv/migration/cdc/pkg/config" "github.com/tikv/migration/cdc/pkg/context" + cdcContext "github.com/tikv/migration/cdc/pkg/context" cerror "github.com/tikv/migration/cdc/pkg/errors" "github.com/tikv/migration/cdc/pkg/orchestrator" "github.com/tikv/migration/cdc/pkg/p2p" + "github.com/tikv/migration/cdc/pkg/regionspan" "github.com/tikv/migration/cdc/pkg/version" "go.uber.org/zap" ) -// scheduler is an interface for scheduling tables. -// Since in our design, we do not record checkpoints per table, +// scheduler is an interface for scheduling keyspans. +// Since in our design, we do not record checkpoints per keyspan, // how we calculate the global watermarks (checkpoint-ts and resolved-ts) -// is heavily coupled with how tables are scheduled. +// is heavily coupled with how keyspans are scheduled. // That is why we have a scheduler interface that also reports the global watermarks. type scheduler interface { // Tick is called periodically from the owner, and returns @@ -40,12 +43,11 @@ type scheduler interface { Tick( ctx context.Context, state *orchestrator.ChangefeedReactorState, - currentTables []model.TableID, captures map[model.CaptureID]*model.CaptureInfo, ) (newCheckpointTs, newResolvedTs model.Ts, err error) - // MoveTable is used to trigger manual table moves. - MoveTable(tableID model.TableID, target model.CaptureID) + // MoveKeySpan is used to trigger manual keyspan moves. + MoveKeySpan(keyspanID model.KeySpanID, target model.CaptureID) // Rebalance is used to trigger manual workload rebalances. Rebalance() @@ -64,6 +66,10 @@ type schedulerV2 struct { handlerErrChs []<-chan error stats *schedulerStats + + currentKeySpansID []model.KeySpanID + currentKeySpans map[model.KeySpanID]regionspan.Span + updateCurrentKeySpans func(ctx cdcContext.Context) ([]model.KeySpanID, map[model.KeySpanID]regionspan.Span, error) } // NewSchedulerV2 creates a new schedulerV2 @@ -75,10 +81,11 @@ func NewSchedulerV2( messageRouter p2p.MessageRouter, ) (*schedulerV2, error) { ret := &schedulerV2{ - changeFeedID: changeFeedID, - messageServer: messageServer, - messageRouter: messageRouter, - stats: &schedulerStats{}, + changeFeedID: changeFeedID, + messageServer: messageServer, + messageRouter: messageRouter, + stats: &schedulerStats{}, + updateCurrentKeySpans: updateCurrentKeySpansImpl, } ret.BaseScheduleDispatcher = pscheduler.NewBaseScheduleDispatcher(changeFeedID, ret, checkpointTs) if err := ret.registerPeerMessageHandlers(ctx); err != nil { @@ -112,29 +119,43 @@ func newScheduler(ctx context.Context, startTs uint64) (scheduler, error) { func (s *schedulerV2) Tick( ctx context.Context, state *orchestrator.ChangefeedReactorState, - currentTables []model.TableID, + // currentKeySpans []model.KeySpanID, captures map[model.CaptureID]*model.CaptureInfo, ) (checkpoint, resolvedTs model.Ts, err error) { if err := s.checkForHandlerErrors(ctx); err != nil { return pscheduler.CheckpointCannotProceed, pscheduler.CheckpointCannotProceed, errors.Trace(err) } - return s.BaseScheduleDispatcher.Tick(ctx, state.Status.CheckpointTs, currentTables, captures) + s.currentKeySpansID, s.currentKeySpans, err = s.updateCurrentKeySpans(ctx) + if err != nil { + return pscheduler.CheckpointCannotProceed, pscheduler.CheckpointCannotProceed, errors.Trace(err) + } + log.Debug("current key spans ID", zap.Any("currentKeySpansID", s.currentKeySpansID)) + return s.BaseScheduleDispatcher.Tick(ctx, state.Status.CheckpointTs, s.currentKeySpansID, captures) } -func (s *schedulerV2) DispatchTable( +func (s *schedulerV2) DispatchKeySpan( ctx context.Context, changeFeedID model.ChangeFeedID, - tableID model.TableID, + keyspanID model.KeySpanID, captureID model.CaptureID, isDelete bool, ) (done bool, err error) { - topic := model.DispatchTableTopic(changeFeedID) - message := &model.DispatchTableMessage{ + topic := model.DispatchKeySpanTopic(changeFeedID) + message := &model.DispatchKeySpanMessage{ OwnerRev: ctx.GlobalVars().OwnerRevision, - ID: tableID, + ID: keyspanID, IsDelete: isDelete, } + if !isDelete { + message.Start = s.currentKeySpans[keyspanID].Start + message.End = s.currentKeySpans[keyspanID].End + fmt.Println(message.Start, message.End) + } + log.Debug("try to send message", + zap.String("topic", topic), + zap.Any("message", message)) + ok, err := s.trySendMessage(ctx, captureID, topic, message) if err != nil { return false, errors.Trace(err) @@ -234,12 +255,12 @@ func (s *schedulerV2) registerPeerMessageHandlers(ctx context.Context) (ret erro errCh, err := s.messageServer.SyncAddHandler( ctx, - model.DispatchTableResponseTopic(s.changeFeedID), - &model.DispatchTableResponseMessage{}, + model.DispatchKeySpanResponseTopic(s.changeFeedID), + &model.DispatchKeySpanResponseMessage{}, func(sender string, messageI interface{}) error { - message := messageI.(*model.DispatchTableResponseMessage) + message := messageI.(*model.DispatchKeySpanResponseMessage) s.stats.RecordDispatchResponse() - s.OnAgentFinishedTableOperation(sender, message.ID) + s.OnAgentFinishedKeySpanOperation(sender, message.ID) return nil }) if err != nil { @@ -287,7 +308,7 @@ func (s *schedulerV2) registerPeerMessageHandlers(ctx context.Context) (ret erro func (s *schedulerV2) deregisterPeerMessageHandlers(ctx context.Context) { err := s.messageServer.SyncRemoveHandler( ctx, - model.DispatchTableResponseTopic(s.changeFeedID)) + model.DispatchKeySpanResponseTopic(s.changeFeedID)) if err != nil { log.Error("failed to remove peer message handler", zap.Error(err)) } diff --git a/cdc/cdc/owner/scheduler_test.go b/cdc/cdc/owner/scheduler_test.go index 3f5fee8f..976adc96 100644 --- a/cdc/cdc/owner/scheduler_test.go +++ b/cdc/cdc/owner/scheduler_test.go @@ -27,6 +27,7 @@ import ( cdcContext "github.com/tikv/migration/cdc/pkg/context" "github.com/tikv/migration/cdc/pkg/orchestrator" "github.com/tikv/migration/cdc/pkg/p2p" + "github.com/tikv/migration/cdc/pkg/regionspan" "github.com/tikv/migration/cdc/pkg/version" ) @@ -72,6 +73,14 @@ func TestSchedulerBasics(t *testing.T) { mockOwnerNode.Router) require.NoError(t, err) + sched.updateCurrentKeySpans = func(ctx cdcContext.Context) ([]model.KeySpanID, map[model.KeySpanID]regionspan.Span, error) { + return []model.KeySpanID{1, 2, 3}, map[model.KeySpanID]regionspan.Span{ + 1: {Start: []byte{'1'}, End: []byte{'2'}}, + 2: {Start: []byte{'2'}, End: []byte{'3'}}, + 3: {Start: []byte{'3'}, End: []byte{'4'}}, + }, nil + } + for atomic.LoadInt64(&sched.stats.AnnounceSentCount) < numNodes { checkpointTs, resolvedTs, err := sched.Tick(ctx, &orchestrator.ChangefeedReactorState{ ID: "cf-1", @@ -79,7 +88,7 @@ func TestSchedulerBasics(t *testing.T) { ResolvedTs: 1000, CheckpointTs: 1000, }, - }, []model.TableID{1, 2, 3}, mockCaptures) + }, mockCaptures) require.NoError(t, err) require.Equal(t, pscheduler.CheckpointCannotProceed, checkpointTs) require.Equal(t, pscheduler.CheckpointCannotProceed, resolvedTs) @@ -97,8 +106,8 @@ func TestSchedulerBasics(t *testing.T) { t, mockOwnerNode.ID, mockCluster, - model.DispatchTableTopic("cf-1"), - &model.DispatchTableMessage{}) + model.DispatchKeySpanTopic("cf-1"), + &model.DispatchKeySpanMessage{}) for id, ch := range announceCh { var msg interface{} @@ -134,12 +143,12 @@ func TestSchedulerBasics(t *testing.T) { ResolvedTs: 1000, CheckpointTs: 1000, }, - }, []model.TableID{1, 2, 3}, mockCaptures) + }, mockCaptures) require.NoError(t, err) require.Equal(t, pscheduler.CheckpointCannotProceed, checkpointTs) require.Equal(t, pscheduler.CheckpointCannotProceed, resolvedTs) } - log.Info("Tables have been dispatched") + log.Info("KeySpans have been dispatched") for id, ch := range dispatchCh { var msg interface{} @@ -149,17 +158,17 @@ func TestSchedulerBasics(t *testing.T) { case msg = <-ch: } - require.IsType(t, &model.DispatchTableMessage{}, msg) - dispatchTableMessage := msg.(*model.DispatchTableMessage) - require.Equal(t, int64(1), dispatchTableMessage.OwnerRev) - require.False(t, dispatchTableMessage.IsDelete) - require.Contains(t, []model.TableID{1, 2, 3}, dispatchTableMessage.ID) + require.IsType(t, &model.DispatchKeySpanMessage{}, msg) + dispatchKeySpanMessage := msg.(*model.DispatchKeySpanMessage) + require.Equal(t, int64(1), dispatchKeySpanMessage.OwnerRev) + require.False(t, dispatchKeySpanMessage.IsDelete) + require.Contains(t, []model.KeySpanID{1, 2, 3}, dispatchKeySpanMessage.ID) _, err := mockCluster.Nodes[id].Router.GetClient(mockOwnerNode.ID).SendMessage( ctx, - model.DispatchTableResponseTopic("cf-1"), - &model.DispatchTableResponseMessage{ - ID: dispatchTableMessage.ID, + model.DispatchKeySpanResponseTopic("cf-1"), + &model.DispatchKeySpanResponseMessage{ + ID: dispatchKeySpanMessage.ID, }) require.NoError(t, err) } @@ -174,7 +183,7 @@ func TestSchedulerBasics(t *testing.T) { ResolvedTs: 1000, CheckpointTs: 1000, }, - }, []model.TableID{1, 2, 3}, mockCaptures) + }, mockCaptures) require.NoError(t, err) require.Equal(t, model.Ts(1000), checkpointTs) require.Equal(t, model.Ts(1000), resolvedTs) @@ -227,6 +236,14 @@ func TestSchedulerNoPeer(t *testing.T) { mockOwnerNode.Router) require.NoError(t, err) + sched.updateCurrentKeySpans = func(ctx cdcContext.Context) ([]model.KeySpanID, map[model.KeySpanID]regionspan.Span, error) { + return []model.KeySpanID{1, 2, 3}, map[model.KeySpanID]regionspan.Span{ + 1: {Start: []byte{'1'}, End: []byte{'2'}}, + 2: {Start: []byte{'2'}, End: []byte{'3'}}, + 3: {Start: []byte{'3'}, End: []byte{'4'}}, + }, nil + } + // Ticks the scheduler 10 times. It should not panic. for i := 0; i < 10; i++ { checkpointTs, resolvedTs, err := sched.Tick(ctx, &orchestrator.ChangefeedReactorState{ @@ -235,7 +252,7 @@ func TestSchedulerNoPeer(t *testing.T) { ResolvedTs: 1000, CheckpointTs: 1000, }, - }, []model.TableID{1, 2, 3}, mockCaptures) + }, mockCaptures) require.NoError(t, err) require.Equal(t, pscheduler.CheckpointCannotProceed, checkpointTs) require.Equal(t, pscheduler.CheckpointCannotProceed, resolvedTs) @@ -251,7 +268,7 @@ func TestSchedulerNoPeer(t *testing.T) { ResolvedTs: 1000, CheckpointTs: 1000, }, - }, []model.TableID{1, 2, 3}, mockCaptures) + }, mockCaptures) require.NoError(t, err) require.Equal(t, pscheduler.CheckpointCannotProceed, checkpointTs) require.Equal(t, pscheduler.CheckpointCannotProceed, resolvedTs) diff --git a/cdc/cdc/owner/scheduler_v1.go b/cdc/cdc/owner/scheduler_v1.go index c5ce6bf1..90e9ca20 100644 --- a/cdc/cdc/owner/scheduler_v1.go +++ b/cdc/cdc/owner/scheduler_v1.go @@ -19,67 +19,78 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/log" + "github.com/tikv/client-go/v2/tikv" "github.com/tikv/migration/cdc/cdc/model" schedulerv2 "github.com/tikv/migration/cdc/cdc/scheduler" cdcContext "github.com/tikv/migration/cdc/pkg/context" cerror "github.com/tikv/migration/cdc/pkg/errors" "github.com/tikv/migration/cdc/pkg/orchestrator" + "github.com/tikv/migration/cdc/pkg/regionspan" "go.uber.org/zap" ) type schedulerJobType string const ( - schedulerJobTypeAddTable schedulerJobType = "ADD" - schedulerJobTypeRemoveTable schedulerJobType = "REMOVE" + schedulerJobTypeAddKeySpan schedulerJobType = "ADD" + schedulerJobTypeRemoveKeySpan schedulerJobType = "REMOVE" ) type schedulerJob struct { - Tp schedulerJobType - TableID model.TableID + Tp schedulerJobType + KeySpanID model.KeySpanID + Start []byte + End []byte // if the operation is a delete operation, boundaryTs is checkpoint ts // if the operation is an add operation, boundaryTs is start ts BoundaryTs uint64 TargetCapture model.CaptureID } -type moveTableJob struct { - tableID model.TableID - target model.CaptureID +type moveKeySpanJob struct { + keyspanID model.KeySpanID + target model.CaptureID } type oldScheduler struct { - state *orchestrator.ChangefeedReactorState - currentTables []model.TableID - captures map[model.CaptureID]*model.CaptureInfo + state *orchestrator.ChangefeedReactorState + currentKeySpansID []model.KeySpanID + currentKeySpans map[model.KeySpanID]regionspan.Span + captures map[model.CaptureID]*model.CaptureInfo - moveTableTargets map[model.TableID]model.CaptureID - moveTableJobQueue []*moveTableJob + moveKeySpanTargets map[model.KeySpanID]model.CaptureID + moveKeySpanJobQueue []*moveKeySpanJob needRebalanceNextTick bool lastTickCaptureCount int + + updateCurrentKeySpans func(ctx cdcContext.Context) ([]model.KeySpanID, map[model.KeySpanID]regionspan.Span, error) } func newSchedulerV1() scheduler { return &schedulerV1CompatWrapper{&oldScheduler{ - moveTableTargets: make(map[model.TableID]model.CaptureID), + moveKeySpanTargets: make(map[model.KeySpanID]model.CaptureID), + updateCurrentKeySpans: updateCurrentKeySpansImpl, }} } -// Tick is the main function of scheduler. It dispatches tables to captures and handles move-table and rebalance events. +// Tick is the main function of scheduler. It dispatches keyspans to captures and handles move-keyspan and rebalance events. // Tick returns a bool representing whether the changefeed's state can be updated in this tick. -// The state can be updated only if all the tables which should be listened to have been dispatched to captures and no operations have been sent to captures in this tick. +// The state can be updated only if all the keyspans which should be listened to have been dispatched to captures and no operations have been sent to captures in this tick. func (s *oldScheduler) Tick( + ctx cdcContext.Context, state *orchestrator.ChangefeedReactorState, - currentTables []model.TableID, + // currentKeySpans []model.KeySpanID, captures map[model.CaptureID]*model.CaptureInfo, ) (shouldUpdateState bool, err error) { s.state = state - s.currentTables = currentTables - s.captures = captures + s.currentKeySpansID, s.currentKeySpans, err = s.updateCurrentKeySpans(ctx) + if err != nil { + return false, errors.Trace(err) + } s.cleanUpFinishedOperations() - pendingJob, err := s.syncTablesWithCurrentTables() + pendingJob, err := s.syncKeySpansWithCurrentKeySpans() if err != nil { return false, errors.Trace(err) } @@ -89,60 +100,60 @@ func (s *oldScheduler) Tick( } s.handleJobs(pendingJob) - // only if the pending job list is empty and no table is being rebalanced or moved, + // only if the pending job list is empty and no keyspan is being rebalanced or moved, // can the global resolved ts and checkpoint ts be updated shouldUpdateState = len(pendingJob) == 0 shouldUpdateState = s.rebalance() && shouldUpdateState - shouldUpdateStateInMoveTable, err := s.handleMoveTableJob() + shouldUpdateStateInMoveKeySpan, err := s.handleMoveKeySpanJob() if err != nil { return false, errors.Trace(err) } - shouldUpdateState = shouldUpdateStateInMoveTable && shouldUpdateState + shouldUpdateState = shouldUpdateStateInMoveKeySpan && shouldUpdateState s.lastTickCaptureCount = len(captures) return shouldUpdateState, nil } -func (s *oldScheduler) MoveTable(tableID model.TableID, target model.CaptureID) { - s.moveTableJobQueue = append(s.moveTableJobQueue, &moveTableJob{ - tableID: tableID, - target: target, +func (s *oldScheduler) MoveKeySpan(keyspanID model.KeySpanID, target model.CaptureID) { + s.moveKeySpanJobQueue = append(s.moveKeySpanJobQueue, &moveKeySpanJob{ + keyspanID: keyspanID, + target: target, }) } -// handleMoveTableJob handles the move table job add be MoveTable function -func (s *oldScheduler) handleMoveTableJob() (shouldUpdateState bool, err error) { +// handleMoveKeySpanJob handles the move keyspan job add be MoveKeySpan function +func (s *oldScheduler) handleMoveKeySpanJob() (shouldUpdateState bool, err error) { shouldUpdateState = true - if len(s.moveTableJobQueue) == 0 { + if len(s.moveKeySpanJobQueue) == 0 { return } - table2CaptureIndex, err := s.table2CaptureIndex() + keyspan2CaptureIndex, err := s.keyspan2CaptureIndex() if err != nil { return false, errors.Trace(err) } - for _, job := range s.moveTableJobQueue { - source, exist := table2CaptureIndex[job.tableID] + for _, job := range s.moveKeySpanJobQueue { + source, exist := keyspan2CaptureIndex[job.keyspanID] if !exist { return } - s.moveTableTargets[job.tableID] = job.target + s.moveKeySpanTargets[job.keyspanID] = job.target job := job shouldUpdateState = false - // for all move table job, here just remove the table from the source capture. - // and the table removed by this function will be added to target capture by syncTablesWithCurrentTables in the next tick. + // for all move keyspan job, here just remove the keyspan from the source capture. + // and the keyspan removed by this function will be added to target capture by syncKeySpansWithCurrentKeySpans in the next tick. s.state.PatchTaskStatus(source, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { if status == nil { - // the capture may be down, just skip remove this table + // the capture may be down, just skip remove this keyspan return status, false, nil } - if status.Operation != nil && status.Operation[job.tableID] != nil { - // skip removing this table to avoid the remove operation created by the rebalance function interfering with the operation created by another function + if status.Operation != nil && status.Operation[job.keyspanID] != nil { + // skip removing this keyspan to avoid the remove operation created by the rebalance function interfering with the operation created by another function return status, false, nil } - status.RemoveTable(job.tableID, s.state.Status.CheckpointTs, false) + status.RemoveKeySpan(job.keyspanID, s.state.Status.CheckpointTs, false) return status, true, nil }) } - s.moveTableJobQueue = nil + s.moveKeySpanJobQueue = nil return } @@ -150,27 +161,27 @@ func (s *oldScheduler) Rebalance() { s.needRebalanceNextTick = true } -func (s *oldScheduler) table2CaptureIndex() (map[model.TableID]model.CaptureID, error) { - table2CaptureIndex := make(map[model.TableID]model.CaptureID) +func (s *oldScheduler) keyspan2CaptureIndex() (map[model.KeySpanID]model.CaptureID, error) { + keyspan2CaptureIndex := make(map[model.KeySpanID]model.CaptureID) for captureID, taskStatus := range s.state.TaskStatuses { - for tableID := range taskStatus.Tables { - if preCaptureID, exist := table2CaptureIndex[tableID]; exist && preCaptureID != captureID { - return nil, cerror.ErrTableListenReplicated.GenWithStackByArgs(tableID, preCaptureID, captureID) + for keyspanID := range taskStatus.KeySpans { + if preCaptureID, exist := keyspan2CaptureIndex[keyspanID]; exist && preCaptureID != captureID { + return nil, cerror.ErrKeySpanListenReplicated.GenWithStackByArgs(keyspanID, preCaptureID, captureID) } - table2CaptureIndex[tableID] = captureID + keyspan2CaptureIndex[keyspanID] = captureID } - for tableID := range taskStatus.Operation { - if preCaptureID, exist := table2CaptureIndex[tableID]; exist && preCaptureID != captureID { - return nil, cerror.ErrTableListenReplicated.GenWithStackByArgs(tableID, preCaptureID, captureID) + for keyspanID := range taskStatus.Operation { + if preCaptureID, exist := keyspan2CaptureIndex[keyspanID]; exist && preCaptureID != captureID { + return nil, cerror.ErrKeySpanListenReplicated.GenWithStackByArgs(keyspanID, preCaptureID, captureID) } - table2CaptureIndex[tableID] = captureID + keyspan2CaptureIndex[keyspanID] = captureID } } - return table2CaptureIndex, nil + return keyspan2CaptureIndex, nil } // dispatchToTargetCaptures sets the TargetCapture of scheduler jobs -// If the TargetCapture of a job is not set, it chooses a capture with the minimum workload(minimum number of tables) +// If the TargetCapture of a job is not set, it chooses a capture with the minimum workload(minimum number of keyspans) // and sets the TargetCapture to the capture. func (s *oldScheduler) dispatchToTargetCaptures(pendingJobs []*schedulerJob) { workloads := make(map[model.CaptureID]uint64) @@ -188,18 +199,18 @@ func (s *oldScheduler) dispatchToTargetCaptures(pendingJobs []*schedulerJob) { for _, pendingJob := range pendingJobs { if pendingJob.TargetCapture == "" { - target, exist := s.moveTableTargets[pendingJob.TableID] + target, exist := s.moveKeySpanTargets[pendingJob.KeySpanID] if !exist { continue } pendingJob.TargetCapture = target - delete(s.moveTableTargets, pendingJob.TableID) + delete(s.moveKeySpanTargets, pendingJob.KeySpanID) continue } switch pendingJob.Tp { - case schedulerJobTypeAddTable: + case schedulerJobTypeAddKeySpan: workloads[pendingJob.TargetCapture] += 1 - case schedulerJobTypeRemoveTable: + case schedulerJobTypeRemoveKeySpan: workloads[pendingJob.TargetCapture] -= 1 default: log.Panic("Unreachable, please report a bug", @@ -233,38 +244,40 @@ func (s *oldScheduler) dispatchToTargetCaptures(pendingJobs []*schedulerJob) { } } -// syncTablesWithCurrentTables iterates all current tables to check whether it should be listened or not. -// this function will return schedulerJob to make sure all tables will be listened. -func (s *oldScheduler) syncTablesWithCurrentTables() ([]*schedulerJob, error) { +// syncKeySpansWithCurrentKeySpans iterates all current keyspans to check whether it should be listened or not. +// this function will return schedulerJob to make sure all keyspans will be listened. +func (s *oldScheduler) syncKeySpansWithCurrentKeySpans() ([]*schedulerJob, error) { var pendingJob []*schedulerJob - allTableListeningNow, err := s.table2CaptureIndex() + allKeySpanListeningNow, err := s.keyspan2CaptureIndex() if err != nil { return nil, errors.Trace(err) } globalCheckpointTs := s.state.Status.CheckpointTs - for _, tableID := range s.currentTables { - if _, exist := allTableListeningNow[tableID]; exist { - delete(allTableListeningNow, tableID) + for _, keyspanID := range s.currentKeySpansID { + if _, exist := allKeySpanListeningNow[keyspanID]; exist { + delete(allKeySpanListeningNow, keyspanID) continue } - // For each table which should be listened but is not, add an adding-table job to the pending job list + // For each keyspan which should be listened but is not, add an adding-keyspan job to the pending job list pendingJob = append(pendingJob, &schedulerJob{ - Tp: schedulerJobTypeAddTable, - TableID: tableID, + Tp: schedulerJobTypeAddKeySpan, + KeySpanID: keyspanID, + Start: s.currentKeySpans[keyspanID].Start, + End: s.currentKeySpans[keyspanID].End, BoundaryTs: globalCheckpointTs, }) } - // The remaining tables are the tables which should be not listened - tablesThatShouldNotBeListened := allTableListeningNow - for tableID, captureID := range tablesThatShouldNotBeListened { + // The remaining keyspans are the keyspans which should be not listened + keyspansThatShouldNotBeListened := allKeySpanListeningNow + for keyspanID, captureID := range keyspansThatShouldNotBeListened { opts := s.state.TaskStatuses[captureID].Operation - if opts != nil && opts[tableID] != nil && opts[tableID].Delete { - // the table is being removed, skip + if opts != nil && opts[keyspanID] != nil && opts[keyspanID].Delete { + // the keyspan is being removed, skip continue } pendingJob = append(pendingJob, &schedulerJob{ - Tp: schedulerJobTypeRemoveTable, - TableID: tableID, + Tp: schedulerJobTypeRemoveKeySpan, + KeySpanID: keyspanID, BoundaryTs: globalCheckpointTs, TargetCapture: captureID, }) @@ -277,26 +290,27 @@ func (s *oldScheduler) handleJobs(jobs []*schedulerJob) { job := job s.state.PatchTaskStatus(job.TargetCapture, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { switch job.Tp { - case schedulerJobTypeAddTable: + case schedulerJobTypeAddKeySpan: if status == nil { - // if task status is not found, we can just skip adding the adding-table operation, since this table will be added in the next tick + // if task status is not found, we can just skip adding the adding-keyspan operation, since this keyspan will be added in the next tick log.Warn("task status of the capture is not found, may be the capture is already down. specify a new capture and redo the job", zap.Any("job", job)) return status, false, nil } - status.AddTable(job.TableID, &model.TableReplicaInfo{ - StartTs: job.BoundaryTs, - MarkTableID: 0, // mark table ID will be set in processors + status.AddKeySpan(job.KeySpanID, &model.KeySpanReplicaInfo{ + StartTs: job.BoundaryTs, + Start: job.Start, + End: job.End, }, job.BoundaryTs) - case schedulerJobTypeRemoveTable: - failpoint.Inject("OwnerRemoveTableError", func() { - // just skip removing this table + case schedulerJobTypeRemoveKeySpan: + failpoint.Inject("OwnerRemoveKeySpanError", func() { + // just skip removing this keyspan failpoint.Return(status, false, nil) }) if status == nil { log.Warn("Task status of the capture is not found. Maybe the capture is already down. Specify a new capture and redo the job", zap.Any("job", job)) return status, false, nil } - status.RemoveTable(job.TableID, job.BoundaryTs, false) + status.RemoveKeySpan(job.KeySpanID, job.BoundaryTs, false) default: log.Panic("Unreachable, please report a bug", zap.Any("job", job)) } @@ -318,9 +332,9 @@ func (s *oldScheduler) cleanUpFinishedOperations() { if status == nil { return nil, changed, nil } - for tableID, operation := range status.Operation { + for keyspanID, operation := range status.Operation { if operation.Status == model.OperFinished { - delete(status.Operation, tableID) + delete(status.Operation, keyspanID) changed = true } } @@ -331,11 +345,11 @@ func (s *oldScheduler) cleanUpFinishedOperations() { func (s *oldScheduler) rebalance() (shouldUpdateState bool) { if !s.shouldRebalance() { - // if no table is rebalanced, we can update the resolved ts and checkpoint ts + // if no keyspan is rebalanced, we can update the resolved ts and checkpoint ts return true } - // we only support rebalance by table number for now - return s.rebalanceByTableNum() + // we only support rebalance by keyspan number for now + return s.rebalanceByKeySpanNum() } func (s *oldScheduler) shouldRebalance() bool { @@ -344,7 +358,7 @@ func (s *oldScheduler) shouldRebalance() bool { return true } if s.lastTickCaptureCount != len(s.captures) { - // a new capture online and no table distributed to the capture + // a new capture online and no keyspan distributed to the capture // or some captures offline return true } @@ -352,56 +366,90 @@ func (s *oldScheduler) shouldRebalance() bool { return false } -// rebalanceByTableNum removes tables from captures replicating an above-average number of tables. -// the removed table will be dispatched again by syncTablesWithCurrentTables function -func (s *oldScheduler) rebalanceByTableNum() (shouldUpdateState bool) { - totalTableNum := len(s.currentTables) +// rebalanceByKeySpanNum removes keyspans from captures replicating an above-average number of keyspans. +// the removed keyspan will be dispatched again by syncKeySpansWithCurrentKeySpans function +func (s *oldScheduler) rebalanceByKeySpanNum() (shouldUpdateState bool) { + totalKeySpanNum := len(s.currentKeySpans) captureNum := len(s.captures) - upperLimitPerCapture := int(math.Ceil(float64(totalTableNum) / float64(captureNum))) + upperLimitPerCapture := int(math.Ceil(float64(totalKeySpanNum) / float64(captureNum))) shouldUpdateState = true log.Info("Start rebalancing", zap.String("changefeed", s.state.ID), - zap.Int("table-num", totalTableNum), + zap.Int("keyspan-num", totalKeySpanNum), zap.Int("capture-num", captureNum), zap.Int("target-limit", upperLimitPerCapture)) for captureID, taskStatus := range s.state.TaskStatuses { - tableNum2Remove := len(taskStatus.Tables) - upperLimitPerCapture - if tableNum2Remove <= 0 { + keyspanNum2Remove := len(taskStatus.KeySpans) - upperLimitPerCapture + if keyspanNum2Remove <= 0 { continue } - // here we pick `tableNum2Remove` tables to delete, - // and then the removed tables will be dispatched by `syncTablesWithCurrentTables` function in the next tick - for tableID := range taskStatus.Tables { - tableID := tableID - if tableNum2Remove <= 0 { + // here we pick `keyspanNum2Remove` keyspans to delete, + // and then the removed keyspans will be dispatched by `syncKeySpansWithCurrentKeySpans` function in the next tick + for keyspanID := range taskStatus.KeySpans { + keyspanID := keyspanID + if keyspanNum2Remove <= 0 { break } shouldUpdateState = false s.state.PatchTaskStatus(captureID, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { if status == nil { - // the capture may be down, just skip remove this table + // the capture may be down, just skip remove this keyspan return status, false, nil } - if status.Operation != nil && status.Operation[tableID] != nil { - // skip remove this table to avoid the remove operation created by rebalance function to influence the operation created by other function + if status.Operation != nil && status.Operation[keyspanID] != nil { + // skip remove this keyspan to avoid the remove operation created by rebalance function to influence the operation created by other function return status, false, nil } - status.RemoveTable(tableID, s.state.Status.CheckpointTs, false) - log.Info("Rebalance: Move table", - zap.Int64("table-id", tableID), + status.RemoveKeySpan(keyspanID, s.state.Status.CheckpointTs, false) + log.Info("Rebalance: Move keyspan", + zap.Uint64("keyspan-id", keyspanID), zap.String("capture", captureID), zap.String("changefeed-id", s.state.ID)) return status, true, nil }) - tableNum2Remove-- + keyspanNum2Remove-- } } return } +func updateCurrentKeySpansImpl(ctx cdcContext.Context) ([]model.KeySpanID, map[model.KeySpanID]regionspan.Span, error) { + limit := -1 + tikvRequestMaxBackoff := 20000 + bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) + + regionCache := ctx.GlobalVars().RegionCache + regions, err := regionCache.BatchLoadRegionsWithKeyRange(bo, []byte{regionspan.RawKvStartKey}, []byte{regionspan.RawKvEndKey}, limit) + if err != nil { + return nil, nil, err + } + + currentKeySpans := map[model.KeySpanID]regionspan.Span{} + currentKeySpansID := make([]model.KeySpanID, 0, len(regions)) + for i, region := range regions { + startKey := region.StartKey() + endKey := region.EndKey() + + if i == 0 { + startKey = []byte{regionspan.RawKvStartKey} + } + if i == len(regions)-1 { + endKey = []byte{regionspan.RawKvEndKey} + } + + keyspan := regionspan.Span{Start: startKey, End: endKey} + id := region.GetID() + + currentKeySpans[id] = keyspan + currentKeySpansID = append(currentKeySpansID, id) + } + + return currentKeySpansID, currentKeySpans, nil +} + // schedulerV1CompatWrapper is used to wrap the old scheduler to // support the compatibility with the new scheduler. // It incorporates watermark calculations into the scheduler, which @@ -411,12 +459,13 @@ type schedulerV1CompatWrapper struct { } func (w *schedulerV1CompatWrapper) Tick( - _ cdcContext.Context, + ctx cdcContext.Context, state *orchestrator.ChangefeedReactorState, - currentTables []model.TableID, + // currentKeySpans []model.KeySpanID, captures map[model.CaptureID]*model.CaptureInfo, ) (newCheckpointTs, newResolvedTs model.Ts, err error) { - shouldUpdateState, err := w.inner.Tick(state, currentTables, captures) + + shouldUpdateState, err := w.inner.Tick(ctx, state, captures) if err != nil { return schedulerv2.CheckpointCannotProceed, schedulerv2.CheckpointCannotProceed, err } @@ -429,8 +478,8 @@ func (w *schedulerV1CompatWrapper) Tick( return checkpointTs, resolvedTs, nil } -func (w *schedulerV1CompatWrapper) MoveTable(tableID model.TableID, target model.CaptureID) { - w.inner.MoveTable(tableID, target) +func (w *schedulerV1CompatWrapper) MoveKeySpan(keyspanID model.KeySpanID, target model.CaptureID) { + w.inner.MoveKeySpan(keyspanID, target) } func (w *schedulerV1CompatWrapper) Rebalance() { diff --git a/cdc/cdc/owner/scheduler_v1_test.go b/cdc/cdc/owner/scheduler_v1_test.go index 3e0c1b27..aa7f425b 100644 --- a/cdc/cdc/owner/scheduler_v1_test.go +++ b/cdc/cdc/owner/scheduler_v1_test.go @@ -19,8 +19,10 @@ import ( "github.com/pingcap/check" "github.com/tikv/migration/cdc/cdc/model" + cdcContext "github.com/tikv/migration/cdc/pkg/context" "github.com/tikv/migration/cdc/pkg/etcd" "github.com/tikv/migration/cdc/pkg/orchestrator" + "github.com/tikv/migration/cdc/pkg/regionspan" "github.com/tikv/migration/cdc/pkg/util/testleak" ) @@ -57,10 +59,10 @@ func (s *schedulerSuite) addCapture(captureID model.CaptureID) { s.tester.MustApplyPatches() } -func (s *schedulerSuite) finishTableOperation(captureID model.CaptureID, tableIDs ...model.TableID) { +func (s *schedulerSuite) finishKeySpanOperation(captureID model.CaptureID, keyspanIDs ...model.KeySpanID) { s.state.PatchTaskStatus(captureID, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { - for _, tableID := range tableIDs { - status.Operation[tableID].Status = model.OperFinished + for _, keyspanID := range keyspanIDs { + status.Operation[keyspanID].Status = model.OperFinished } return status, true, nil }) @@ -68,11 +70,11 @@ func (s *schedulerSuite) finishTableOperation(captureID model.CaptureID, tableID if workload == nil { workload = make(model.TaskWorkload) } - for _, tableID := range tableIDs { - if s.state.TaskStatuses[captureID].Operation[tableID].Delete { - delete(workload, tableID) + for _, keyspanID := range keyspanIDs { + if s.state.TaskStatuses[captureID].Operation[keyspanID].Delete { + delete(workload, keyspanID) } else { - workload[tableID] = model.WorkloadInfo{ + workload[keyspanID] = model.WorkloadInfo{ Workload: 1, } } @@ -89,7 +91,11 @@ func (s *schedulerSuite) TestScheduleOneCapture(c *check.C) { captureID := "test-capture-0" s.addCapture(captureID) - _, _ = s.scheduler.Tick(s.state, []model.TableID{}, s.captures) + ctx := cdcContext.NewBackendContext4Test(false) + ctx, cancel := cdcContext.WithCancel(ctx) + defer cancel() + + _, _ = s.scheduler.Tick(ctx, s.state, s.captures) // Manually simulate the scenario where the corresponding key was deleted in the etcd key := &etcd.CDCKey{ @@ -104,56 +110,73 @@ func (s *schedulerSuite) TestScheduleOneCapture(c *check.C) { captureID = "test-capture-1" s.addCapture(captureID) - // add three tables - shouldUpdateState, err := s.scheduler.Tick(s.state, []model.TableID{1, 2, 3, 4}, s.captures) + // add three keyspans + s.scheduler.updateCurrentKeySpans = func(ctx cdcContext.Context) ([]model.KeySpanID, map[model.KeySpanID]regionspan.Span, error) { + return []model.KeySpanID{1, 2, 3, 4}, map[model.KeySpanID]regionspan.Span{ + 1: {Start: []byte{'1'}, End: []byte{'2'}}, + 2: {Start: []byte{'2'}, End: []byte{'3'}}, + 3: {Start: []byte{'3'}, End: []byte{'4'}}, + 4: {Start: []byte{'4'}, End: []byte{'5'}}, + }, nil + } + shouldUpdateState, err := s.scheduler.Tick(ctx, s.state, s.captures) // []model.KeySpanID{1, 2, 3, 4}, c.Assert(err, check.IsNil) c.Assert(shouldUpdateState, check.IsFalse) s.tester.MustApplyPatches() - c.Assert(s.state.TaskStatuses[captureID].Tables, check.DeepEquals, map[model.TableID]*model.TableReplicaInfo{ + c.Assert(s.state.TaskStatuses[captureID].KeySpans, check.DeepEquals, map[model.KeySpanID]*model.KeySpanReplicaInfo{ 1: {StartTs: 0}, 2: {StartTs: 0}, 3: {StartTs: 0}, 4: {StartTs: 0}, }) - c.Assert(s.state.TaskStatuses[captureID].Operation, check.DeepEquals, map[model.TableID]*model.TableOperation{ + c.Assert(s.state.TaskStatuses[captureID].Operation, check.DeepEquals, map[model.KeySpanID]*model.KeySpanOperation{ 1: {Delete: false, BoundaryTs: 0, Status: model.OperDispatched}, 2: {Delete: false, BoundaryTs: 0, Status: model.OperDispatched}, 3: {Delete: false, BoundaryTs: 0, Status: model.OperDispatched}, 4: {Delete: false, BoundaryTs: 0, Status: model.OperDispatched}, }) - shouldUpdateState, err = s.scheduler.Tick(s.state, []model.TableID{1, 2, 3, 4}, s.captures) + + shouldUpdateState, err = s.scheduler.Tick(ctx, s.state, s.captures) // []model.KeySpanID{1, 2, 3, 4}, + c.Assert(err, check.IsNil) c.Assert(shouldUpdateState, check.IsTrue) s.tester.MustApplyPatches() - // two tables finish adding operation - s.finishTableOperation(captureID, 2, 3) + // two keyspans finish adding operation + s.finishKeySpanOperation(captureID, 2, 3) - // remove table 1,2 and add table 4,5 - shouldUpdateState, err = s.scheduler.Tick(s.state, []model.TableID{3, 4, 5}, s.captures) + s.scheduler.updateCurrentKeySpans = func(ctx cdcContext.Context) ([]model.KeySpanID, map[model.KeySpanID]regionspan.Span, error) { + return []model.KeySpanID{3, 4, 5}, map[model.KeySpanID]regionspan.Span{ + 3: {Start: []byte{'3'}, End: []byte{'4'}}, + 4: {Start: []byte{'4'}, End: []byte{'5'}}, + 5: {Start: []byte{'5'}, End: []byte{'6'}}, + }, nil + } + // remove keyspan 1,2 and add keyspan 4,5 + shouldUpdateState, err = s.scheduler.Tick(ctx, s.state, s.captures) // []model.KeySpanID{3, 4, 5}, c.Assert(err, check.IsNil) c.Assert(shouldUpdateState, check.IsFalse) s.tester.MustApplyPatches() - c.Assert(s.state.TaskStatuses[captureID].Tables, check.DeepEquals, map[model.TableID]*model.TableReplicaInfo{ + c.Assert(s.state.TaskStatuses[captureID].KeySpans, check.DeepEquals, map[model.KeySpanID]*model.KeySpanReplicaInfo{ 3: {StartTs: 0}, 4: {StartTs: 0}, 5: {StartTs: 0}, }) - c.Assert(s.state.TaskStatuses[captureID].Operation, check.DeepEquals, map[model.TableID]*model.TableOperation{ + c.Assert(s.state.TaskStatuses[captureID].Operation, check.DeepEquals, map[model.KeySpanID]*model.KeySpanOperation{ 1: {Delete: true, BoundaryTs: 0, Status: model.OperDispatched}, 2: {Delete: true, BoundaryTs: 0, Status: model.OperDispatched}, 4: {Delete: false, BoundaryTs: 0, Status: model.OperDispatched}, 5: {Delete: false, BoundaryTs: 0, Status: model.OperDispatched}, }) - // move a non exist table to a non exist capture - s.scheduler.MoveTable(2, "fake-capture") - // move tables to a non exist capture - s.scheduler.MoveTable(3, "fake-capture") - s.scheduler.MoveTable(4, "fake-capture") - shouldUpdateState, err = s.scheduler.Tick(s.state, []model.TableID{3, 4, 5}, s.captures) + // move a non exist keyspan to a non exist capture + s.scheduler.MoveKeySpan(2, "fake-capture") + // move keyspans to a non exist capture + s.scheduler.MoveKeySpan(3, "fake-capture") + s.scheduler.MoveKeySpan(4, "fake-capture") + shouldUpdateState, err = s.scheduler.Tick(ctx, s.state, s.captures) // []model.KeySpanID{3, 4, 5}, c.Assert(err, check.IsNil) c.Assert(shouldUpdateState, check.IsFalse) s.tester.MustApplyPatches() - c.Assert(s.state.TaskStatuses[captureID].Tables, check.DeepEquals, map[model.TableID]*model.TableReplicaInfo{ + c.Assert(s.state.TaskStatuses[captureID].KeySpans, check.DeepEquals, map[model.KeySpanID]*model.KeySpanReplicaInfo{ 4: {StartTs: 0}, 5: {StartTs: 0}, }) - c.Assert(s.state.TaskStatuses[captureID].Operation, check.DeepEquals, map[model.TableID]*model.TableOperation{ + c.Assert(s.state.TaskStatuses[captureID].Operation, check.DeepEquals, map[model.KeySpanID]*model.KeySpanOperation{ 1: {Delete: true, BoundaryTs: 0, Status: model.OperDispatched}, 2: {Delete: true, BoundaryTs: 0, Status: model.OperDispatched}, 3: {Delete: true, BoundaryTs: 0, Status: model.OperDispatched}, @@ -162,124 +185,139 @@ func (s *schedulerSuite) TestScheduleOneCapture(c *check.C) { }) // finish all operations - s.finishTableOperation(captureID, 1, 2, 3, 4, 5) + s.finishKeySpanOperation(captureID, 1, 2, 3, 4, 5) - shouldUpdateState, err = s.scheduler.Tick(s.state, []model.TableID{3, 4, 5}, s.captures) + shouldUpdateState, err = s.scheduler.Tick(ctx, s.state, s.captures) // []model.KeySpanID{3, 4, 5}, c.Assert(err, check.IsNil) c.Assert(shouldUpdateState, check.IsTrue) s.tester.MustApplyPatches() - c.Assert(s.state.TaskStatuses[captureID].Tables, check.DeepEquals, map[model.TableID]*model.TableReplicaInfo{ + c.Assert(s.state.TaskStatuses[captureID].KeySpans, check.DeepEquals, map[model.KeySpanID]*model.KeySpanReplicaInfo{ 4: {StartTs: 0}, 5: {StartTs: 0}, }) - c.Assert(s.state.TaskStatuses[captureID].Operation, check.DeepEquals, map[model.TableID]*model.TableOperation{}) + c.Assert(s.state.TaskStatuses[captureID].Operation, check.DeepEquals, map[model.KeySpanID]*model.KeySpanOperation{}) - // table 3 is missing by expected, because the table was trying to move to a invalid capture - // and the move will failed, the table 3 will be add in next tick - shouldUpdateState, err = s.scheduler.Tick(s.state, []model.TableID{3, 4, 5}, s.captures) + // keyspan 3 is missing by expected, because the keyspan was trying to move to a invalid capture + // and the move will failed, the keyspan 3 will be add in next tick + shouldUpdateState, err = s.scheduler.Tick(ctx, s.state, s.captures) // []model.KeySpanID{3, 4, 5}, c.Assert(err, check.IsNil) c.Assert(shouldUpdateState, check.IsFalse) s.tester.MustApplyPatches() - c.Assert(s.state.TaskStatuses[captureID].Tables, check.DeepEquals, map[model.TableID]*model.TableReplicaInfo{ + c.Assert(s.state.TaskStatuses[captureID].KeySpans, check.DeepEquals, map[model.KeySpanID]*model.KeySpanReplicaInfo{ 4: {StartTs: 0}, 5: {StartTs: 0}, }) - c.Assert(s.state.TaskStatuses[captureID].Operation, check.DeepEquals, map[model.TableID]*model.TableOperation{}) + c.Assert(s.state.TaskStatuses[captureID].Operation, check.DeepEquals, map[model.KeySpanID]*model.KeySpanOperation{}) - shouldUpdateState, err = s.scheduler.Tick(s.state, []model.TableID{3, 4, 5}, s.captures) + shouldUpdateState, err = s.scheduler.Tick(ctx, s.state, s.captures) // []model.KeySpanID{3, 4, 5}, c.Assert(err, check.IsNil) c.Assert(shouldUpdateState, check.IsFalse) s.tester.MustApplyPatches() - c.Assert(s.state.TaskStatuses[captureID].Tables, check.DeepEquals, map[model.TableID]*model.TableReplicaInfo{ + c.Assert(s.state.TaskStatuses[captureID].KeySpans, check.DeepEquals, map[model.KeySpanID]*model.KeySpanReplicaInfo{ 3: {StartTs: 0}, 4: {StartTs: 0}, 5: {StartTs: 0}, }) - c.Assert(s.state.TaskStatuses[captureID].Operation, check.DeepEquals, map[model.TableID]*model.TableOperation{ + c.Assert(s.state.TaskStatuses[captureID].Operation, check.DeepEquals, map[model.KeySpanID]*model.KeySpanOperation{ 3: {Delete: false, BoundaryTs: 0, Status: model.OperDispatched}, }) } -func (s *schedulerSuite) TestScheduleMoveTable(c *check.C) { +func (s *schedulerSuite) TestScheduleMoveKeySpan(c *check.C) { defer testleak.AfterTest(c)() s.reset(c) captureID1 := "test-capture-1" captureID2 := "test-capture-2" s.addCapture(captureID1) - // add a table - shouldUpdateState, err := s.scheduler.Tick(s.state, []model.TableID{1}, s.captures) + ctx := cdcContext.NewBackendContext4Test(false) + ctx, cancel := cdcContext.WithCancel(ctx) + defer cancel() + + // add a keyspan + s.scheduler.updateCurrentKeySpans = func(ctx cdcContext.Context) ([]model.KeySpanID, map[model.KeySpanID]regionspan.Span, error) { + return []model.KeySpanID{1}, map[model.KeySpanID]regionspan.Span{ + 1: {Start: []byte{'1'}, End: []byte{'2'}}, + }, nil + } + shouldUpdateState, err := s.scheduler.Tick(ctx, s.state, s.captures) // []model.KeySpanID{1}, c.Assert(err, check.IsNil) c.Assert(shouldUpdateState, check.IsFalse) s.tester.MustApplyPatches() - c.Assert(s.state.TaskStatuses[captureID1].Tables, check.DeepEquals, map[model.TableID]*model.TableReplicaInfo{ + c.Assert(s.state.TaskStatuses[captureID1].KeySpans, check.DeepEquals, map[model.KeySpanID]*model.KeySpanReplicaInfo{ 1: {StartTs: 0}, }) - c.Assert(s.state.TaskStatuses[captureID1].Operation, check.DeepEquals, map[model.TableID]*model.TableOperation{ + c.Assert(s.state.TaskStatuses[captureID1].Operation, check.DeepEquals, map[model.KeySpanID]*model.KeySpanOperation{ 1: {Delete: false, BoundaryTs: 0, Status: model.OperDispatched}, }) - s.finishTableOperation(captureID1, 1) - shouldUpdateState, err = s.scheduler.Tick(s.state, []model.TableID{1}, s.captures) + s.finishKeySpanOperation(captureID1, 1) + shouldUpdateState, err = s.scheduler.Tick(ctx, s.state, s.captures) // []model.KeySpanID{1}, c.Assert(err, check.IsNil) c.Assert(shouldUpdateState, check.IsTrue) s.tester.MustApplyPatches() s.addCapture(captureID2) - // add a table - shouldUpdateState, err = s.scheduler.Tick(s.state, []model.TableID{1, 2}, s.captures) + // add a keyspan + s.scheduler.updateCurrentKeySpans = func(ctx cdcContext.Context) ([]model.KeySpanID, map[model.KeySpanID]regionspan.Span, error) { + return []model.KeySpanID{1, 2}, map[model.KeySpanID]regionspan.Span{ + 1: {Start: []byte{'1'}, End: []byte{'2'}}, + 2: {Start: []byte{'2'}, End: []byte{'3'}}, + }, nil + } + shouldUpdateState, err = s.scheduler.Tick(ctx, s.state, s.captures) // []model.KeySpanID{1, 2}, c.Assert(err, check.IsNil) c.Assert(shouldUpdateState, check.IsFalse) s.tester.MustApplyPatches() - c.Assert(s.state.TaskStatuses[captureID1].Tables, check.DeepEquals, map[model.TableID]*model.TableReplicaInfo{ + c.Assert(s.state.TaskStatuses[captureID1].KeySpans, check.DeepEquals, map[model.KeySpanID]*model.KeySpanReplicaInfo{ 1: {StartTs: 0}, }) - c.Assert(s.state.TaskStatuses[captureID1].Operation, check.DeepEquals, map[model.TableID]*model.TableOperation{}) - c.Assert(s.state.TaskStatuses[captureID2].Tables, check.DeepEquals, map[model.TableID]*model.TableReplicaInfo{ + c.Assert(s.state.TaskStatuses[captureID1].Operation, check.DeepEquals, map[model.KeySpanID]*model.KeySpanOperation{}) + c.Assert(s.state.TaskStatuses[captureID2].KeySpans, check.DeepEquals, map[model.KeySpanID]*model.KeySpanReplicaInfo{ 2: {StartTs: 0}, }) - c.Assert(s.state.TaskStatuses[captureID2].Operation, check.DeepEquals, map[model.TableID]*model.TableOperation{ + c.Assert(s.state.TaskStatuses[captureID2].Operation, check.DeepEquals, map[model.KeySpanID]*model.KeySpanOperation{ 2: {Delete: false, BoundaryTs: 0, Status: model.OperDispatched}, }) - s.finishTableOperation(captureID2, 2) + s.finishKeySpanOperation(captureID2, 2) - s.scheduler.MoveTable(2, captureID1) - shouldUpdateState, err = s.scheduler.Tick(s.state, []model.TableID{1, 2}, s.captures) + s.scheduler.MoveKeySpan(2, captureID1) + shouldUpdateState, err = s.scheduler.Tick(ctx, s.state, s.captures) // []model.KeySpanID{1, 2}, c.Assert(err, check.IsNil) c.Assert(shouldUpdateState, check.IsFalse) s.tester.MustApplyPatches() - c.Assert(s.state.TaskStatuses[captureID1].Tables, check.DeepEquals, map[model.TableID]*model.TableReplicaInfo{ + c.Assert(s.state.TaskStatuses[captureID1].KeySpans, check.DeepEquals, map[model.KeySpanID]*model.KeySpanReplicaInfo{ 1: {StartTs: 0}, }) - c.Assert(s.state.TaskStatuses[captureID1].Operation, check.DeepEquals, map[model.TableID]*model.TableOperation{}) - c.Assert(s.state.TaskStatuses[captureID2].Tables, check.DeepEquals, map[model.TableID]*model.TableReplicaInfo{}) - c.Assert(s.state.TaskStatuses[captureID2].Operation, check.DeepEquals, map[model.TableID]*model.TableOperation{ + c.Assert(s.state.TaskStatuses[captureID1].Operation, check.DeepEquals, map[model.KeySpanID]*model.KeySpanOperation{}) + c.Assert(s.state.TaskStatuses[captureID2].KeySpans, check.DeepEquals, map[model.KeySpanID]*model.KeySpanReplicaInfo{}) + c.Assert(s.state.TaskStatuses[captureID2].Operation, check.DeepEquals, map[model.KeySpanID]*model.KeySpanOperation{ 2: {Delete: true, BoundaryTs: 0, Status: model.OperDispatched}, }) - s.finishTableOperation(captureID2, 2) + s.finishKeySpanOperation(captureID2, 2) - shouldUpdateState, err = s.scheduler.Tick(s.state, []model.TableID{1, 2}, s.captures) + shouldUpdateState, err = s.scheduler.Tick(ctx, s.state, s.captures) // []model.KeySpanID{1, 2}, c.Assert(err, check.IsNil) c.Assert(shouldUpdateState, check.IsTrue) s.tester.MustApplyPatches() - c.Assert(s.state.TaskStatuses[captureID1].Tables, check.DeepEquals, map[model.TableID]*model.TableReplicaInfo{ + c.Assert(s.state.TaskStatuses[captureID1].KeySpans, check.DeepEquals, map[model.KeySpanID]*model.KeySpanReplicaInfo{ 1: {StartTs: 0}, }) - c.Assert(s.state.TaskStatuses[captureID1].Operation, check.DeepEquals, map[model.TableID]*model.TableOperation{}) - c.Assert(s.state.TaskStatuses[captureID2].Tables, check.DeepEquals, map[model.TableID]*model.TableReplicaInfo{}) - c.Assert(s.state.TaskStatuses[captureID2].Operation, check.DeepEquals, map[model.TableID]*model.TableOperation{}) + c.Assert(s.state.TaskStatuses[captureID1].Operation, check.DeepEquals, map[model.KeySpanID]*model.KeySpanOperation{}) + c.Assert(s.state.TaskStatuses[captureID2].KeySpans, check.DeepEquals, map[model.KeySpanID]*model.KeySpanReplicaInfo{}) + c.Assert(s.state.TaskStatuses[captureID2].Operation, check.DeepEquals, map[model.KeySpanID]*model.KeySpanOperation{}) - shouldUpdateState, err = s.scheduler.Tick(s.state, []model.TableID{1, 2}, s.captures) + shouldUpdateState, err = s.scheduler.Tick(ctx, s.state, s.captures) // []model.KeySpanID{1, 2}, c.Assert(err, check.IsNil) c.Assert(shouldUpdateState, check.IsFalse) s.tester.MustApplyPatches() - c.Assert(s.state.TaskStatuses[captureID1].Tables, check.DeepEquals, map[model.TableID]*model.TableReplicaInfo{ + c.Assert(s.state.TaskStatuses[captureID1].KeySpans, check.DeepEquals, map[model.KeySpanID]*model.KeySpanReplicaInfo{ 1: {StartTs: 0}, 2: {StartTs: 0}, }) - c.Assert(s.state.TaskStatuses[captureID1].Operation, check.DeepEquals, map[model.TableID]*model.TableOperation{ + c.Assert(s.state.TaskStatuses[captureID1].Operation, check.DeepEquals, map[model.KeySpanID]*model.KeySpanOperation{ 2: {Delete: false, BoundaryTs: 0, Status: model.OperDispatched}, }) - c.Assert(s.state.TaskStatuses[captureID2].Tables, check.DeepEquals, map[model.TableID]*model.TableReplicaInfo{}) - c.Assert(s.state.TaskStatuses[captureID2].Operation, check.DeepEquals, map[model.TableID]*model.TableOperation{}) + c.Assert(s.state.TaskStatuses[captureID2].KeySpans, check.DeepEquals, map[model.KeySpanID]*model.KeySpanReplicaInfo{}) + c.Assert(s.state.TaskStatuses[captureID2].Operation, check.DeepEquals, map[model.KeySpanID]*model.KeySpanOperation{}) } func (s *schedulerSuite) TestScheduleRebalance(c *check.C) { @@ -293,26 +331,40 @@ func (s *schedulerSuite) TestScheduleRebalance(c *check.C) { s.addCapture(captureID3) s.state.PatchTaskStatus(captureID1, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { - status.Tables = make(map[model.TableID]*model.TableReplicaInfo) - status.Tables[1] = &model.TableReplicaInfo{StartTs: 1} - status.Tables[2] = &model.TableReplicaInfo{StartTs: 1} - status.Tables[3] = &model.TableReplicaInfo{StartTs: 1} - status.Tables[4] = &model.TableReplicaInfo{StartTs: 1} - status.Tables[5] = &model.TableReplicaInfo{StartTs: 1} - status.Tables[6] = &model.TableReplicaInfo{StartTs: 1} + status.KeySpans = make(map[model.KeySpanID]*model.KeySpanReplicaInfo) + status.KeySpans[1] = &model.KeySpanReplicaInfo{StartTs: 1} + status.KeySpans[2] = &model.KeySpanReplicaInfo{StartTs: 1} + status.KeySpans[3] = &model.KeySpanReplicaInfo{StartTs: 1} + status.KeySpans[4] = &model.KeySpanReplicaInfo{StartTs: 1} + status.KeySpans[5] = &model.KeySpanReplicaInfo{StartTs: 1} + status.KeySpans[6] = &model.KeySpanReplicaInfo{StartTs: 1} return status, true, nil }) s.tester.MustApplyPatches() - // rebalance table - shouldUpdateState, err := s.scheduler.Tick(s.state, []model.TableID{1, 2, 3, 4, 5, 6}, s.captures) + ctx := cdcContext.NewBackendContext4Test(false) + ctx, cancel := cdcContext.WithCancel(ctx) + defer cancel() + + // rebalance keyspan + s.scheduler.updateCurrentKeySpans = func(ctx cdcContext.Context) ([]model.KeySpanID, map[model.KeySpanID]regionspan.Span, error) { + return []model.KeySpanID{1, 2, 3, 4, 5, 6}, map[model.KeySpanID]regionspan.Span{ + 1: {Start: []byte{'1'}, End: []byte{'1'}}, + 2: {Start: []byte{'2'}, End: []byte{'2'}}, + 3: {Start: []byte{'3'}, End: []byte{'3'}}, + 4: {Start: []byte{'4'}, End: []byte{'4'}}, + 5: {Start: []byte{'5'}, End: []byte{'5'}}, + 6: {Start: []byte{'6'}, End: []byte{'6'}}, + }, nil + } + shouldUpdateState, err := s.scheduler.Tick(ctx, s.state, s.captures) // []model.KeySpanID{1, 2, 3, 4, 5, 6}, c.Assert(err, check.IsNil) c.Assert(shouldUpdateState, check.IsFalse) s.tester.MustApplyPatches() - // 4 tables remove in capture 1, this 4 tables will be added to another capture in next tick - c.Assert(s.state.TaskStatuses[captureID1].Tables, check.HasLen, 2) - c.Assert(s.state.TaskStatuses[captureID2].Tables, check.HasLen, 0) - c.Assert(s.state.TaskStatuses[captureID3].Tables, check.HasLen, 0) + // 4 keyspans remove in capture 1, this 4 keyspans will be added to another capture in next tick + c.Assert(s.state.TaskStatuses[captureID1].KeySpans, check.HasLen, 2) + c.Assert(s.state.TaskStatuses[captureID2].KeySpans, check.HasLen, 0) + c.Assert(s.state.TaskStatuses[captureID3].KeySpans, check.HasLen, 0) s.state.PatchTaskStatus(captureID1, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { for _, opt := range status.Operation { @@ -323,35 +375,35 @@ func (s *schedulerSuite) TestScheduleRebalance(c *check.C) { s.state.PatchTaskWorkload(captureID1, func(workload model.TaskWorkload) (model.TaskWorkload, bool, error) { c.Assert(workload, check.IsNil) workload = make(model.TaskWorkload) - for tableID := range s.state.TaskStatuses[captureID1].Tables { - workload[tableID] = model.WorkloadInfo{Workload: 1} + for keyspanID := range s.state.TaskStatuses[captureID1].KeySpans { + workload[keyspanID] = model.WorkloadInfo{Workload: 1} } return workload, true, nil }) s.tester.MustApplyPatches() // clean finished operation - shouldUpdateState, err = s.scheduler.Tick(s.state, []model.TableID{1, 2, 3, 4, 5, 6}, s.captures) + shouldUpdateState, err = s.scheduler.Tick(ctx, s.state, s.captures) // []model.KeySpanID{1, 2, 3, 4, 5, 6}, c.Assert(err, check.IsNil) c.Assert(shouldUpdateState, check.IsTrue) s.tester.MustApplyPatches() - // 4 tables add to another capture in this tick + // 4 keyspans add to another capture in this tick c.Assert(s.state.TaskStatuses[captureID1].Operation, check.HasLen, 0) - // rebalance table - shouldUpdateState, err = s.scheduler.Tick(s.state, []model.TableID{1, 2, 3, 4, 5, 6}, s.captures) + // rebalance keyspan + shouldUpdateState, err = s.scheduler.Tick(ctx, s.state, s.captures) // []model.KeySpanID{1, 2, 3, 4, 5, 6}, c.Assert(err, check.IsNil) c.Assert(shouldUpdateState, check.IsFalse) s.tester.MustApplyPatches() - // 4 tables add to another capture in this tick - c.Assert(s.state.TaskStatuses[captureID1].Tables, check.HasLen, 2) - c.Assert(s.state.TaskStatuses[captureID2].Tables, check.HasLen, 2) - c.Assert(s.state.TaskStatuses[captureID3].Tables, check.HasLen, 2) - tableIDs := make(map[model.TableID]struct{}) + // 4 keyspans add to another capture in this tick + c.Assert(s.state.TaskStatuses[captureID1].KeySpans, check.HasLen, 2) + c.Assert(s.state.TaskStatuses[captureID2].KeySpans, check.HasLen, 2) + c.Assert(s.state.TaskStatuses[captureID3].KeySpans, check.HasLen, 2) + keyspanIDs := make(map[model.KeySpanID]struct{}) for _, status := range s.state.TaskStatuses { - for tableID := range status.Tables { - tableIDs[tableID] = struct{}{} + for keyspanID := range status.KeySpans { + keyspanIDs[keyspanID] = struct{}{} } } - c.Assert(tableIDs, check.DeepEquals, map[model.TableID]struct{}{1: {}, 2: {}, 3: {}, 4: {}, 5: {}, 6: {}}) + c.Assert(keyspanIDs, check.DeepEquals, map[model.KeySpanID]struct{}{1: {}, 2: {}, 3: {}, 4: {}, 5: {}, 6: {}}) } diff --git a/cdc/cdc/owner/schema.go b/cdc/cdc/owner/schema.go deleted file mode 100644 index 9716fdd8..00000000 --- a/cdc/cdc/owner/schema.go +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package owner - -import ( - "github.com/pingcap/errors" - "github.com/pingcap/log" - tidbkv "github.com/pingcap/tidb/kv" - timeta "github.com/pingcap/tidb/meta" - timodel "github.com/pingcap/tidb/parser/model" - "github.com/tikv/migration/cdc/cdc/entry" - "github.com/tikv/migration/cdc/cdc/kv" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/cyclic/mark" - "github.com/tikv/migration/cdc/pkg/filter" - "go.uber.org/zap" -) - -type schemaWrap4Owner struct { - schemaSnapshot *entry.SingleSchemaSnapshot - filter *filter.Filter - config *config.ReplicaConfig - - allPhysicalTablesCache []model.TableID - ddlHandledTs model.Ts -} - -func newSchemaWrap4Owner(kvStorage tidbkv.Storage, startTs model.Ts, config *config.ReplicaConfig) (*schemaWrap4Owner, error) { - var meta *timeta.Meta - if kvStorage != nil { - var err error - meta, err = kv.GetSnapshotMeta(kvStorage, startTs) - if err != nil { - return nil, errors.Trace(err) - } - } - schemaSnap, err := entry.NewSingleSchemaSnapshotFromMeta(meta, startTs, config.ForceReplicate) - if err != nil { - return nil, errors.Trace(err) - } - f, err := filter.NewFilter(config) - if err != nil { - return nil, errors.Trace(err) - } - return &schemaWrap4Owner{ - schemaSnapshot: schemaSnap, - filter: f, - config: config, - ddlHandledTs: startTs, - }, nil -} - -// AllPhysicalTables returns the table IDs of all tables and partition tables. -func (s *schemaWrap4Owner) AllPhysicalTables() []model.TableID { - if s.allPhysicalTablesCache != nil { - return s.allPhysicalTablesCache - } - tables := s.schemaSnapshot.Tables() - s.allPhysicalTablesCache = make([]model.TableID, 0, len(tables)) - for _, tblInfo := range tables { - if s.shouldIgnoreTable(tblInfo) { - continue - } - - if pi := tblInfo.GetPartitionInfo(); pi != nil { - for _, partition := range pi.Definitions { - s.allPhysicalTablesCache = append(s.allPhysicalTablesCache, partition.ID) - } - } else { - s.allPhysicalTablesCache = append(s.allPhysicalTablesCache, tblInfo.ID) - } - } - return s.allPhysicalTablesCache -} - -func (s *schemaWrap4Owner) HandleDDL(job *timodel.Job) error { - if job.BinlogInfo.FinishedTS <= s.ddlHandledTs { - return nil - } - s.allPhysicalTablesCache = nil - err := s.schemaSnapshot.HandleDDL(job) - if err != nil { - return errors.Trace(err) - } - s.ddlHandledTs = job.BinlogInfo.FinishedTS - return nil -} - -func (s *schemaWrap4Owner) IsIneligibleTableID(tableID model.TableID) bool { - return s.schemaSnapshot.IsIneligibleTableID(tableID) -} - -func (s *schemaWrap4Owner) BuildDDLEvent(job *timodel.Job) (*model.DDLEvent, error) { - ddlEvent := new(model.DDLEvent) - preTableInfo, err := s.schemaSnapshot.PreTableInfo(job) - if err != nil { - return nil, errors.Trace(err) - } - err = s.schemaSnapshot.FillSchemaName(job) - if err != nil { - return nil, errors.Trace(err) - } - ddlEvent.FromJob(job, preTableInfo) - return ddlEvent, nil -} - -func (s *schemaWrap4Owner) SinkTableInfos() []*model.SimpleTableInfo { - var sinkTableInfos []*model.SimpleTableInfo - for tableID := range s.schemaSnapshot.CloneTables() { - tblInfo, ok := s.schemaSnapshot.TableByID(tableID) - if !ok { - log.Panic("table not found for table ID", zap.Int64("tid", tableID)) - } - if s.shouldIgnoreTable(tblInfo) { - continue - } - dbInfo, ok := s.schemaSnapshot.SchemaByTableID(tableID) - if !ok { - log.Panic("schema not found for table ID", zap.Int64("tid", tableID)) - } - - // TODO separate function for initializing SimpleTableInfo - sinkTableInfo := new(model.SimpleTableInfo) - sinkTableInfo.Schema = dbInfo.Name.O - sinkTableInfo.TableID = tableID - sinkTableInfo.Table = tblInfo.TableName.Table - sinkTableInfo.ColumnInfo = make([]*model.ColumnInfo, len(tblInfo.Cols())) - for i, colInfo := range tblInfo.Cols() { - sinkTableInfo.ColumnInfo[i] = new(model.ColumnInfo) - sinkTableInfo.ColumnInfo[i].FromTiColumnInfo(colInfo) - } - sinkTableInfos = append(sinkTableInfos, sinkTableInfo) - } - return sinkTableInfos -} - -func (s *schemaWrap4Owner) shouldIgnoreTable(tableInfo *model.TableInfo) bool { - schemaName := tableInfo.TableName.Schema - tableName := tableInfo.TableName.Table - if s.filter.ShouldIgnoreTable(schemaName, tableName) { - return true - } - if s.config.Cyclic.IsEnabled() && mark.IsMarkTable(schemaName, tableName) { - // skip the mark table if cyclic is enabled - return true - } - if !tableInfo.IsEligible(s.config.ForceReplicate) { - log.Warn("skip ineligible table", zap.Int64("tid", tableInfo.ID), zap.Stringer("table", tableInfo.TableName)) - return true - } - return false -} diff --git a/cdc/cdc/owner/schema_test.go b/cdc/cdc/owner/schema_test.go deleted file mode 100644 index e7210c35..00000000 --- a/cdc/cdc/owner/schema_test.go +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package owner - -import ( - "sort" - - "github.com/pingcap/check" - timodel "github.com/pingcap/tidb/parser/model" - "github.com/pingcap/tidb/parser/mysql" - "github.com/tikv/client-go/v2/oracle" - "github.com/tikv/migration/cdc/cdc/entry" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/util/testleak" -) - -var _ = check.Suite(&schemaSuite{}) - -type schemaSuite struct{} - -func (s *schemaSuite) TestAllPhysicalTables(c *check.C) { - defer testleak.AfterTest(c)() - helper := entry.NewSchemaTestHelper(c) - defer helper.Close() - ver, err := helper.Storage().CurrentVersion(oracle.GlobalTxnScope) - c.Assert(err, check.IsNil) - schema, err := newSchemaWrap4Owner(helper.Storage(), ver.Ver, config.GetDefaultReplicaConfig()) - c.Assert(err, check.IsNil) - c.Assert(schema.AllPhysicalTables(), check.HasLen, 0) - // add normal table - job := helper.DDL2Job("create table test.t1(id int primary key)") - tableIDT1 := job.BinlogInfo.TableInfo.ID - c.Assert(schema.HandleDDL(job), check.IsNil) - c.Assert(schema.AllPhysicalTables(), check.DeepEquals, []model.TableID{tableIDT1}) - // add ineligible table - c.Assert(schema.HandleDDL(helper.DDL2Job("create table test.t2(id int)")), check.IsNil) - c.Assert(schema.AllPhysicalTables(), check.DeepEquals, []model.TableID{tableIDT1}) - // add partition table - job = helper.DDL2Job(`CREATE TABLE test.employees ( - id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, - fname VARCHAR(25) NOT NULL, - lname VARCHAR(25) NOT NULL, - store_id INT NOT NULL, - department_id INT NOT NULL - ) - - PARTITION BY RANGE(id) ( - PARTITION p0 VALUES LESS THAN (5), - PARTITION p1 VALUES LESS THAN (10), - PARTITION p2 VALUES LESS THAN (15), - PARTITION p3 VALUES LESS THAN (20) - )`) - c.Assert(schema.HandleDDL(job), check.IsNil) - expectedTableIDs := []model.TableID{tableIDT1} - for _, p := range job.BinlogInfo.TableInfo.GetPartitionInfo().Definitions { - expectedTableIDs = append(expectedTableIDs, p.ID) - } - sortTableIDs := func(tableIDs []model.TableID) { - sort.Slice(tableIDs, func(i, j int) bool { - return tableIDs[i] < tableIDs[j] - }) - } - sortTableIDs(expectedTableIDs) - sortTableIDs(schema.AllPhysicalTables()) - c.Assert(schema.AllPhysicalTables(), check.DeepEquals, expectedTableIDs) -} - -func (s *schemaSuite) TestIsIneligibleTableID(c *check.C) { - defer testleak.AfterTest(c)() - helper := entry.NewSchemaTestHelper(c) - defer helper.Close() - ver, err := helper.Storage().CurrentVersion(oracle.GlobalTxnScope) - c.Assert(err, check.IsNil) - schema, err := newSchemaWrap4Owner(helper.Storage(), ver.Ver, config.GetDefaultReplicaConfig()) - c.Assert(err, check.IsNil) - // add normal table - job := helper.DDL2Job("create table test.t1(id int primary key)") - tableIDT1 := job.BinlogInfo.TableInfo.ID - c.Assert(schema.HandleDDL(job), check.IsNil) - // add ineligible table - job = helper.DDL2Job("create table test.t2(id int)") - tableIDT2 := job.BinlogInfo.TableInfo.ID - c.Assert(schema.HandleDDL(job), check.IsNil) - c.Assert(schema.IsIneligibleTableID(tableIDT1), check.IsFalse) - c.Assert(schema.IsIneligibleTableID(tableIDT2), check.IsTrue) -} - -func (s *schemaSuite) TestBuildDDLEvent(c *check.C) { - defer testleak.AfterTest(c)() - helper := entry.NewSchemaTestHelper(c) - defer helper.Close() - ver, err := helper.Storage().CurrentVersion(oracle.GlobalTxnScope) - c.Assert(err, check.IsNil) - schema, err := newSchemaWrap4Owner(helper.Storage(), ver.Ver, config.GetDefaultReplicaConfig()) - c.Assert(err, check.IsNil) - // add normal table - job := helper.DDL2Job("create table test.t1(id int primary key)") - event, err := schema.BuildDDLEvent(job) - c.Assert(err, check.IsNil) - c.Assert(event, check.DeepEquals, &model.DDLEvent{ - StartTs: job.StartTS, - CommitTs: job.BinlogInfo.FinishedTS, - Query: "create table test.t1(id int primary key)", - Type: timodel.ActionCreateTable, - TableInfo: &model.SimpleTableInfo{ - Schema: "test", - Table: "t1", - TableID: job.TableID, - ColumnInfo: []*model.ColumnInfo{{Name: "id", Type: mysql.TypeLong}}, - }, - PreTableInfo: nil, - }) - c.Assert(schema.HandleDDL(job), check.IsNil) - job = helper.DDL2Job("ALTER TABLE test.t1 ADD COLUMN c1 CHAR(16) NOT NULL") - event, err = schema.BuildDDLEvent(job) - c.Assert(err, check.IsNil) - c.Assert(event, check.DeepEquals, &model.DDLEvent{ - StartTs: job.StartTS, - CommitTs: job.BinlogInfo.FinishedTS, - Query: "ALTER TABLE test.t1 ADD COLUMN c1 CHAR(16) NOT NULL", - Type: timodel.ActionAddColumn, - TableInfo: &model.SimpleTableInfo{ - Schema: "test", - Table: "t1", - TableID: job.TableID, - ColumnInfo: []*model.ColumnInfo{{Name: "id", Type: mysql.TypeLong}, {Name: "c1", Type: mysql.TypeString}}, - }, - PreTableInfo: &model.SimpleTableInfo{ - Schema: "test", - Table: "t1", - TableID: job.TableID, - ColumnInfo: []*model.ColumnInfo{{Name: "id", Type: mysql.TypeLong}}, - }, - }) -} - -func (s *schemaSuite) TestSinkTableInfos(c *check.C) { - defer testleak.AfterTest(c)() - helper := entry.NewSchemaTestHelper(c) - defer helper.Close() - ver, err := helper.Storage().CurrentVersion(oracle.GlobalTxnScope) - c.Assert(err, check.IsNil) - schema, err := newSchemaWrap4Owner(helper.Storage(), ver.Ver, config.GetDefaultReplicaConfig()) - c.Assert(err, check.IsNil) - // add normal table - job := helper.DDL2Job("create table test.t1(id int primary key)") - tableIDT1 := job.BinlogInfo.TableInfo.ID - c.Assert(schema.HandleDDL(job), check.IsNil) - // add ineligible table - job = helper.DDL2Job("create table test.t2(id int)") - c.Assert(schema.HandleDDL(job), check.IsNil) - c.Assert(schema.SinkTableInfos(), check.DeepEquals, []*model.SimpleTableInfo{ - { - Schema: "test", - Table: "t1", - TableID: tableIDT1, - ColumnInfo: []*model.ColumnInfo{{Name: "id", Type: mysql.TypeLong}}, - }, - }) -} diff --git a/cdc/cdc/processor/agent.go b/cdc/cdc/processor/agent.go index 67bc5c89..4d019c82 100644 --- a/cdc/cdc/processor/agent.go +++ b/cdc/cdc/processor/agent.go @@ -80,7 +80,7 @@ func newAgent( ctx context.Context, messageServer *p2p.MessageServer, messageRouter p2p.MessageRouter, - executor scheduler.TableExecutor, + executor scheduler.KeySpanExecutor, changeFeedID model.ChangeFeedID, ) (processorAgent, error) { ret := &agentImpl{ @@ -126,7 +126,7 @@ func newAgent( } // We tolerate the situation where there is no owner. // If we are registered in Etcd, an elected Owner will have to - // contact us before it can schedule any table. + // contact us before it can schedule any keyspan. log.Info("no owner found. We will wait for an owner to contact us.", zap.String("changefeed-id", changeFeedID), zap.Error(err)) @@ -158,14 +158,14 @@ func (a *agentImpl) Tick(ctx context.Context) error { return nil } -func (a *agentImpl) FinishTableOperation( +func (a *agentImpl) FinishKeySpanOperation( ctx context.Context, - tableID model.TableID, + keyspanID model.KeySpanID, ) (bool, error) { done, err := a.trySendMessage( ctx, a.ownerCaptureID, - model.DispatchTableResponseTopic(a.changeFeed), - &model.DispatchTableResponseMessage{ID: tableID}) + model.DispatchKeySpanResponseTopic(a.changeFeed), + &model.DispatchKeySpanResponseMessage{ID: keyspanID}) if err != nil { return false, errors.Trace(err) } @@ -174,7 +174,7 @@ func (a *agentImpl) FinishTableOperation( func (a *agentImpl) SyncTaskStatuses( ctx context.Context, - running, adding, removing []model.TableID, + running, adding, removing []model.KeySpanID, ) (bool, error) { done, err := a.trySendMessage( ctx, @@ -235,7 +235,7 @@ func (a *agentImpl) Barrier(_ context.Context) (done bool) { if a.ownerCaptureID == "" { // We should wait for the first owner to contact us. // We need to wait for the sync request anyways, and - // there would not be any table to replicate for now. + // there would not be any keyspan to replicate for now. log.Debug("waiting for owner to request sync", zap.String("changefeed-id", a.changeFeed)) return false @@ -330,15 +330,17 @@ func (a *agentImpl) registerPeerMessageHandlers() (ret error) { errCh, err := a.messageServer.SyncAddHandler( ctx, - model.DispatchTableTopic(a.changeFeed), - &model.DispatchTableMessage{}, + model.DispatchKeySpanTopic(a.changeFeed), + &model.DispatchKeySpanMessage{}, func(sender string, value interface{}) error { ownerCapture := sender - message := value.(*model.DispatchTableMessage) + message := value.(*model.DispatchKeySpanMessage) a.OnOwnerDispatchedTask( ownerCapture, message.OwnerRev, message.ID, + message.Start, + message.End, message.IsDelete) return nil }) @@ -370,7 +372,7 @@ func (a *agentImpl) deregisterPeerMessageHandlers() error { ctx, cancel := stdContext.WithTimeout(stdContext.Background(), messageHandlerOperationsTimeout) defer cancel() - err := a.messageServer.SyncRemoveHandler(ctx, model.DispatchTableTopic(a.changeFeed)) + err := a.messageServer.SyncRemoveHandler(ctx, model.DispatchKeySpanTopic(a.changeFeed)) if err != nil { return errors.Trace(err) } diff --git a/cdc/cdc/processor/agent_test.go b/cdc/cdc/processor/agent_test.go index a544fd2a..5180af16 100644 --- a/cdc/cdc/processor/agent_test.go +++ b/cdc/cdc/processor/agent_test.go @@ -50,8 +50,8 @@ type agentTestSuite struct { etcdClient *clientv3.Client etcdKVClient *mockEtcdKVClient - tableExecutor *pscheduler.MockTableExecutor - dispatchResponseCh chan *model.DispatchTableResponseMessage + keyspanExecutor *pscheduler.MockKeySpanExecutor + dispatchResponseCh chan *model.DispatchKeySpanResponseMessage syncCh chan *model.SyncMessage checkpointCh chan *model.CheckpointMessage @@ -72,13 +72,13 @@ func newAgentTestSuite(t *testing.T) *agentTestSuite { ownerMessageClient := cluster.Nodes[ownerCaptureID].Router.GetClient(processorCaptureID) require.NotNil(t, ownerMessageClient) - dispatchResponseCh := make(chan *model.DispatchTableResponseMessage, 1) - _, err := ownerMessageServer.SyncAddHandler(ctx, model.DispatchTableResponseTopic("cf-1"), - &model.DispatchTableResponseMessage{}, + dispatchResponseCh := make(chan *model.DispatchKeySpanResponseMessage, 1) + _, err := ownerMessageServer.SyncAddHandler(ctx, model.DispatchKeySpanResponseTopic("cf-1"), + &model.DispatchKeySpanResponseMessage{}, func(senderID string, msg interface{}) error { require.Equal(t, processorCaptureID, senderID) - require.IsType(t, &model.DispatchTableResponseMessage{}, msg) - dispatchResponseCh <- msg.(*model.DispatchTableResponseMessage) + require.IsType(t, &model.DispatchKeySpanResponseMessage{}, msg) + dispatchResponseCh <- msg.(*model.DispatchKeySpanResponseMessage) return nil }, ) @@ -128,7 +128,7 @@ func (s *agentTestSuite) CreateAgent(t *testing.T) (*agentImpl, error) { cdcEtcdClient := etcd.NewCDCEtcdClient(s.ctx, s.etcdClient) messageServer := s.cluster.Nodes["capture-1"].Server messageRouter := s.cluster.Nodes["capture-1"].Router - s.tableExecutor = pscheduler.NewMockTableExecutor(t) + s.keyspanExecutor = pscheduler.NewMockKeySpanExecutor(t) ctx := cdcContext.NewContext(s.ctx, &cdcContext.GlobalVars{ EtcdClient: &cdcEtcdClient, @@ -137,7 +137,7 @@ func (s *agentTestSuite) CreateAgent(t *testing.T) (*agentImpl, error) { }) s.cdcCtx = ctx - ret, err := newAgent(ctx, messageServer, messageRouter, s.tableExecutor, "cf-1") + ret, err := newAgent(ctx, messageServer, messageRouter, s.keyspanExecutor, "cf-1") if err != nil { return nil, err } @@ -209,26 +209,26 @@ func TestAgentBasics(t *testing.T) { }, syncMsg) } - _, err = suite.ownerMessageClient.SendMessage(suite.ctx, model.DispatchTableTopic("cf-1"), &model.DispatchTableMessage{ + _, err = suite.ownerMessageClient.SendMessage(suite.ctx, model.DispatchKeySpanTopic("cf-1"), &model.DispatchKeySpanMessage{ OwnerRev: 1, ID: 1, IsDelete: false, }) require.NoError(t, err) - // Test Point 3: Accept an incoming DispatchTableMessage, and the AddTable method in TableExecutor can return false. - suite.tableExecutor.On("AddTable", mock.Anything, model.TableID(1)).Return(false, nil).Once() - suite.tableExecutor.On("AddTable", mock.Anything, model.TableID(1)).Return(true, nil).Run( + // Test Point 3: Accept an incoming DispatchKeySpanMessage, and the AddKeySpan method in KeySpanExecutor can return false. + suite.keyspanExecutor.On("AddKeySpan", mock.Anything, model.KeySpanID(1)).Return(false, nil).Once() + suite.keyspanExecutor.On("AddKeySpan", mock.Anything, model.KeySpanID(1)).Return(true, nil).Run( func(_ mock.Arguments) { - delete(suite.tableExecutor.Adding, 1) - suite.tableExecutor.Running[1] = struct{}{} + delete(suite.keyspanExecutor.Adding, 1) + suite.keyspanExecutor.Running[1] = struct{}{} }).Once() - suite.tableExecutor.On("GetCheckpoint").Return(model.Ts(1000), model.Ts(1000)) + suite.keyspanExecutor.On("GetCheckpoint").Return(model.Ts(1000), model.Ts(1000)) require.Eventually(t, func() bool { err = agent.Tick(suite.cdcCtx) require.NoError(t, err) - if len(suite.tableExecutor.Running) != 1 { + if len(suite.keyspanExecutor.Running) != 1 { return false } select { @@ -245,24 +245,24 @@ func TestAgentBasics(t *testing.T) { return false }, 5*time.Second, 100*time.Millisecond) - suite.tableExecutor.AssertExpectations(t) - suite.tableExecutor.ExpectedCalls = nil - suite.tableExecutor.Calls = nil + suite.keyspanExecutor.AssertExpectations(t) + suite.keyspanExecutor.ExpectedCalls = nil + suite.keyspanExecutor.Calls = nil - // Test Point 4: Accept an incoming DispatchTableMessage, and the AddTable method in TableExecutor can return true. + // Test Point 4: Accept an incoming DispatchKeySpanMessage, and the AddKeySpan method in KeySpanExecutor can return true. err = agent.Tick(suite.cdcCtx) require.NoError(t, err) - suite.tableExecutor.AssertExpectations(t) - suite.tableExecutor.ExpectedCalls = nil - suite.tableExecutor.Calls = nil + suite.keyspanExecutor.AssertExpectations(t) + suite.keyspanExecutor.ExpectedCalls = nil + suite.keyspanExecutor.Calls = nil require.Eventually(t, func() bool { select { case <-suite.ctx.Done(): return false case msg := <-suite.dispatchResponseCh: - require.Equal(t, &model.DispatchTableResponseMessage{ + require.Equal(t, &model.DispatchKeySpanResponseMessage{ ID: 1, }, msg) return true diff --git a/cdc/cdc/processor/doc.go b/cdc/cdc/processor/doc.go index 89c304f1..2c4a1da6 100644 --- a/cdc/cdc/processor/doc.go +++ b/cdc/cdc/processor/doc.go @@ -14,22 +14,22 @@ /* Package processor implements the processor logic based on ETCD worker(pkg/orchestrator). -There are three main modules: Manager, Processor and TablePipeline(cdc/processor/pipeline). +There are three main modules: Manager, Processor and KeySpanPipeline(cdc/processor/pipeline). The Manager's main responsibility is to maintain the Processor's life cycle, like create and destroy the processor instances. -The Processor's main responsibility is to maintain the TablePipeline's life cycle according to the state stored by ETCD, +The Processor's main responsibility is to maintain the KeySpanPipeline's life cycle according to the state stored by ETCD, and calculate the local resolved TS and local checkpoint Ts and put them into ETCD. -The TablePipeline listens to the kv change logs of a specified table(with its mark table if it exists), and sends logs to Sink After sorting and mounting. +The KeySpanPipeline listens to the kv change logs of a specified keyspan(with its mark keyspan if it exists), and sends logs to Sink After sorting and mounting. The relationship between the three module is as follows: -One Capture(with processor role) -> Processor Manager -> Processor(changefeed1) -> TablePipeline(tableA) +One Capture(with processor role) -> Processor Manager -> Processor(changefeed1) -> KeySpanPipeline(keyspanA) ╲ ╲ - ╲ -> TablePipeline(tableB) + ╲ -> KeySpanPipeline(keyspanB) ╲ ╲ - -> Processor(changefeed2) -> TablePipeline(tableC) + -> Processor(changefeed2) -> KeySpanPipeline(keyspanC) ╲ - -> TablePipeline(tableD) + -> KeySpanPipeline(keyspanD) */ diff --git a/cdc/cdc/processor/manager.go b/cdc/cdc/processor/manager.go index 6be1c6f6..009cccb2 100644 --- a/cdc/cdc/processor/manager.go +++ b/cdc/cdc/processor/manager.go @@ -98,9 +98,9 @@ func (m *Manager) Tick(stdCtx context.Context, state orchestrator.ReactorState) if changefeedState.Status.AdminJobType.IsStopState() || changefeedState.TaskStatuses[captureID].AdminJobType.IsStopState() { continue } - // the processor should start after at least one table has been added to this capture + // the processor should start after at least one keyspan has been added to this capture taskStatus := changefeedState.TaskStatuses[captureID] - if taskStatus == nil || (len(taskStatus.Tables) == 0 && len(taskStatus.Operation) == 0) { + if taskStatus == nil || (len(taskStatus.KeySpans) == 0 && len(taskStatus.Operation) == 0) { continue } failpoint.Inject("processorManagerHandleNewChangefeedDelay", nil) diff --git a/cdc/cdc/processor/manager_test.go b/cdc/cdc/processor/manager_test.go index 41ff50b4..302256c6 100644 --- a/cdc/cdc/processor/manager_test.go +++ b/cdc/cdc/processor/manager_test.go @@ -22,7 +22,7 @@ import ( "github.com/pingcap/check" "github.com/pingcap/errors" "github.com/tikv/migration/cdc/cdc/model" - tablepipeline "github.com/tikv/migration/cdc/cdc/processor/pipeline" + keyspanpipeline "github.com/tikv/migration/cdc/cdc/processor/pipeline" "github.com/tikv/migration/cdc/pkg/config" cdcContext "github.com/tikv/migration/cdc/pkg/context" cerrors "github.com/tikv/migration/cdc/pkg/errors" @@ -41,21 +41,21 @@ var _ = check.Suite(&managerSuite{}) // NewManager4Test creates a new processor manager for test func NewManager4Test( c *check.C, - createTablePipeline func(ctx cdcContext.Context, tableID model.TableID, replicaInfo *model.TableReplicaInfo) (tablepipeline.TablePipeline, error), + createKeySpanPipeline func(ctx cdcContext.Context, keyspanID model.KeySpanID, replicaInfo *model.KeySpanReplicaInfo) (keyspanpipeline.KeySpanPipeline, error), ) *Manager { m := NewManager() m.newProcessor = func(ctx cdcContext.Context) *processor { - return newProcessor4Test(ctx, c, createTablePipeline) + return newProcessor4Test(ctx, c, createKeySpanPipeline) } return m } func (s *managerSuite) resetSuit(ctx cdcContext.Context, c *check.C) { - s.manager = NewManager4Test(c, func(ctx cdcContext.Context, tableID model.TableID, replicaInfo *model.TableReplicaInfo) (tablepipeline.TablePipeline, error) { - return &mockTablePipeline{ - tableID: tableID, - name: fmt.Sprintf("`test`.`table%d`", tableID), - status: tablepipeline.TableStatusRunning, + s.manager = NewManager4Test(c, func(ctx cdcContext.Context, keyspanID model.KeySpanID, replicaInfo *model.KeySpanReplicaInfo) (keyspanpipeline.KeySpanPipeline, error) { + return &mockKeySpanPipeline{ + keyspanID: keyspanID, + name: fmt.Sprintf("`test`.`keyspan%d`", keyspanID), + status: keyspanpipeline.KeySpanStatusRunning, resolvedTs: replicaInfo.StartTs, checkpointTs: replicaInfo.StartTs, }, nil @@ -64,7 +64,7 @@ func (s *managerSuite) resetSuit(ctx cdcContext.Context, c *check.C) { captureInfoBytes, err := ctx.GlobalVars().CaptureInfo.Marshal() c.Assert(err, check.IsNil) s.tester = orchestrator.NewReactorStateTester(c, s.state, map[string]string{ - fmt.Sprintf("/tidb/cdc/capture/%s", ctx.GlobalVars().CaptureInfo.ID): string(captureInfoBytes), + fmt.Sprintf("/tikv/cdc/capture/%s", ctx.GlobalVars().CaptureInfo.ID): string(captureInfoBytes), }) } @@ -100,7 +100,7 @@ func (s *managerSuite) TestChangefeed(c *check.C) { }) s.state.Changefeeds["test-changefeed"].PatchTaskStatus(ctx.GlobalVars().CaptureInfo.ID, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { return &model.TaskStatus{ - Tables: map[int64]*model.TableReplicaInfo{1: {}}, + KeySpans: map[uint64]*model.KeySpanReplicaInfo{1: {}}, }, true, nil }) s.tester.MustApplyPatches() @@ -151,7 +151,7 @@ func (s *managerSuite) TestDebugInfo(c *check.C) { }) s.state.Changefeeds["test-changefeed"].PatchTaskStatus(ctx.GlobalVars().CaptureInfo.ID, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { return &model.TaskStatus{ - Tables: map[int64]*model.TableReplicaInfo{1: {}}, + KeySpans: map[uint64]*model.KeySpanReplicaInfo{1: {}}, }, true, nil }) s.tester.MustApplyPatches() @@ -205,7 +205,7 @@ func (s *managerSuite) TestClose(c *check.C) { }) s.state.Changefeeds["test-changefeed"].PatchTaskStatus(ctx.GlobalVars().CaptureInfo.ID, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { return &model.TaskStatus{ - Tables: map[int64]*model.TableReplicaInfo{1: {}}, + KeySpans: map[uint64]*model.KeySpanReplicaInfo{1: {}}, }, true, nil }) s.tester.MustApplyPatches() diff --git a/cdc/cdc/processor/metrics.go b/cdc/cdc/processor/metrics.go index b63d60fe..6ce91181 100644 --- a/cdc/cdc/processor/metrics.go +++ b/cdc/cdc/processor/metrics.go @@ -20,49 +20,49 @@ import ( var ( resolvedTsGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "processor", Name: "resolved_ts", Help: "local resolved ts of processor", }, []string{"changefeed", "capture"}) resolvedTsLagGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "processor", Name: "resolved_ts_lag", Help: "local resolved ts lag of processor", }, []string{"changefeed", "capture"}) checkpointTsGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "processor", Name: "checkpoint_ts", Help: "global checkpoint ts of processor", }, []string{"changefeed", "capture"}) checkpointTsLagGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "processor", Name: "checkpoint_ts_lag", Help: "global checkpoint ts lag of processor", }, []string{"changefeed", "capture"}) - syncTableNumGauge = prometheus.NewGaugeVec( + syncKeySpanNumGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "processor", - Name: "num_of_tables", - Help: "number of synchronized table of processor", + Name: "num_of_keyspans", + Help: "number of synchronized keyspan of processor", }, []string{"changefeed", "capture"}) processorErrorCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "processor", Name: "exit_with_error_count", Help: "counter for processor exits with error", }, []string{"changefeed", "capture"}) processorSchemaStorageGcTsGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "processor", Name: "schema_storage_gc_ts", Help: "the TS of the currently maintained oldest snapshot in SchemaStorage", @@ -75,7 +75,7 @@ func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(resolvedTsLagGauge) registry.MustRegister(checkpointTsGauge) registry.MustRegister(checkpointTsLagGauge) - registry.MustRegister(syncTableNumGauge) + registry.MustRegister(syncKeySpanNumGauge) registry.MustRegister(processorErrorCounter) registry.MustRegister(processorSchemaStorageGcTsGauge) } diff --git a/cdc/cdc/processor/pipeline/actor_node_context.go b/cdc/cdc/processor/pipeline/actor_node_context.go index 83805e32..1dd8a3cf 100644 --- a/cdc/cdc/processor/pipeline/actor_node_context.go +++ b/cdc/cdc/processor/pipeline/actor_node_context.go @@ -12,118 +12,3 @@ // limitations under the License. package pipeline - -import ( - sdtContext "context" - "sync/atomic" - - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/pkg/actor" - "github.com/tikv/migration/cdc/pkg/actor/message" - "github.com/tikv/migration/cdc/pkg/context" - "github.com/tikv/migration/cdc/pkg/pipeline" - "go.uber.org/zap" -) - -// send a tick message to actor if we get 32 pipeline messages -const messagesPerTick = 32 - -// actorNodeContext implements the NodeContext interface, with this we do not need -// to change too much logic to implement the table actor. -// the SendToNextNode buffer the pipeline message and tick the actor system -// the Throw function handle error and stop the actor -type actorNodeContext struct { - sdtContext.Context - outputCh chan pipeline.Message - tableActorRouter *actor.Router - tableActorID actor.ID - changefeedVars *context.ChangefeedVars - globalVars *context.GlobalVars - tickMessageThreshold int32 - // noTickMessageCount is the count of pipeline message that no tick message is sent to actor - noTickMessageCount int32 -} - -func NewContext(stdCtx sdtContext.Context, - tableActorRouter *actor.Router, - tableActorID actor.ID, - changefeedVars *context.ChangefeedVars, - globalVars *context.GlobalVars) *actorNodeContext { - return &actorNodeContext{ - Context: stdCtx, - outputCh: make(chan pipeline.Message, defaultOutputChannelSize), - tableActorRouter: tableActorRouter, - tableActorID: tableActorID, - changefeedVars: changefeedVars, - globalVars: globalVars, - tickMessageThreshold: messagesPerTick, - noTickMessageCount: 0, - } -} - -func (c *actorNodeContext) setTickMessageThreshold(threshold int32) { - atomic.StoreInt32(&c.tickMessageThreshold, threshold) -} - -func (c *actorNodeContext) GlobalVars() *context.GlobalVars { - return c.globalVars -} - -func (c *actorNodeContext) ChangefeedVars() *context.ChangefeedVars { - return c.changefeedVars -} - -func (c *actorNodeContext) Throw(err error) { - if err == nil { - return - } - log.Error("puller stopped", zap.Error(err)) - _ = c.tableActorRouter.SendB(c, c.tableActorID, message.StopMessage()) -} - -// SendToNextNode send msg to the outputCh and notify the actor system, -// to reduce the actor message, only send tick message per threshold -func (c *actorNodeContext) SendToNextNode(msg pipeline.Message) { - c.outputCh <- msg - c.trySendTickMessage() -} - -func (c *actorNodeContext) TrySendToNextNode(msg pipeline.Message) bool { - added := false - select { - case c.outputCh <- msg: - added = true - default: - } - if added { - c.trySendTickMessage() - } - return added -} - -func (c *actorNodeContext) Message() pipeline.Message { - return <-c.outputCh -} - -func (c *actorNodeContext) tryGetProcessedMessage() *pipeline.Message { - select { - case msg, ok := <-c.outputCh: - if !ok { - return nil - } - return &msg - default: - return nil - } -} - -func (c *actorNodeContext) trySendTickMessage() { - threshold := atomic.LoadInt32(&c.tickMessageThreshold) - atomic.AddInt32(&c.noTickMessageCount, 1) - count := atomic.LoadInt32(&c.noTickMessageCount) - // resolvedTs message will be sent by puller periodically - if count >= threshold { - _ = c.tableActorRouter.Send(c.tableActorID, message.TickMessage()) - atomic.StoreInt32(&c.noTickMessageCount, 0) - } -} diff --git a/cdc/cdc/processor/pipeline/actor_node_context_test.go b/cdc/cdc/processor/pipeline/actor_node_context_test.go index a3b435e2..1dd8a3cf 100644 --- a/cdc/cdc/processor/pipeline/actor_node_context_test.go +++ b/cdc/cdc/processor/pipeline/actor_node_context_test.go @@ -12,148 +12,3 @@ // limitations under the License. package pipeline - -import ( - sdtContext "context" - "testing" - "time" - - "github.com/pingcap/errors" - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/processor/pipeline/system" - "github.com/tikv/migration/cdc/pkg/actor" - "github.com/tikv/migration/cdc/pkg/actor/message" - "github.com/tikv/migration/cdc/pkg/context" - "github.com/tikv/migration/cdc/pkg/pipeline" -) - -func TestContext(t *testing.T) { - t.Parallel() - ctx := NewContext(sdtContext.TODO(), nil, 1, &context.ChangefeedVars{ID: "zzz"}, &context.GlobalVars{}) - require.NotNil(t, ctx.GlobalVars()) - require.Equal(t, "zzz", ctx.ChangefeedVars().ID) - require.Equal(t, actor.ID(1), ctx.tableActorID) - ctx.SendToNextNode(pipeline.BarrierMessage(1)) - require.Equal(t, int32(1), ctx.noTickMessageCount) - wait(t, 500*time.Millisecond, func() { - msg := ctx.Message() - require.Equal(t, pipeline.MessageTypeBarrier, msg.Tp) - }) -} - -func TestTryGetProcessedMessageFromChan(t *testing.T) { - t.Parallel() - ctx := NewContext(sdtContext.TODO(), nil, 1, nil, nil) - ctx.outputCh = make(chan pipeline.Message, 1) - require.Nil(t, ctx.tryGetProcessedMessage()) - ctx.outputCh <- pipeline.TickMessage() - require.NotNil(t, ctx.tryGetProcessedMessage()) - close(ctx.outputCh) - require.Nil(t, ctx.tryGetProcessedMessage()) -} - -func TestThrow(t *testing.T) { - t.Parallel() - ctx, cancel := sdtContext.WithCancel(sdtContext.TODO()) - sys := system.NewSystem() - defer func() { - cancel() - require.Nil(t, sys.Stop()) - }() - - require.Nil(t, sys.Start(ctx)) - actorID := sys.ActorID("abc", 1) - mb := actor.NewMailbox(actorID, defaultOutputChannelSize) - ch := make(chan message.Message, defaultOutputChannelSize) - fa := &forwardActor{ch: ch} - require.Nil(t, sys.System().Spawn(mb, fa)) - actorContext := NewContext(ctx, sys.Router(), actorID, nil, nil) - actorContext.Throw(nil) - time.Sleep(100 * time.Millisecond) - require.Equal(t, 0, len(ch)) - actorContext.Throw(errors.New("error")) - tick := time.After(500 * time.Millisecond) - select { - case <-tick: - t.Fatal("timeout") - case m := <-ch: - require.Equal(t, message.TypeStop, m.Tp) - } -} - -func TestActorNodeContextTrySendToNextNode(t *testing.T) { - t.Parallel() - ctx := NewContext(sdtContext.TODO(), nil, 1, &context.ChangefeedVars{ID: "zzz"}, &context.GlobalVars{}) - ctx.outputCh = make(chan pipeline.Message, 1) - require.True(t, ctx.TrySendToNextNode(pipeline.BarrierMessage(1))) - require.False(t, ctx.TrySendToNextNode(pipeline.BarrierMessage(1))) - ctx.outputCh = make(chan pipeline.Message, 1) - close(ctx.outputCh) - require.Panics(t, func() { ctx.TrySendToNextNode(pipeline.BarrierMessage(1)) }) -} - -func TestSendToNextNodeNoTickMessage(t *testing.T) { - t.Parallel() - ctx, cancel := sdtContext.WithCancel(sdtContext.TODO()) - sys := system.NewSystem() - defer func() { - cancel() - require.Nil(t, sys.Stop()) - }() - - require.Nil(t, sys.Start(ctx)) - actorID := sys.ActorID("abc", 1) - mb := actor.NewMailbox(actorID, defaultOutputChannelSize) - ch := make(chan message.Message, defaultOutputChannelSize) - fa := &forwardActor{ch: ch} - require.Nil(t, sys.System().Spawn(mb, fa)) - actorContext := NewContext(ctx, sys.Router(), actorID, nil, nil) - actorContext.setTickMessageThreshold(2) - actorContext.SendToNextNode(pipeline.BarrierMessage(1)) - time.Sleep(100 * time.Millisecond) - require.Equal(t, 0, len(ch)) - actorContext.SendToNextNode(pipeline.BarrierMessage(2)) - tick := time.After(500 * time.Millisecond) - select { - case <-tick: - t.Fatal("timeout") - case m := <-ch: - require.Equal(t, message.TypeTick, m.Tp) - } - actorContext.SendToNextNode(pipeline.BarrierMessage(1)) - time.Sleep(100 * time.Millisecond) - require.Equal(t, 0, len(ch)) -} - -type forwardActor struct { - contextAware bool - - ch chan<- message.Message -} - -func (f *forwardActor) Poll(ctx sdtContext.Context, msgs []message.Message) bool { - for _, msg := range msgs { - if f.contextAware { - select { - case f.ch <- msg: - case <-ctx.Done(): - } - } else { - f.ch <- msg - } - } - return true -} - -func wait(t *testing.T, timeout time.Duration, f func()) { - wait := make(chan int) - go func() { - f() - wait <- 0 - }() - select { - case <-wait: - case <-time.After(timeout): - t.Fatal("Timed out") - } -} diff --git a/cdc/cdc/processor/pipeline/cyclic_mark.go b/cdc/cdc/processor/pipeline/cyclic_mark.go deleted file mode 100644 index afb6bd57..00000000 --- a/cdc/cdc/processor/pipeline/cyclic_mark.go +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package pipeline - -import ( - "container/list" - - "github.com/pingcap/errors" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/entry" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/cyclic/mark" - "github.com/tikv/migration/cdc/pkg/pipeline" - "go.uber.org/zap" -) - -// cyclicMarkNode match the mark row events and normal row events. -// Set the ReplicaID of normal row events and filter the mark row events -// and filter the normal row events by the FilterReplicaID config item. -type cyclicMarkNode struct { - localReplicaID uint64 - filterReplicaID map[uint64]struct{} - markTableID model.TableID - - // startTs -> events - unknownReplicaIDEvents map[model.Ts][]*model.PolymorphicEvent - // startTs -> replicaID - currentReplicaIDs map[model.Ts]uint64 - currentCommitTs uint64 - - // todo : remove this flag after table actor is GA - isTableActorMode bool -} - -func newCyclicMarkNode(markTableID model.TableID) pipeline.Node { - return &cyclicMarkNode{ - markTableID: markTableID, - unknownReplicaIDEvents: make(map[model.Ts][]*model.PolymorphicEvent), - currentReplicaIDs: make(map[model.Ts]uint64), - } -} - -func (n *cyclicMarkNode) Init(ctx pipeline.NodeContext) error { - return n.InitTableActor(ctx.ChangefeedVars().Info.Config.Cyclic.ReplicaID, ctx.ChangefeedVars().Info.Config.Cyclic.FilterReplicaID, false) -} - -func (n *cyclicMarkNode) InitTableActor(localReplicaID uint64, filterReplicaID []uint64, isTableActorMode bool) error { - n.localReplicaID = localReplicaID - n.filterReplicaID = make(map[uint64]struct{}) - for _, rID := range filterReplicaID { - n.filterReplicaID[rID] = struct{}{} - } - n.isTableActorMode = isTableActorMode - // do nothing - return nil -} - -// Receive receives the message from the previous node. -// In the previous nodes(puller node and sorter node), -// the change logs of mark table and normal table are listen by one puller, -// and sorted by one sorter. -// So, this node will receive a commitTs-ordered stream -// which include the mark row events and normal row events. -// Under the above conditions, we need to cache at most one -// transaction's row events to matching row events. -// For every row event, Receive function flushes -// every the last transaction's row events, -// and adds the mark row event or normal row event into the cache. -func (n *cyclicMarkNode) Receive(ctx pipeline.NodeContext) error { - msg := ctx.Message() - _, err := n.TryHandleDataMessage(ctx, msg) - return err -} - -func (n *cyclicMarkNode) TryHandleDataMessage(ctx pipeline.NodeContext, msg pipeline.Message) (bool, error) { - // limit the queue size when the table actor mode is enabled - if n.isTableActorMode && ctx.(*cyclicNodeContext).queue.Len() >= defaultSyncResolvedBatch { - return false, nil - } - switch msg.Tp { - case pipeline.MessageTypePolymorphicEvent: - event := msg.PolymorphicEvent - n.flush(ctx, event.CRTs) - if event.RawKV.OpType == model.OpTypeResolved { - ctx.SendToNextNode(msg) - return true, nil - } - tableID, err := entry.DecodeTableID(event.RawKV.Key) - if err != nil { - return false, errors.Trace(err) - } - if tableID == n.markTableID { - n.appendMarkRowEvent(ctx, event) - } else { - n.appendNormalRowEvent(ctx, event) - } - return true, nil - } - ctx.SendToNextNode(msg) - return true, nil -} - -// appendNormalRowEvent adds the normal row into the cache. -func (n *cyclicMarkNode) appendNormalRowEvent(ctx pipeline.NodeContext, event *model.PolymorphicEvent) { - if event.CRTs != n.currentCommitTs { - log.Panic("the CommitTs of the received event is not equal to the currentCommitTs, please report a bug", zap.Reflect("event", event), zap.Uint64("currentCommitTs", n.currentCommitTs)) - } - if replicaID, exist := n.currentReplicaIDs[event.StartTs]; exist { - // we already know the replicaID of this startTs, it means that the mark row of this startTs is already in cached. - n.sendNormalRowEventToNextNode(ctx, replicaID, event) - return - } - // for all normal row events which we don't know the replicaID for now. we cache them in unknownReplicaIDEvents. - n.unknownReplicaIDEvents[event.StartTs] = append(n.unknownReplicaIDEvents[event.StartTs], event) -} - -// appendMarkRowEvent adds the mark row event into the cache. -func (n *cyclicMarkNode) appendMarkRowEvent(ctx pipeline.NodeContext, event *model.PolymorphicEvent) { - if event.CRTs != n.currentCommitTs { - log.Panic("the CommitTs of the received event is not equal to the currentCommitTs, please report a bug", zap.Reflect("event", event), zap.Uint64("currentCommitTs", n.currentCommitTs)) - } - markRow := event.Row - if markRow == nil { - return - } - replicaID := extractReplicaID(markRow) - // Establishing the mapping from StartTs to ReplicaID - n.currentReplicaIDs[markRow.StartTs] = replicaID - if events, exist := n.unknownReplicaIDEvents[markRow.StartTs]; exist { - // the replicaID of these events we did not know before, but now we know through received mark row now. - delete(n.unknownReplicaIDEvents, markRow.StartTs) - n.sendNormalRowEventToNextNode(ctx, replicaID, events...) - } -} - -func (n *cyclicMarkNode) flush(ctx pipeline.NodeContext, commitTs uint64) { - if n.currentCommitTs == commitTs { - return - } - // all mark events and normal events in current transaction is received now. - // there are still unmatched normal events in the cache, their replicaID should be local replicaID. - for _, events := range n.unknownReplicaIDEvents { - n.sendNormalRowEventToNextNode(ctx, n.localReplicaID, events...) - } - if len(n.unknownReplicaIDEvents) != 0 { - n.unknownReplicaIDEvents = make(map[model.Ts][]*model.PolymorphicEvent) - } - if len(n.currentReplicaIDs) != 0 { - n.currentReplicaIDs = make(map[model.Ts]uint64) - } - n.currentCommitTs = commitTs -} - -// sendNormalRowEventToNextNode filter the specified normal row events -// by the FilterReplicaID config item, and send events to the next node. -func (n *cyclicMarkNode) sendNormalRowEventToNextNode(ctx pipeline.NodeContext, replicaID uint64, events ...*model.PolymorphicEvent) { - if _, shouldFilter := n.filterReplicaID[replicaID]; shouldFilter { - return - } - for _, event := range events { - event.Row.ReplicaID = replicaID - ctx.SendToNextNode(pipeline.PolymorphicEventMessage(event)) - } -} - -func (n *cyclicMarkNode) Destroy(ctx pipeline.NodeContext) error { - // do nothing - return nil -} - -// extractReplicaID extracts replica ID from the given mark row. -func extractReplicaID(markRow *model.RowChangedEvent) uint64 { - for _, c := range markRow.Columns { - if c == nil { - continue - } - if c.Name == mark.CyclicReplicaIDCol { - return c.Value.(uint64) - } - } - log.Panic("bad mark table, " + mark.CyclicReplicaIDCol + " not found") - return 0 -} - -// cyclicNodeContext implements the NodeContext, cyclicMarkNode can be reused in table actor -// to buffer all messages with a queue, it will not block the actor system -type cyclicNodeContext struct { - *actorNodeContext - queue list.List -} - -func NewCyclicNodeContext(ctx *actorNodeContext) *cyclicNodeContext { - return &cyclicNodeContext{ - actorNodeContext: ctx, - } -} - -// SendToNextNode implement the NodeContext interface, push the message to a queue -// the queue size is limited by TryHandleDataMessage，size is defaultSyncResolvedBatch -func (c *cyclicNodeContext) SendToNextNode(msg pipeline.Message) { - c.queue.PushBack(msg) -} - -// Message implements the NodeContext -func (c *cyclicNodeContext) Message() pipeline.Message { - msg := c.tryGetProcessedMessage() - if msg != nil { - return *msg - } - return pipeline.Message{} -} - -func (c *cyclicNodeContext) tryGetProcessedMessage() *pipeline.Message { - el := c.queue.Front() - if el == nil { - return nil - } - msg := c.queue.Remove(el).(pipeline.Message) - return &msg -} diff --git a/cdc/cdc/processor/pipeline/cyclic_mark_test.go b/cdc/cdc/processor/pipeline/cyclic_mark_test.go deleted file mode 100644 index 23e48b56..00000000 --- a/cdc/cdc/processor/pipeline/cyclic_mark_test.go +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package pipeline - -import ( - "context" - "sort" - "sync" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/pingcap/tidb/tablecodec" - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/config" - cdcContext "github.com/tikv/migration/cdc/pkg/context" - "github.com/tikv/migration/cdc/pkg/cyclic/mark" - "github.com/tikv/migration/cdc/pkg/pipeline" -) - -func TestCyclicMarkNode(t *testing.T) { - markTableID := model.TableID(161025) - testCases := []struct { - input []*model.RowChangedEvent - expected []*model.RowChangedEvent - filterID []uint64 - replicaID uint64 - }{ - { - input: []*model.RowChangedEvent{}, - expected: []*model.RowChangedEvent{}, - filterID: []uint64{}, - replicaID: 1, - }, - { - input: []*model.RowChangedEvent{{Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 2}}, - expected: []*model.RowChangedEvent{{Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 2, ReplicaID: 1}}, - filterID: []uint64{}, - replicaID: 1, - }, - { - input: []*model.RowChangedEvent{ - {StartTs: 1, CommitTs: 2, Table: &model.TableName{Schema: "tidb_cdc", TableID: markTableID}, Columns: []*model.Column{{Name: mark.CyclicReplicaIDCol, Value: uint64(10)}}}, - }, - expected: []*model.RowChangedEvent{}, - filterID: []uint64{}, - replicaID: 1, - }, - { - input: []*model.RowChangedEvent{ - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 2}, - {StartTs: 1, CommitTs: 2, Table: &model.TableName{Schema: "tidb_cdc", TableID: markTableID}, Columns: []*model.Column{{Name: mark.CyclicReplicaIDCol, Value: uint64(10)}}}, - }, - expected: []*model.RowChangedEvent{}, - filterID: []uint64{10}, - replicaID: 1, - }, - { - input: []*model.RowChangedEvent{ - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 2}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 3, CommitTs: 2}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 2, CommitTs: 2}, - {StartTs: 3, CommitTs: 2, Table: &model.TableName{Schema: "tidb_cdc", TableID: markTableID}, Columns: []*model.Column{{Name: mark.CyclicReplicaIDCol, Value: uint64(10)}}}, - {StartTs: 1, CommitTs: 2, Table: &model.TableName{Schema: "tidb_cdc", TableID: markTableID}, Columns: []*model.Column{{Name: mark.CyclicReplicaIDCol, Value: uint64(11)}}}, - }, - expected: []*model.RowChangedEvent{ - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 2, ReplicaID: 11}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 2, CommitTs: 2, ReplicaID: 1}, - }, - filterID: []uint64{10}, - replicaID: 1, - }, - { - input: []*model.RowChangedEvent{ - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 2}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 3, CommitTs: 2}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 2, CommitTs: 2}, - {StartTs: 3, CommitTs: 2, Table: &model.TableName{Schema: "tidb_cdc", TableID: markTableID}, Columns: []*model.Column{{Name: mark.CyclicReplicaIDCol, Value: uint64(10)}}}, - {StartTs: 1, CommitTs: 5, Table: &model.TableName{Schema: "tidb_cdc", TableID: markTableID}, Columns: []*model.Column{{Name: mark.CyclicReplicaIDCol, Value: uint64(11)}}}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 5}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 3, CommitTs: 5}, - }, - expected: []*model.RowChangedEvent{ - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 2, ReplicaID: 1}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 2, CommitTs: 2, ReplicaID: 1}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 5, ReplicaID: 11}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 3, CommitTs: 5, ReplicaID: 1}, - }, - filterID: []uint64{10}, - replicaID: 1, - }, - { - input: []*model.RowChangedEvent{ - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 2}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 3, CommitTs: 2}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 2, CommitTs: 2}, - {StartTs: 3, CommitTs: 2, Table: &model.TableName{Schema: "tidb_cdc", TableID: markTableID}, Columns: []*model.Column{{Name: mark.CyclicReplicaIDCol, Value: uint64(10)}}}, - {StartTs: 1, CommitTs: 5, Table: &model.TableName{Schema: "tidb_cdc", TableID: markTableID}, Columns: []*model.Column{{Name: mark.CyclicReplicaIDCol, Value: uint64(11)}}}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 5}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 3, CommitTs: 5}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 5, CommitTs: 8}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 3, CommitTs: 8}, - {StartTs: 5, CommitTs: 8, Table: &model.TableName{Schema: "tidb_cdc", TableID: markTableID}, Columns: []*model.Column{{Name: mark.CyclicReplicaIDCol, Value: uint64(12)}}}, - }, - expected: []*model.RowChangedEvent{ - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 1, CommitTs: 2, ReplicaID: 1}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 2, CommitTs: 2, ReplicaID: 1}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 3, CommitTs: 5, ReplicaID: 1}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 3, CommitTs: 8, ReplicaID: 1}, - {Table: &model.TableName{Table: "a", TableID: 1}, StartTs: 5, CommitTs: 8, ReplicaID: 12}, - }, - filterID: []uint64{10, 11}, - replicaID: 1, - }, - } - - for _, tc := range testCases { - ctx := cdcContext.NewContext(context.Background(), &cdcContext.GlobalVars{}) - ctx = cdcContext.WithChangefeedVars(ctx, &cdcContext.ChangefeedVars{ - Info: &model.ChangeFeedInfo{ - Config: &config.ReplicaConfig{ - Cyclic: &config.CyclicConfig{ - Enable: true, - ReplicaID: tc.replicaID, - FilterReplicaID: tc.filterID, - }, - }, - }, - }) - n := newCyclicMarkNode(markTableID) - err := n.Init(pipeline.MockNodeContext4Test(ctx, pipeline.Message{}, nil)) - require.Nil(t, err) - outputCh := make(chan pipeline.Message) - var wg sync.WaitGroup - wg.Add(2) - go func() { - defer wg.Done() - defer close(outputCh) - var lastCommitTs model.Ts - for _, row := range tc.input { - event := model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypePut, - Key: tablecodec.GenTableRecordPrefix(row.Table.TableID), - StartTs: row.StartTs, - CRTs: row.CommitTs, - }) - event.Row = row - err := n.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.PolymorphicEventMessage(event), outputCh)) - require.Nil(t, err) - lastCommitTs = row.CommitTs - } - err := n.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.PolymorphicEventMessage(model.NewResolvedPolymorphicEvent(0, lastCommitTs+1)), outputCh)) - require.Nil(t, err) - }() - output := []*model.RowChangedEvent{} - go func() { - defer wg.Done() - for row := range outputCh { - if row.PolymorphicEvent.RawKV.OpType == model.OpTypeResolved { - continue - } - output = append(output, row.PolymorphicEvent.Row) - } - }() - wg.Wait() - // check the commitTs is increasing - var lastCommitTs model.Ts - for _, row := range output { - require.GreaterOrEqual(t, row.CommitTs, lastCommitTs) - // Ensure that the ReplicaID of the row is set correctly. - require.NotEqual(t, 0, row.ReplicaID) - lastCommitTs = row.CommitTs - } - sort.Slice(output, func(i, j int) bool { - if output[i].CommitTs == output[j].CommitTs { - return output[i].StartTs < output[j].StartTs - } - return output[i].CommitTs < output[j].CommitTs - }) - require.Equal(t, tc.expected, output, cmp.Diff(output, tc.expected)) - } - - // table actor - for _, tc := range testCases { - ctx := NewCyclicNodeContext(NewContext(context.TODO(), nil, 1, &cdcContext.ChangefeedVars{ - Info: &model.ChangeFeedInfo{ - Config: &config.ReplicaConfig{ - Cyclic: &config.CyclicConfig{ - Enable: true, - ReplicaID: tc.replicaID, - FilterReplicaID: tc.filterID, - }, - }, - }, - }, nil)) - n := newCyclicMarkNode(markTableID).(*cyclicMarkNode) - err := n.Init(ctx) - require.Nil(t, err) - output := []*model.RowChangedEvent{} - putToOutput := func(row *pipeline.Message) { - if row == nil || row.PolymorphicEvent.RawKV.OpType == model.OpTypeResolved { - return - } - output = append(output, row.PolymorphicEvent.Row) - } - - var lastCommitTs model.Ts - for _, row := range tc.input { - event := model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypePut, - Key: tablecodec.GenTableRecordPrefix(row.Table.TableID), - StartTs: row.StartTs, - CRTs: row.CommitTs, - }) - event.Row = row - ok, err := n.TryHandleDataMessage(ctx, pipeline.PolymorphicEventMessage(event)) - require.Nil(t, err) - require.True(t, ok) - putToOutput(ctx.tryGetProcessedMessage()) - lastCommitTs = row.CommitTs - } - ok, err := n.TryHandleDataMessage(ctx, pipeline.PolymorphicEventMessage(model.NewResolvedPolymorphicEvent(0, lastCommitTs+1))) - require.True(t, ok) - putToOutput(ctx.tryGetProcessedMessage()) - require.Nil(t, err) - for { - msg := ctx.tryGetProcessedMessage() - if msg == nil { - break - } - putToOutput(msg) - } - - // check the commitTs is increasing - lastCommitTs = 0 - for _, row := range output { - require.GreaterOrEqual(t, row.CommitTs, lastCommitTs) - // Ensure that the ReplicaID of the row is set correctly. - require.NotEqual(t, 0, row.ReplicaID) - lastCommitTs = row.CommitTs - } - sort.Slice(output, func(i, j int) bool { - if output[i].CommitTs == output[j].CommitTs { - return output[i].StartTs < output[j].StartTs - } - return output[i].CommitTs < output[j].CommitTs - }) - require.Equal(t, tc.expected, output, cmp.Diff(output, tc.expected)) - } -} diff --git a/cdc/cdc/processor/pipeline/keyspan.go b/cdc/cdc/processor/pipeline/keyspan.go new file mode 100644 index 00000000..1cd4c92c --- /dev/null +++ b/cdc/cdc/processor/pipeline/keyspan.go @@ -0,0 +1,203 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package pipeline + +import ( + "context" + "time" + + "github.com/pingcap/log" + "github.com/tikv/migration/cdc/cdc/model" + "github.com/tikv/migration/cdc/cdc/sink" + "github.com/tikv/migration/cdc/cdc/sink/common" + serverConfig "github.com/tikv/migration/cdc/pkg/config" + cdcContext "github.com/tikv/migration/cdc/pkg/context" + cerror "github.com/tikv/migration/cdc/pkg/errors" + "github.com/tikv/migration/cdc/pkg/pipeline" + "go.uber.org/zap" +) + +// KeySpanPipeline is a pipeline which capture the change log from tikv in a keyspan +type KeySpanPipeline interface { + // ID returns the ID of source keyspan and mark keyspan + ID() (keyspanID uint64) + // Name returns the quoted schema and keyspan name + Name() string + // ResolvedTs returns the resolved ts in this keyspan pipeline + ResolvedTs() model.Ts + // CheckpointTs returns the checkpoint ts in this keyspan pipeline + CheckpointTs() model.Ts + // UpdateBarrierTs updates the barrier ts in this keyspan pipeline + UpdateBarrierTs(ts model.Ts) + // AsyncStop tells the pipeline to stop, and returns true is the pipeline is already stopped. + AsyncStop(targetTs model.Ts) bool + // Workload returns the workload of this keyspan + Workload() model.WorkloadInfo + // Status returns the status of this keyspan pipeline + Status() KeySpanStatus + // Cancel stops this keyspan pipeline immediately and destroy all resources created by this keyspan pipeline + Cancel() + // Wait waits for keyspan pipeline destroyed + Wait() +} + +type keyspanPipelineImpl struct { + p *pipeline.Pipeline + + keyspanID uint64 + keyspanName string // quoted schema and keyspan, used in metircs only + + // sorterNode *sorterNode + sinkNode *sinkNode + cancel context.CancelFunc + + replConfig *serverConfig.ReplicaConfig +} + +// TODO find a better name or avoid using an interface +// We use an interface here for ease in unit testing. +type keyspanFlowController interface { + Consume(commitTs uint64, size uint64, blockCallBack func() error) error + Release(resolvedTs uint64) + Abort() + GetConsumption() uint64 +} + +// ResolvedTs returns the resolved ts in this keyspan pipeline +func (t *keyspanPipelineImpl) ResolvedTs() model.Ts { + // TODO: after TiCDC introduces p2p based resolved ts mechanism, TiCDC nodes + // will be able to cooperate replication status directly. Then we will add + // another replication barrier for consistent replication instead of reusing + // the global resolved-ts. + + return 0 +} + +// CheckpointTs returns the checkpoint ts in this keyspan pipeline +func (t *keyspanPipelineImpl) CheckpointTs() model.Ts { + return t.sinkNode.CheckpointTs() +} + +// UpdateBarrierTs updates the barrier ts in this keyspan pipeline +func (t *keyspanPipelineImpl) UpdateBarrierTs(ts model.Ts) { + err := t.p.SendToFirstNode(pipeline.BarrierMessage(ts)) + if err != nil && !cerror.ErrSendToClosedPipeline.Equal(err) && !cerror.ErrPipelineTryAgain.Equal(err) { + log.Panic("unexpect error from send to first node", zap.Error(err)) + } +} + +// AsyncStop tells the pipeline to stop, and returns true is the pipeline is already stopped. +func (t *keyspanPipelineImpl) AsyncStop(targetTs model.Ts) bool { + err := t.p.SendToFirstNode(pipeline.CommandMessage(&pipeline.Command{ + Tp: pipeline.CommandTypeStop, + })) + log.Info("send async stop signal to keyspan", zap.Uint64("keyspanID", t.keyspanID), zap.Uint64("targetTs", targetTs)) + if err != nil { + if cerror.ErrPipelineTryAgain.Equal(err) { + return false + } + if cerror.ErrSendToClosedPipeline.Equal(err) { + return true + } + log.Panic("unexpect error from send to first node", zap.Error(err)) + } + return true +} + +var workload = model.WorkloadInfo{Workload: 1} + +// Workload returns the workload of this keyspan +func (t *keyspanPipelineImpl) Workload() model.WorkloadInfo { + // TODO(leoppro) calculate the workload of this keyspan + // We temporarily set the value to constant 1 + return workload +} + +// Status returns the status of this keyspan pipeline +func (t *keyspanPipelineImpl) Status() KeySpanStatus { + return t.sinkNode.Status() +} + +// ID returns the ID of source keyspan and mark keyspan +// TODO: Maybe tikv cdc don't need markKeySpanID. +func (t *keyspanPipelineImpl) ID() (keyspanID uint64) { + return t.keyspanID +} + +// Name returns the quoted schema and keyspan name +func (t *keyspanPipelineImpl) Name() string { + return t.keyspanName +} + +// Cancel stops this keyspan pipeline immediately and destroy all resources created by this keyspan pipeline +func (t *keyspanPipelineImpl) Cancel() { + t.cancel() +} + +// Wait waits for keyspan pipeline destroyed +func (t *keyspanPipelineImpl) Wait() { + t.p.Wait() +} + +// Assume 1KB per row in upstream TiDB, it takes about 250 MB (1024*4*64) for +// replicating 1024 keyspans in the worst case. +const defaultOutputChannelSize = 64 + +// There are 4 or 5 runners in keyspan pipeline: header, puller, +// sink, cyclic if cyclic replication is enabled +const defaultRunnersSize = 3 + +// NewKeySpanPipeline creates a keyspan pipeline +// TODO(leoppro): implement a mock kvclient to test the keyspan pipeline +func NewKeySpanPipeline(ctx cdcContext.Context, + // mounter entry.Mounter, + keyspanID model.KeySpanID, + replicaInfo *model.KeySpanReplicaInfo, + sink sink.Sink, + targetTs model.Ts) KeySpanPipeline { + ctx, cancel := cdcContext.WithCancel(ctx) + replConfig := ctx.ChangefeedVars().Info.Config + keyspanPipeline := &keyspanPipelineImpl{ + keyspanID: keyspanID, + cancel: cancel, + replConfig: replConfig, + keyspanName: string(replicaInfo.Start) + "-" + string(replicaInfo.End), + } + + perKeySpanMemoryQuota := serverConfig.GetGlobalServerConfig().PerKeySpanMemoryQuota + + log.Debug("creating keyspan flow controller", + zap.String("changefeed-id", ctx.ChangefeedVars().ID), + zap.Uint64("quota", perKeySpanMemoryQuota)) + + flowController := common.NewKeySpanFlowController(perKeySpanMemoryQuota) + // config := ctx.ChangefeedVars().Info.Config + // cyclicEnabled := config.Cyclic != nil && config.Cyclic.IsEnabled() + runnerSize := defaultRunnersSize + /* + if cyclicEnabled { + runnerSize++ + } + */ + + p := pipeline.NewPipeline(ctx, 500*time.Millisecond, runnerSize, defaultOutputChannelSize) + sinkNode := newSinkNode(keyspanID, sink, replicaInfo.StartTs, targetTs, flowController) + + p.AppendNode(ctx, "puller", newPullerNode(keyspanID, replicaInfo)) + p.AppendNode(ctx, "sink", sinkNode) + + keyspanPipeline.p = p + keyspanPipeline.sinkNode = sinkNode + return keyspanPipeline +} diff --git a/cdc/cdc/processor/pipeline/main_test.go b/cdc/cdc/processor/pipeline/main_test.go index 46cd6b47..37b63048 100644 --- a/cdc/cdc/processor/pipeline/main_test.go +++ b/cdc/cdc/processor/pipeline/main_test.go @@ -17,12 +17,8 @@ import ( "testing" "github.com/tikv/migration/cdc/pkg/leakutil" - "go.uber.org/goleak" ) func TestMain(m *testing.M) { - leakutil.SetUpLeakTest( - m, - goleak.IgnoreTopFunction("github.com/tikv/migration/cdc/cdc/sorter/unified.newBackEndPool.func1"), - ) + leakutil.SetUpLeakTest(m) } diff --git a/cdc/cdc/processor/pipeline/metrics.go b/cdc/cdc/processor/pipeline/metrics.go index 86dff481..95b387cf 100644 --- a/cdc/cdc/processor/pipeline/metrics.go +++ b/cdc/cdc/processor/pipeline/metrics.go @@ -18,33 +18,33 @@ import ( ) var ( - tableResolvedTsGauge = prometheus.NewGaugeVec( + keyspanResolvedTsGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "processor", - Name: "table_resolved_ts", + Name: "keyspan_resolved_ts", Help: "local resolved ts of processor", - }, []string{"changefeed", "capture", "table"}) + }, []string{"changefeed", "capture", "keyspan"}) txnCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "processor", Name: "txn_count", Help: "txn count received/executed by this processor", }, []string{"type", "changefeed", "capture"}) - tableMemoryHistogram = prometheus.NewHistogramVec( + keyspanMemoryHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "processor", - Name: "table_memory_consumption", - Help: "estimated memory consumption for a table after the sorter", + Name: "keyspan_memory_consumption", + Help: "estimated memory consumption for a keyspan after the sorter", Buckets: prometheus.ExponentialBuckets(1*1024*1024 /* mb */, 2, 10), }, []string{"changefeed", "capture"}) ) // InitMetrics registers all metrics used in processor func InitMetrics(registry *prometheus.Registry) { - registry.MustRegister(tableResolvedTsGauge) + registry.MustRegister(keyspanResolvedTsGauge) registry.MustRegister(txnCounter) - registry.MustRegister(tableMemoryHistogram) + registry.MustRegister(keyspanMemoryHistogram) } diff --git a/cdc/cdc/processor/pipeline/puller.go b/cdc/cdc/processor/pipeline/puller.go index cc2026f3..2522ab4d 100644 --- a/cdc/cdc/processor/pipeline/puller.go +++ b/cdc/cdc/processor/pipeline/puller.go @@ -15,12 +15,12 @@ package pipeline import ( "context" + "strconv" "github.com/pingcap/errors" "github.com/tikv/client-go/v2/oracle" "github.com/tikv/migration/cdc/cdc/model" "github.com/tikv/migration/cdc/cdc/puller" - cdcContext "github.com/tikv/migration/cdc/pkg/context" "github.com/tikv/migration/cdc/pkg/pipeline" "github.com/tikv/migration/cdc/pkg/regionspan" "github.com/tikv/migration/cdc/pkg/util" @@ -28,32 +28,26 @@ import ( ) type pullerNode struct { - tableName string // quoted schema and table, used in metircs only + // keyspanName string // quoted schema and keyspan, used in metircs only - tableID model.TableID - replicaInfo *model.TableReplicaInfo + keyspanID model.KeySpanID + replicaInfo *model.KeySpanReplicaInfo cancel context.CancelFunc wg *errgroup.Group } func newPullerNode( - tableID model.TableID, replicaInfo *model.TableReplicaInfo, tableName string) pipeline.Node { + keyspanID model.KeySpanID, replicaInfo *model.KeySpanReplicaInfo) pipeline.Node { return &pullerNode{ - tableID: tableID, + keyspanID: keyspanID, replicaInfo: replicaInfo, - tableName: tableName, } } -func (n *pullerNode) tableSpan(ctx cdcContext.Context) []regionspan.Span { - // start table puller - config := ctx.ChangefeedVars().Info.Config - spans := make([]regionspan.Span, 0, 4) - spans = append(spans, regionspan.GetTableSpan(n.tableID)) - - if config.Cyclic.IsEnabled() && n.replicaInfo.MarkTableID != 0 { - spans = append(spans, regionspan.GetTableSpan(n.replicaInfo.MarkTableID)) - } +func (n *pullerNode) keyspan() []regionspan.Span { + // start keyspan puller + spans := make([]regionspan.Span, 0, 1) + spans = append(spans, regionspan.Span{Start: n.replicaInfo.Start, End: n.replicaInfo.End}) return spans } @@ -63,15 +57,14 @@ func (n *pullerNode) Init(ctx pipeline.NodeContext) error { func (n *pullerNode) InitWithWaitGroup(ctx pipeline.NodeContext, wg *errgroup.Group) error { n.wg = wg - metricTableResolvedTsGauge := tableResolvedTsGauge.WithLabelValues(ctx.ChangefeedVars().ID, ctx.GlobalVars().CaptureInfo.AdvertiseAddr, n.tableName) + metricKeySpanResolvedTsGauge := keyspanResolvedTsGauge.WithLabelValues(ctx.ChangefeedVars().ID, ctx.GlobalVars().CaptureInfo.AdvertiseAddr, strconv.FormatUint(n.keyspanID, 10)) ctxC, cancel := context.WithCancel(ctx) - ctxC = util.PutTableInfoInCtx(ctxC, n.tableID, n.tableName) + ctxC = util.PutKeySpanIDInCtx(ctxC, n.keyspanID) ctxC = util.PutCaptureAddrInCtx(ctxC, ctx.GlobalVars().CaptureInfo.AdvertiseAddr) ctxC = util.PutChangefeedIDInCtx(ctxC, ctx.ChangefeedVars().ID) - // NOTICE: always pull the old value internally - // See also: https://github.com/tikv/migration/cdc/issues/2301. + plr := puller.NewPuller(ctxC, ctx.GlobalVars().PDClient, ctx.GlobalVars().GrpcPool, ctx.GlobalVars().RegionCache, ctx.GlobalVars().KVStorage, - n.replicaInfo.StartTs, n.tableSpan(ctx), true) + n.replicaInfo.StartTs, n.keyspan(), true) n.wg.Go(func() error { ctx.Throw(errors.Trace(plr.Run(ctxC))) return nil @@ -85,8 +78,9 @@ func (n *pullerNode) InitWithWaitGroup(ctx pipeline.NodeContext, wg *errgroup.Gr if rawKV == nil { continue } + rawKV.KeySpanID = n.keyspanID if rawKV.OpType == model.OpTypeResolved { - metricTableResolvedTsGauge.Set(float64(oracle.ExtractPhysical(rawKV.CRTs))) + metricKeySpanResolvedTsGauge.Set(float64(oracle.ExtractPhysical(rawKV.CRTs))) } pEvent := model.NewPolymorphicEvent(rawKV) ctx.SendToNextNode(pipeline.PolymorphicEventMessage(pEvent)) @@ -105,7 +99,7 @@ func (n *pullerNode) Receive(ctx pipeline.NodeContext) error { } func (n *pullerNode) Destroy(ctx pipeline.NodeContext) error { - tableResolvedTsGauge.DeleteLabelValues(ctx.ChangefeedVars().ID, ctx.GlobalVars().CaptureInfo.AdvertiseAddr, n.tableName) + keyspanResolvedTsGauge.DeleteLabelValues(ctx.ChangefeedVars().ID, ctx.GlobalVars().CaptureInfo.AdvertiseAddr, strconv.FormatUint(n.keyspanID, 10)) n.cancel() return n.wg.Wait() } diff --git a/cdc/cdc/processor/pipeline/sink.go b/cdc/cdc/processor/pipeline/sink.go index d4e83247..ab6d079c 100755 --- a/cdc/cdc/processor/pipeline/sink.go +++ b/cdc/cdc/processor/pipeline/sink.go @@ -33,42 +33,42 @@ const ( defaultSyncResolvedBatch = 64 ) -// TableStatus is status of the table pipeline -type TableStatus int32 +// KeySpanStatus is status of the keyspan pipeline +type KeySpanStatus int32 -// TableStatus for table pipeline +// KeySpanStatus for keyspan pipeline const ( - TableStatusInitializing TableStatus = iota - TableStatusRunning - TableStatusStopped + KeySpanStatusInitializing KeySpanStatus = iota + KeySpanStatusRunning + KeySpanStatusStopped ) -func (s TableStatus) String() string { +func (s KeySpanStatus) String() string { switch s { - case TableStatusInitializing: + case KeySpanStatusInitializing: return "Initializing" - case TableStatusRunning: + case KeySpanStatusRunning: return "Running" - case TableStatusStopped: + case KeySpanStatusStopped: return "Stopped" } return "Unknown" } -// Load TableStatus with THREAD-SAFE -func (s *TableStatus) Load() TableStatus { - return TableStatus(atomic.LoadInt32((*int32)(s))) +// Load KeySpanStatus with THREAD-SAFE +func (s *KeySpanStatus) Load() KeySpanStatus { + return KeySpanStatus(atomic.LoadInt32((*int32)(s))) } -// Store TableStatus with THREAD-SAFE -func (s *TableStatus) Store(new TableStatus) { +// Store KeySpanStatus with THREAD-SAFE +func (s *KeySpanStatus) Store(new KeySpanStatus) { atomic.StoreInt32((*int32)(s), int32(new)) } type sinkNode struct { - sink sink.Sink - status TableStatus - tableID model.TableID + sink sink.Sink + status KeySpanStatus + keyspanID model.KeySpanID resolvedTs model.Ts checkpointTs model.Ts @@ -76,19 +76,19 @@ type sinkNode struct { barrierTs model.Ts eventBuffer []*model.PolymorphicEvent - rowBuffer []*model.RowChangedEvent + rawKVBuffer []*model.RawKVEntry - flowController tableFlowController + flowController keyspanFlowController - replicaConfig *config.ReplicaConfig - isTableActorMode bool + replicaConfig *config.ReplicaConfig + isKeySpanActorMode bool } -func newSinkNode(tableID model.TableID, sink sink.Sink, startTs model.Ts, targetTs model.Ts, flowController tableFlowController) *sinkNode { +func newSinkNode(keyspanID model.KeySpanID, sink sink.Sink, startTs model.Ts, targetTs model.Ts, flowController keyspanFlowController) *sinkNode { return &sinkNode{ - tableID: tableID, + keyspanID: keyspanID, sink: sink, - status: TableStatusInitializing, + status: KeySpanStatusInitializing, targetTs: targetTs, resolvedTs: startTs, checkpointTs: startTs, @@ -100,38 +100,38 @@ func newSinkNode(tableID model.TableID, sink sink.Sink, startTs model.Ts, target func (n *sinkNode) ResolvedTs() model.Ts { return atomic.LoadUint64(&n.resolvedTs) } func (n *sinkNode) CheckpointTs() model.Ts { return atomic.LoadUint64(&n.checkpointTs) } -func (n *sinkNode) Status() TableStatus { return n.status.Load() } +func (n *sinkNode) Status() KeySpanStatus { return n.status.Load() } func (n *sinkNode) Init(ctx pipeline.NodeContext) error { n.replicaConfig = ctx.ChangefeedVars().Info.Config return n.InitWithReplicaConfig(false, ctx.ChangefeedVars().Info.Config) } -func (n *sinkNode) InitWithReplicaConfig(isTableActorMode bool, replicaConfig *config.ReplicaConfig) error { +func (n *sinkNode) InitWithReplicaConfig(isKeySpanActorMode bool, replicaConfig *config.ReplicaConfig) error { n.replicaConfig = replicaConfig - n.isTableActorMode = isTableActorMode + n.isKeySpanActorMode = isKeySpanActorMode return nil } // stop is called when sink receives a stop command or checkpointTs reaches targetTs. -// In this method, the builtin table sink will be closed by calling `Close`, and +// In this method, the builtin keyspan sink will be closed by calling `Close`, and // no more events can be sent to this sink node afterwards. func (n *sinkNode) stop(ctx context.Context) (err error) { - // table stopped status must be set after underlying sink is closed - defer n.status.Store(TableStatusStopped) + // keyspan stopped status must be set after underlying sink is closed + defer n.status.Store(KeySpanStatusStopped) err = n.sink.Close(ctx) if err != nil { return } - log.Info("sink is closed", zap.Int64("tableID", n.tableID)) - err = cerror.ErrTableProcessorStoppedSafely.GenWithStackByArgs() + log.Info("sink is closed", zap.Uint64("keyspanID", n.keyspanID)) + err = cerror.ErrKeySpanProcessorStoppedSafely.GenWithStackByArgs() return } func (n *sinkNode) flushSink(ctx context.Context, resolvedTs model.Ts) (err error) { defer func() { if err != nil { - n.status.Store(TableStatusStopped) + n.status.Store(KeySpanStatusStopped) return } if n.checkpointTs >= n.targetTs { @@ -150,19 +150,19 @@ func (n *sinkNode) flushSink(ctx context.Context, resolvedTs model.Ts) (err erro if err := n.emitRow2Sink(ctx); err != nil { return errors.Trace(err) } - checkpointTs, err := n.sink.FlushRowChangedEvents(ctx, n.tableID, resolvedTs) + checkpointTs, err := n.sink.FlushChangedEvents(ctx, n.keyspanID, resolvedTs) if err != nil { return errors.Trace(err) } // we must call flowController.Release immediately after we call - // FlushRowChangedEvents to prevent deadlock cause by checkpointTs + // FlushChangedEvents to prevent deadlock cause by checkpointTs // fall back n.flowController.Release(checkpointTs) // the checkpointTs may fall back in some situation such as: - // 1. This table is newly added to the processor - // 2. There is one table in the processor that has a smaller + // 1. This keyspan is newly added to the processor + // 2. There is one keyspan in the processor that has a smaller // checkpointTs than this one if checkpointTs <= n.checkpointTs { return nil @@ -173,40 +173,12 @@ func (n *sinkNode) flushSink(ctx context.Context, resolvedTs model.Ts) (err erro } func (n *sinkNode) emitEvent(ctx context.Context, event *model.PolymorphicEvent) error { - if event == nil || event.Row == nil { + if event == nil { log.Warn("skip emit nil event", zap.Any("event", event)) return nil } - colLen := len(event.Row.Columns) - preColLen := len(event.Row.PreColumns) - // Some transactions could generate empty row change event, such as - // begin; insert into t (id) values (1); delete from t where id=1; commit; - // Just ignore these row changed events - if colLen == 0 && preColLen == 0 { - log.Warn("skip emit empty row event", zap.Any("event", event)) - return nil - } - - // This indicates that it is an update event, - // and after enable old value internally by default(but disable in the configuration). - // We need to handle the update event to be compatible with the old format. - if !n.replicaConfig.EnableOldValue && colLen != 0 && preColLen != 0 && colLen == preColLen { - if shouldSplitUpdateEvent(event) { - deleteEvent, insertEvent, err := splitUpdateEvent(event) - if err != nil { - return errors.Trace(err) - } - // NOTICE: Please do not change the order, the delete event always comes before the insert event. - n.eventBuffer = append(n.eventBuffer, deleteEvent, insertEvent) - } else { - // If the handle key columns are not updated, PreColumns is directly ignored. - event.Row.PreColumns = nil - n.eventBuffer = append(n.eventBuffer, event) - } - } else { - n.eventBuffer = append(n.eventBuffer, event) - } + n.eventBuffer = append(n.eventBuffer, event) if len(n.eventBuffer) >= defaultSyncResolvedBatch { if err := n.emitRow2Sink(ctx); err != nil { @@ -216,81 +188,17 @@ func (n *sinkNode) emitEvent(ctx context.Context, event *model.PolymorphicEvent) return nil } -// shouldSplitUpdateEvent determines if the split event is needed to align the old format based on -// whether the handle key column has been modified. -// If the handle key column is modified, -// we need to use splitUpdateEvent to split the update event into a delete and an insert event. -func shouldSplitUpdateEvent(updateEvent *model.PolymorphicEvent) bool { - // nil event will never be split. - if updateEvent == nil { - return false - } - - handleKeyCount := 0 - equivalentHandleKeyCount := 0 - for i := range updateEvent.Row.Columns { - if updateEvent.Row.Columns[i].Flag.IsHandleKey() && updateEvent.Row.PreColumns[i].Flag.IsHandleKey() { - handleKeyCount++ - colValueString := model.ColumnValueString(updateEvent.Row.Columns[i].Value) - preColValueString := model.ColumnValueString(updateEvent.Row.PreColumns[i].Value) - if colValueString == preColValueString { - equivalentHandleKeyCount++ - } - } - } - - // If the handle key columns are not updated, so we do **not** need to split the event row. - return !(handleKeyCount == equivalentHandleKeyCount) -} - -// splitUpdateEvent splits an update event into a delete and an insert event. -func splitUpdateEvent(updateEvent *model.PolymorphicEvent) (*model.PolymorphicEvent, *model.PolymorphicEvent, error) { - if updateEvent == nil { - return nil, nil, errors.New("nil event cannot be split") - } - - // If there is an update to handle key columns, - // we need to split the event into two events to be compatible with the old format. - // NOTICE: Here we don't need a full deep copy because our two events need Columns and PreColumns respectively, - // so it won't have an impact and no more full deep copy wastes memory. - deleteEvent := *updateEvent - deleteEventRow := *updateEvent.Row - deleteEventRowKV := *updateEvent.RawKV - deleteEvent.Row = &deleteEventRow - deleteEvent.RawKV = &deleteEventRowKV - - deleteEvent.Row.Columns = nil - for i := range deleteEvent.Row.PreColumns { - // NOTICE: Only the handle key pre column is retained in the delete event. - if !deleteEvent.Row.PreColumns[i].Flag.IsHandleKey() { - deleteEvent.Row.PreColumns[i] = nil - } - } - // Align with the old format if old value disabled. - deleteEvent.Row.TableInfoVersion = 0 - - insertEvent := *updateEvent - insertEventRow := *updateEvent.Row - insertEventRowKV := *updateEvent.RawKV - insertEvent.Row = &insertEventRow - insertEvent.RawKV = &insertEventRowKV - // NOTICE: clean up pre cols for insert event. - insertEvent.Row.PreColumns = nil - - return &deleteEvent, &insertEvent, nil -} - // clear event buffer and row buffer. // Also, it dereferences data that are held by buffers. func (n *sinkNode) clearBuffers() { // Do not hog memory. - if cap(n.rowBuffer) > defaultSyncResolvedBatch { - n.rowBuffer = make([]*model.RowChangedEvent, 0, defaultSyncResolvedBatch) + if cap(n.rawKVBuffer) > defaultSyncResolvedBatch { + n.rawKVBuffer = make([]*model.RawKVEntry, 0, defaultSyncResolvedBatch) } else { - for i := range n.rowBuffer { - n.rowBuffer[i] = nil + for i := range n.rawKVBuffer { + n.rawKVBuffer[i] = nil } - n.rowBuffer = n.rowBuffer[:0] + n.rawKVBuffer = n.rawKVBuffer[:0] } if cap(n.eventBuffer) > defaultSyncResolvedBatch { @@ -305,14 +213,14 @@ func (n *sinkNode) clearBuffers() { func (n *sinkNode) emitRow2Sink(ctx context.Context) error { for _, ev := range n.eventBuffer { - n.rowBuffer = append(n.rowBuffer, ev.Row) + n.rawKVBuffer = append(n.rawKVBuffer, ev.RawKV) } failpoint.Inject("ProcessorSyncResolvedPreEmit", func() { log.Info("Prepare to panic for ProcessorSyncResolvedPreEmit") time.Sleep(10 * time.Second) panic("ProcessorSyncResolvedPreEmit") }) - err := n.sink.EmitRowChangedEvents(ctx, n.rowBuffer...) + err := n.sink.EmitChangedEvents(ctx, n.rawKVBuffer...) if err != nil { return errors.Trace(err) } @@ -327,15 +235,15 @@ func (n *sinkNode) Receive(ctx pipeline.NodeContext) error { } func (n *sinkNode) HandleMessage(ctx context.Context, msg pipeline.Message) (bool, error) { - if n.status == TableStatusStopped { - return false, cerror.ErrTableProcessorStoppedSafely.GenWithStackByArgs() + if n.status == KeySpanStatusStopped { + return false, cerror.ErrKeySpanProcessorStoppedSafely.GenWithStackByArgs() } switch msg.Tp { case pipeline.MessageTypePolymorphicEvent: event := msg.PolymorphicEvent if event.RawKV.OpType == model.OpTypeResolved { - if n.status == TableStatusInitializing { - n.status.Store(TableStatusRunning) + if n.status == KeySpanStatusInitializing { + n.status.Store(KeySpanStatusRunning) } failpoint.Inject("ProcessorSyncResolvedError", func() { failpoint.Return(false, errors.New("processor sync resolved injected error")) @@ -369,7 +277,7 @@ func (n *sinkNode) HandleMessage(ctx context.Context, msg pipeline.Message) (boo } func (n *sinkNode) Destroy(ctx pipeline.NodeContext) error { - n.status.Store(TableStatusStopped) + n.status.Store(KeySpanStatusStopped) n.flowController.Abort() return n.sink.Close(ctx) } diff --git a/cdc/cdc/processor/pipeline/sink_test.go b/cdc/cdc/processor/pipeline/sink_test.go index e186d933..24488dd8 100644 --- a/cdc/cdc/processor/pipeline/sink_test.go +++ b/cdc/cdc/processor/pipeline/sink_test.go @@ -31,11 +31,11 @@ import ( type mockSink struct { received []struct { resolvedTs model.Ts - row *model.RowChangedEvent + rawKVEntry *model.RawKVEntry } } -// mockFlowController is created because a real tableFlowController cannot be used +// mockFlowController is created because a real keyspanFlowController cannot be used // we are testing sinkNode by itself. type mockFlowController struct{} @@ -53,24 +53,20 @@ func (c *mockFlowController) GetConsumption() uint64 { return 0 } -func (s *mockSink) EmitRowChangedEvents(ctx context.Context, rows ...*model.RowChangedEvent) error { - for _, row := range rows { +func (s *mockSink) EmitChangedEvents(ctx context.Context, RawKVEntries ...*model.RawKVEntry) error { + for _, rawKVEntry := range RawKVEntries { s.received = append(s.received, struct { resolvedTs model.Ts - row *model.RowChangedEvent - }{row: row}) + rawKVEntry *model.RawKVEntry + }{rawKVEntry: rawKVEntry}) } return nil } -func (s *mockSink) EmitDDLEvent(ctx context.Context, ddl *model.DDLEvent) error { - panic("unreachable") -} - -func (s *mockSink) FlushRowChangedEvents(ctx context.Context, _ model.TableID, resolvedTs uint64) (uint64, error) { +func (s *mockSink) FlushChangedEvents(ctx context.Context, _ model.KeySpanID, resolvedTs uint64) (uint64, error) { s.received = append(s.received, struct { resolvedTs model.Ts - row *model.RowChangedEvent + rawKVEntry *model.RawKVEntry }{resolvedTs: resolvedTs}) return resolvedTs, nil } @@ -83,13 +79,13 @@ func (s *mockSink) Close(ctx context.Context) error { return nil } -func (s *mockSink) Barrier(ctx context.Context, tableID model.TableID) error { +func (s *mockSink) Barrier(ctx context.Context, keyspanID model.KeySpanID) error { return nil } func (s *mockSink) Check(t *testing.T, expected []struct { resolvedTs model.Ts - row *model.RowChangedEvent + rawKVEntry *model.RawKVEntry }) { require.Equal(t, expected, s.received) } @@ -125,77 +121,77 @@ func TestStatus(t *testing.T) { // test stop at targetTs node := newSinkNode(1, &mockSink{}, 0, 10, &mockFlowController{}) require.Nil(t, node.Init(pipeline.MockNodeContext4Test(ctx, pipeline.Message{}, nil))) - require.Equal(t, TableStatusInitializing, node.Status()) + require.Equal(t, KeySpanStatusInitializing, node.Status()) require.Nil(t, node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.BarrierMessage(20), nil))) - require.Equal(t, TableStatusInitializing, node.Status()) + require.Equal(t, KeySpanStatusInitializing, node.Status()) require.Nil(t, node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.PolymorphicEventMessage(&model.PolymorphicEvent{CRTs: 1, RawKV: &model.RawKVEntry{OpType: model.OpTypePut}, Row: &model.RowChangedEvent{}}), nil))) - require.Equal(t, TableStatusInitializing, node.Status()) + require.Equal(t, KeySpanStatusInitializing, node.Status()) require.Nil(t, node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.PolymorphicEventMessage(&model.PolymorphicEvent{CRTs: 2, RawKV: &model.RawKVEntry{OpType: model.OpTypePut}, Row: &model.RowChangedEvent{}}), nil))) - require.Equal(t, TableStatusInitializing, node.Status()) + require.Equal(t, KeySpanStatusInitializing, node.Status()) require.Nil(t, node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.PolymorphicEventMessage(&model.PolymorphicEvent{CRTs: 2, RawKV: &model.RawKVEntry{OpType: model.OpTypeResolved}, Row: &model.RowChangedEvent{}}), nil))) - require.Equal(t, TableStatusRunning, node.Status()) + require.Equal(t, KeySpanStatusRunning, node.Status()) err := node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.PolymorphicEventMessage(&model.PolymorphicEvent{CRTs: 15, RawKV: &model.RawKVEntry{OpType: model.OpTypeResolved}, Row: &model.RowChangedEvent{}}), nil)) - require.True(t, cerrors.ErrTableProcessorStoppedSafely.Equal(err)) - require.Equal(t, TableStatusStopped, node.Status()) + require.True(t, cerrors.ErrKeySpanProcessorStoppedSafely.Equal(err)) + require.Equal(t, KeySpanStatusStopped, node.Status()) require.Equal(t, uint64(10), node.CheckpointTs()) // test the stop at ts command node = newSinkNode(1, &mockSink{}, 0, 10, &mockFlowController{}) require.Nil(t, node.Init(pipeline.MockNodeContext4Test(ctx, pipeline.Message{}, nil))) - require.Equal(t, TableStatusInitializing, node.Status()) + require.Equal(t, KeySpanStatusInitializing, node.Status()) require.Nil(t, node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.BarrierMessage(20), nil))) - require.Equal(t, TableStatusInitializing, node.Status()) + require.Equal(t, KeySpanStatusInitializing, node.Status()) require.Nil(t, node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.PolymorphicEventMessage(&model.PolymorphicEvent{CRTs: 2, RawKV: &model.RawKVEntry{OpType: model.OpTypeResolved}, Row: &model.RowChangedEvent{}}), nil))) - require.Equal(t, TableStatusRunning, node.Status()) + require.Equal(t, KeySpanStatusRunning, node.Status()) err = node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.CommandMessage(&pipeline.Command{Tp: pipeline.CommandTypeStop}), nil)) - require.True(t, cerrors.ErrTableProcessorStoppedSafely.Equal(err)) - require.Equal(t, TableStatusStopped, node.Status()) + require.True(t, cerrors.ErrKeySpanProcessorStoppedSafely.Equal(err)) + require.Equal(t, KeySpanStatusStopped, node.Status()) err = node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.PolymorphicEventMessage(&model.PolymorphicEvent{CRTs: 7, RawKV: &model.RawKVEntry{OpType: model.OpTypeResolved}, Row: &model.RowChangedEvent{}}), nil)) - require.True(t, cerrors.ErrTableProcessorStoppedSafely.Equal(err)) - require.Equal(t, TableStatusStopped, node.Status()) + require.True(t, cerrors.ErrKeySpanProcessorStoppedSafely.Equal(err)) + require.Equal(t, KeySpanStatusStopped, node.Status()) require.Equal(t, uint64(2), node.CheckpointTs()) // test the stop at ts command is after then resolvedTs and checkpointTs is greater than stop ts node = newSinkNode(1, &mockSink{}, 0, 10, &mockFlowController{}) require.Nil(t, node.Init(pipeline.MockNodeContext4Test(ctx, pipeline.Message{}, nil))) - require.Equal(t, TableStatusInitializing, node.Status()) + require.Equal(t, KeySpanStatusInitializing, node.Status()) require.Nil(t, node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.BarrierMessage(20), nil))) - require.Equal(t, TableStatusInitializing, node.Status()) + require.Equal(t, KeySpanStatusInitializing, node.Status()) require.Nil(t, node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.PolymorphicEventMessage(&model.PolymorphicEvent{CRTs: 7, RawKV: &model.RawKVEntry{OpType: model.OpTypeResolved}, Row: &model.RowChangedEvent{}}), nil))) - require.Equal(t, TableStatusRunning, node.Status()) + require.Equal(t, KeySpanStatusRunning, node.Status()) err = node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.CommandMessage(&pipeline.Command{Tp: pipeline.CommandTypeStop}), nil)) - require.True(t, cerrors.ErrTableProcessorStoppedSafely.Equal(err)) - require.Equal(t, TableStatusStopped, node.Status()) + require.True(t, cerrors.ErrKeySpanProcessorStoppedSafely.Equal(err)) + require.Equal(t, KeySpanStatusStopped, node.Status()) err = node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.PolymorphicEventMessage(&model.PolymorphicEvent{CRTs: 7, RawKV: &model.RawKVEntry{OpType: model.OpTypeResolved}, Row: &model.RowChangedEvent{}}), nil)) - require.True(t, cerrors.ErrTableProcessorStoppedSafely.Equal(err)) - require.Equal(t, TableStatusStopped, node.Status()) + require.True(t, cerrors.ErrKeySpanProcessorStoppedSafely.Equal(err)) + require.Equal(t, KeySpanStatusStopped, node.Status()) require.Equal(t, uint64(7), node.CheckpointTs()) } -// TestStopStatus tests the table status of a pipeline is not set to stopped +// TestStopStatus tests the keyspan status of a pipeline is not set to stopped // until the underlying sink is closed func TestStopStatus(t *testing.T) { ctx := cdcContext.NewContext(context.Background(), &cdcContext.GlobalVars{}) @@ -210,10 +206,10 @@ func TestStopStatus(t *testing.T) { closeCh := make(chan interface{}, 1) node := newSinkNode(1, &mockCloseControlSink{mockSink: mockSink{}, closeCh: closeCh}, 0, 100, &mockFlowController{}) require.Nil(t, node.Init(pipeline.MockNodeContext4Test(ctx, pipeline.Message{}, nil))) - require.Equal(t, TableStatusInitializing, node.Status()) + require.Equal(t, KeySpanStatusInitializing, node.Status()) require.Nil(t, node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.PolymorphicEventMessage(&model.PolymorphicEvent{CRTs: 2, RawKV: &model.RawKVEntry{OpType: model.OpTypeResolved}, Row: &model.RowChangedEvent{}}), nil))) - require.Equal(t, TableStatusRunning, node.Status()) + require.Equal(t, KeySpanStatusRunning, node.Status()) var wg sync.WaitGroup wg.Add(1) @@ -222,12 +218,12 @@ func TestStopStatus(t *testing.T) { // This will block until sink Close returns err := node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.CommandMessage(&pipeline.Command{Tp: pipeline.CommandTypeStop}), nil)) - require.True(t, cerrors.ErrTableProcessorStoppedSafely.Equal(err)) - require.Equal(t, TableStatusStopped, node.Status()) + require.True(t, cerrors.ErrKeySpanProcessorStoppedSafely.Equal(err)) + require.Equal(t, KeySpanStatusStopped, node.Status()) }() // wait to ensure stop message is sent to the sink node time.Sleep(time.Millisecond * 50) - require.Equal(t, TableStatusRunning, node.Status()) + require.Equal(t, KeySpanStatusRunning, node.Status()) closeCh <- struct{}{} wg.Wait() } @@ -244,92 +240,54 @@ func TestManyTs(t *testing.T) { sink := &mockSink{} node := newSinkNode(1, sink, 0, 10, &mockFlowController{}) require.Nil(t, node.Init(pipeline.MockNodeContext4Test(ctx, pipeline.Message{}, nil))) - require.Equal(t, TableStatusInitializing, node.Status()) + require.Equal(t, KeySpanStatusInitializing, node.Status()) require.Nil(t, node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.PolymorphicEventMessage(&model.PolymorphicEvent{ - CRTs: 1, RawKV: &model.RawKVEntry{OpType: model.OpTypePut}, Row: &model.RowChangedEvent{ - CommitTs: 1, - Columns: []*model.Column{ - { - Name: "col1", - Flag: model.BinaryFlag, - Value: "col1-value-updated", - }, - { - Name: "col2", - Flag: model.HandleKeyFlag, - Value: "col2-value", - }, - }, - }, + CRTs: 1, + RawKV: &model.RawKVEntry{ + Key: []byte{1}, + Value: []byte{1}, + OpType: model.OpTypePut, + }, Row: &model.RowChangedEvent{}, }), nil))) - require.Equal(t, TableStatusInitializing, node.Status()) + require.Equal(t, KeySpanStatusInitializing, node.Status()) require.Nil(t, node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.PolymorphicEventMessage(&model.PolymorphicEvent{ - CRTs: 2, RawKV: &model.RawKVEntry{OpType: model.OpTypePut}, Row: &model.RowChangedEvent{ - CommitTs: 2, - Columns: []*model.Column{ - { - Name: "col1", - Flag: model.BinaryFlag, - Value: "col1-value-updated", - }, - { - Name: "col2", - Flag: model.HandleKeyFlag, - Value: "col2-value", - }, - }, - }, + CRTs: 2, + RawKV: &model.RawKVEntry{ + Key: []byte{2}, + Value: []byte{2}, + OpType: model.OpTypePut, + }, Row: &model.RowChangedEvent{}, }), nil))) - require.Equal(t, TableStatusInitializing, node.Status()) + require.Equal(t, KeySpanStatusInitializing, node.Status()) require.Nil(t, node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.PolymorphicEventMessage(&model.PolymorphicEvent{CRTs: 2, RawKV: &model.RawKVEntry{OpType: model.OpTypeResolved}, Row: &model.RowChangedEvent{}}), nil))) - require.Equal(t, TableStatusRunning, node.Status()) + require.Equal(t, KeySpanStatusRunning, node.Status()) sink.Check(t, nil) require.Nil(t, node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.BarrierMessage(1), nil))) - require.Equal(t, TableStatusRunning, node.Status()) + require.Equal(t, KeySpanStatusRunning, node.Status()) sink.Check(t, []struct { resolvedTs model.Ts - row *model.RowChangedEvent + rawKVEntry *model.RawKVEntry }{ { - row: &model.RowChangedEvent{ - CommitTs: 1, - Columns: []*model.Column{ - { - Name: "col1", - Flag: model.BinaryFlag, - Value: "col1-value-updated", - }, - { - Name: "col2", - Flag: model.HandleKeyFlag, - Value: "col2-value", - }, - }, + rawKVEntry: &model.RawKVEntry{ + Key: []byte{1}, + Value: []byte{1}, + OpType: model.OpTypePut, }, }, { - row: &model.RowChangedEvent{ - CommitTs: 2, - Columns: []*model.Column{ - { - Name: "col1", - Flag: model.BinaryFlag, - Value: "col1-value-updated", - }, - { - Name: "col2", - Flag: model.HandleKeyFlag, - Value: "col2-value", - }, - }, + rawKVEntry: &model.RawKVEntry{ + Key: []byte{2}, + Value: []byte{2}, + OpType: model.OpTypePut, }, }, {resolvedTs: 1}, @@ -339,10 +297,10 @@ func TestManyTs(t *testing.T) { require.Equal(t, uint64(1), node.CheckpointTs()) require.Nil(t, node.Receive(pipeline.MockNodeContext4Test(ctx, pipeline.BarrierMessage(5), nil))) - require.Equal(t, TableStatusRunning, node.Status()) + require.Equal(t, KeySpanStatusRunning, node.Status()) sink.Check(t, []struct { resolvedTs model.Ts - row *model.RowChangedEvent + rawKVEntry *model.RawKVEntry }{ {resolvedTs: 2}, }) @@ -351,7 +309,8 @@ func TestManyTs(t *testing.T) { require.Equal(t, uint64(2), node.CheckpointTs()) } -func TestIgnoreEmptyRowChangeEvent(t *testing.T) { +/* +func TestIgnoreEmptyChangeEvent(t *testing.T) { ctx := cdcContext.NewContext(context.Background(), &cdcContext.GlobalVars{}) ctx = cdcContext.WithChangefeedVars(ctx, &cdcContext.ChangefeedVars{ ID: "changefeed-id-test-ignore-empty-row-change-event", @@ -366,7 +325,7 @@ func TestIgnoreEmptyRowChangeEvent(t *testing.T) { // empty row, no Columns and PreColumns. require.Nil(t, node.Receive(pipeline.MockNodeContext4Test(ctx, - pipeline.PolymorphicEventMessage(&model.PolymorphicEvent{CRTs: 1, RawKV: &model.RawKVEntry{OpType: model.OpTypePut}, Row: &model.RowChangedEvent{CommitTs: 1}}), nil))) + pipeline.PolymorphicEventMessage(&model.PolymorphicEvent{CRTs: 1, RawKV: &model.RawKVEntry{OpType: model.OpTypePut}}), nil))) require.Equal(t, 0, len(node.eventBuffer)) } @@ -533,6 +492,7 @@ func TestSplitUpdateEventWhenDisableOldValue(t *testing.T) { require.Equal(t, 2, len(node.eventBuffer[insertEventIndex].Row.Columns)) require.Equal(t, 0, len(node.eventBuffer[insertEventIndex].Row.PreColumns)) } +*/ type flushFlowController struct { mockFlowController @@ -551,7 +511,7 @@ type flushSink struct { // fall back var fallBackResolvedTs = uint64(10) -func (s *flushSink) FlushRowChangedEvents(ctx context.Context, _ model.TableID, resolvedTs uint64) (uint64, error) { +func (s *flushSink) FlushChangedEvents(ctx context.Context, _ model.KeySpanID, resolvedTs uint64) (uint64, error) { if resolvedTs == fallBackResolvedTs { return 0, nil } @@ -559,7 +519,7 @@ func (s *flushSink) FlushRowChangedEvents(ctx context.Context, _ model.TableID, } // TestFlushSinkReleaseFlowController tests sinkNode.flushSink method will always -// call flowController.Release to release the memory quota of the table to avoid +// call flowController.Release to release the memory quota of the keyspan to avoid // deadlock if there is no error occur func TestFlushSinkReleaseFlowController(t *testing.T) { ctx := cdcContext.NewContext(context.Background(), &cdcContext.GlobalVars{}) diff --git a/cdc/cdc/processor/pipeline/sorter.go b/cdc/cdc/processor/pipeline/sorter.go deleted file mode 100644 index 9d89fb74..00000000 --- a/cdc/cdc/processor/pipeline/sorter.go +++ /dev/null @@ -1,323 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package pipeline - -import ( - "context" - "sync/atomic" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/entry" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/redo" - "github.com/tikv/migration/cdc/cdc/sorter" - "github.com/tikv/migration/cdc/cdc/sorter/leveldb" - "github.com/tikv/migration/cdc/cdc/sorter/memory" - "github.com/tikv/migration/cdc/cdc/sorter/unified" - "github.com/tikv/migration/cdc/pkg/actor" - "github.com/tikv/migration/cdc/pkg/actor/message" - "github.com/tikv/migration/cdc/pkg/config" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/pipeline" - "go.uber.org/zap" - "golang.org/x/sync/errgroup" -) - -const ( - flushMemoryMetricsDuration = time.Second * 5 -) - -type sorterNode struct { - sorter sorter.EventSorter - - tableID model.TableID - tableName string // quoted schema and table, used in metircs only - - // for per-table flow control - flowController tableFlowController - - mounter entry.Mounter - - eg *errgroup.Group - cancel context.CancelFunc - - cleanID actor.ID - cleanTask message.Message - cleanRouter *actor.Router - - // The latest resolved ts that sorter has received. - resolvedTs model.Ts - - // The latest barrier ts that sorter has received. - barrierTs model.Ts - - replConfig *config.ReplicaConfig - - // isTableActorMode identify if the sorter node is run is actor mode, todo: remove it after GA - isTableActorMode bool -} - -func newSorterNode( - tableName string, tableID model.TableID, startTs model.Ts, - flowController tableFlowController, mounter entry.Mounter, - replConfig *config.ReplicaConfig, -) *sorterNode { - return &sorterNode{ - tableName: tableName, - tableID: tableID, - flowController: flowController, - mounter: mounter, - resolvedTs: startTs, - barrierTs: startTs, - replConfig: replConfig, - } -} - -func (n *sorterNode) Init(ctx pipeline.NodeContext) error { - wg := errgroup.Group{} - return n.StartActorNode(ctx, false, &wg) -} - -func (n *sorterNode) StartActorNode(ctx pipeline.NodeContext, isTableActorMode bool, eg *errgroup.Group) error { - n.isTableActorMode = isTableActorMode - n.eg = eg - stdCtx, cancel := context.WithCancel(ctx) - n.cancel = cancel - var eventSorter sorter.EventSorter - sortEngine := ctx.ChangefeedVars().Info.Engine - switch sortEngine { - case model.SortInMemory: - eventSorter = memory.NewEntrySorter() - case model.SortUnified, model.SortInFile /* `file` becomes an alias of `unified` for backward compatibility */ : - if sortEngine == model.SortInFile { - log.Warn("File sorter is obsolete and replaced by unified sorter. Please revise your changefeed settings", - zap.String("changefeed-id", ctx.ChangefeedVars().ID), zap.String("table-name", n.tableName)) - } - - if config.GetGlobalServerConfig().Debug.EnableDBSorter { - startTs := ctx.ChangefeedVars().Info.StartTs - actorID := ctx.GlobalVars().SorterSystem.ActorID(uint64(n.tableID)) - router := ctx.GlobalVars().SorterSystem.Router() - compactScheduler := ctx.GlobalVars().SorterSystem.CompactScheduler() - levelSorter := leveldb.NewSorter( - ctx, n.tableID, startTs, router, actorID, compactScheduler, - config.GetGlobalServerConfig().Debug.DB) - n.cleanID = actorID - n.cleanTask = levelSorter.CleanupTask() - n.cleanRouter = ctx.GlobalVars().SorterSystem.CleanerRouter() - eventSorter = levelSorter - } else { - // Sorter dir has been set and checked when server starts. - // See https://github.com/tikv/migration/cdc/blob/9dad09/cdc/server.go#L275 - sortDir := config.GetGlobalServerConfig().Sorter.SortDir - var err error - eventSorter, err = unified.NewUnifiedSorter(sortDir, ctx.ChangefeedVars().ID, n.tableName, n.tableID, ctx.GlobalVars().CaptureInfo.AdvertiseAddr) - if err != nil { - return errors.Trace(err) - } - } - default: - return cerror.ErrUnknownSortEngine.GenWithStackByArgs(sortEngine) - } - failpoint.Inject("ProcessorAddTableError", func() { - failpoint.Return(errors.New("processor add table injected error")) - }) - n.eg.Go(func() error { - ctx.Throw(errors.Trace(eventSorter.Run(stdCtx))) - return nil - }) - n.eg.Go(func() error { - // Since the flowController is implemented by `Cond`, it is not cancelable - // by a context. We need to listen on cancellation and aborts the flowController - // manually. - <-stdCtx.Done() - n.flowController.Abort() - return nil - }) - n.eg.Go(func() error { - lastSentResolvedTs := uint64(0) - lastSendResolvedTsTime := time.Now() // the time at which we last sent a resolved-ts. - lastCRTs := uint64(0) // the commit-ts of the last row changed we sent. - - metricsTableMemoryHistogram := tableMemoryHistogram.WithLabelValues(ctx.ChangefeedVars().ID, ctx.GlobalVars().CaptureInfo.AdvertiseAddr) - metricsTicker := time.NewTicker(flushMemoryMetricsDuration) - defer metricsTicker.Stop() - - for { - select { - case <-stdCtx.Done(): - return nil - case <-metricsTicker.C: - metricsTableMemoryHistogram.Observe(float64(n.flowController.GetConsumption())) - case msg, ok := <-eventSorter.Output(): - if !ok { - // sorter output channel closed - return nil - } - if msg == nil || msg.RawKV == nil { - log.Panic("unexpected empty msg", zap.Reflect("msg", msg)) - } - if msg.RawKV.OpType != model.OpTypeResolved { - // DESIGN NOTE: We send the messages to the mounter in - // this separate goroutine to prevent blocking - // the whole pipeline. - msg.SetUpFinishedChan() - select { - case <-ctx.Done(): - return nil - case n.mounter.Input() <- msg: - } - - commitTs := msg.CRTs - // We interpolate a resolved-ts if none has been sent for some time. - if time.Since(lastSendResolvedTsTime) > resolvedTsInterpolateInterval { - // checks the condition: cur_event_commit_ts > prev_event_commit_ts > last_resolved_ts - // If this is true, it implies that (1) the last transaction has finished, and we are processing - // the first event in a new transaction, (2) a resolved-ts is safe to be sent, but it has not yet. - // This means that we can interpolate prev_event_commit_ts as a resolved-ts, improving the frequency - // at which the sink flushes. - if lastCRTs > lastSentResolvedTs && commitTs > lastCRTs { - lastSentResolvedTs = lastCRTs - lastSendResolvedTsTime = time.Now() - ctx.SendToNextNode(pipeline.PolymorphicEventMessage(model.NewResolvedPolymorphicEvent(0, lastCRTs))) - } - } - - // Must wait before accessing msg.Row - err := msg.WaitPrepare(ctx) - if err != nil { - if errors.Cause(err) != context.Canceled { - ctx.Throw(err) - } - return errors.Trace(err) - } - // We calculate memory consumption by RowChangedEvent size. - // It's much larger than RawKVEntry. - size := uint64(msg.Row.ApproximateBytes()) - // NOTE we allow the quota to be exceeded if blocking means interrupting a transaction. - // Otherwise the pipeline would deadlock. - err = n.flowController.Consume(commitTs, size, func() error { - if lastCRTs > lastSentResolvedTs { - // If we are blocking, we send a Resolved Event here to elicit a sink-flush. - // Not sending a Resolved Event here will very likely deadlock the pipeline. - lastSentResolvedTs = lastCRTs - lastSendResolvedTsTime = time.Now() - ctx.SendToNextNode(pipeline.PolymorphicEventMessage(model.NewResolvedPolymorphicEvent(0, lastCRTs))) - } - return nil - }) - if err != nil { - if cerror.ErrFlowControllerAborted.Equal(err) { - log.Info("flow control cancelled for table", - zap.Int64("tableID", n.tableID), - zap.String("tableName", n.tableName)) - } else { - ctx.Throw(err) - } - return nil - } - lastCRTs = commitTs - } else { - // handle OpTypeResolved - if msg.CRTs < lastSentResolvedTs { - continue - } - lastSentResolvedTs = msg.CRTs - lastSendResolvedTsTime = time.Now() - } - ctx.SendToNextNode(pipeline.PolymorphicEventMessage(msg)) - } - } - }) - n.sorter = eventSorter - return nil -} - -// Receive receives the message from the previous node -func (n *sorterNode) Receive(ctx pipeline.NodeContext) error { - _, err := n.TryHandleDataMessage(ctx, ctx.Message()) - return err -} - -func (n *sorterNode) TryHandleDataMessage(ctx context.Context, msg pipeline.Message) (bool, error) { - switch msg.Tp { - case pipeline.MessageTypePolymorphicEvent: - rawKV := msg.PolymorphicEvent.RawKV - if rawKV != nil && rawKV.OpType == model.OpTypeResolved { - // Puller resolved ts should not fall back. - resolvedTs := rawKV.CRTs - oldResolvedTs := atomic.SwapUint64(&n.resolvedTs, resolvedTs) - if oldResolvedTs > resolvedTs { - log.Panic("resolved ts regression", - zap.Int64("tableID", n.tableID), - zap.Uint64("resolvedTs", resolvedTs), - zap.Uint64("oldResolvedTs", oldResolvedTs)) - } - atomic.StoreUint64(&n.resolvedTs, rawKV.CRTs) - - if resolvedTs > n.barrierTs && - !redo.IsConsistentEnabled(n.replConfig.Consistent.Level) { - // Do not send resolved ts events that is larger than - // barrier ts. - // When DDL puller stall, resolved events that outputted by - // sorter may pile up in memory, as they have to wait DDL. - // - // Disabled if redolog is on, it requires sink reports - // resolved ts, conflicts to this change. - // TODO: Remove redolog check once redolog decouples for global - // resolved ts. - msg = pipeline.PolymorphicEventMessage( - model.NewResolvedPolymorphicEvent(0, n.barrierTs)) - } - } - // todo: remove feature switcher after GA - if n.isTableActorMode { - return n.sorter.TryAddEntry(ctx, msg.PolymorphicEvent) - } - n.sorter.AddEntry(ctx, msg.PolymorphicEvent) - return true, nil - case pipeline.MessageTypeBarrier: - if msg.BarrierTs > n.barrierTs { - n.barrierTs = msg.BarrierTs - } - fallthrough - default: - // todo: remove feature switcher after GA - if n.isTableActorMode { - return ctx.(*actorNodeContext).TrySendToNextNode(msg), nil - } - ctx.(pipeline.NodeContext).SendToNextNode(msg) - return true, nil - } -} - -func (n *sorterNode) Destroy(ctx pipeline.NodeContext) error { - defer tableMemoryHistogram.DeleteLabelValues(ctx.ChangefeedVars().ID, ctx.GlobalVars().CaptureInfo.AdvertiseAddr) - n.cancel() - if n.cleanRouter != nil { - // Clean up data when the table sorter is canceled. - err := n.cleanRouter.SendB(ctx, n.cleanID, n.cleanTask) - if err != nil { - log.Warn("schedule table cleanup task failed", zap.Error(err)) - } - } - return n.eg.Wait() -} - -func (n *sorterNode) ResolvedTs() model.Ts { - return atomic.LoadUint64(&n.resolvedTs) -} diff --git a/cdc/cdc/processor/pipeline/sorter_test.go b/cdc/cdc/processor/pipeline/sorter_test.go deleted file mode 100644 index 22a37d44..00000000 --- a/cdc/cdc/processor/pipeline/sorter_test.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package pipeline - -import ( - "context" - "strings" - "testing" - - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/redo" - "github.com/tikv/migration/cdc/cdc/sorter" - "github.com/tikv/migration/cdc/cdc/sorter/memory" - "github.com/tikv/migration/cdc/cdc/sorter/unified" - "github.com/tikv/migration/cdc/pkg/config" - cdcContext "github.com/tikv/migration/cdc/pkg/context" - "github.com/tikv/migration/cdc/pkg/pipeline" -) - -func TestUnifiedSorterFileLockConflict(t *testing.T) { - dir := t.TempDir() - captureAddr := "0.0.0.0:0" - - // GlobalServerConfig overrides dir parameter in NewUnifiedSorter. - config.GetGlobalServerConfig().Sorter.SortDir = dir - _, err := unified.NewUnifiedSorter(dir, "test-cf", "test", 0, captureAddr) - require.Nil(t, err) - - unified.ResetGlobalPoolWithoutCleanup() - ctx := cdcContext.NewBackendContext4Test(true) - ctx.ChangefeedVars().Info.Engine = model.SortUnified - ctx.ChangefeedVars().Info.SortDir = dir - sorter := sorterNode{} - err = sorter.Init(pipeline.MockNodeContext4Test(ctx, pipeline.Message{}, nil)) - require.True(t, strings.Contains(err.Error(), "file lock conflict")) -} - -func TestSorterResolvedTs(t *testing.T) { - t.Parallel() - sn := newSorterNode("tableName", 1, 1, nil, nil, &config.ReplicaConfig{ - Consistent: &config.ConsistentConfig{}, - }) - sn.sorter = memory.NewEntrySorter() - require.EqualValues(t, 1, sn.ResolvedTs()) - nctx := pipeline.NewNodeContext( - cdcContext.NewContext(context.Background(), nil), - pipeline.PolymorphicEventMessage(model.NewResolvedPolymorphicEvent(0, 2)), - nil, - ) - err := sn.Receive(nctx) - require.Nil(t, err) - require.EqualValues(t, 2, sn.ResolvedTs()) -} - -type checkSorter struct { - ch chan *model.PolymorphicEvent -} - -var _ sorter.EventSorter = (*checkSorter)(nil) - -func (c *checkSorter) Run(ctx context.Context) error { - return nil -} - -func (c *checkSorter) AddEntry(ctx context.Context, entry *model.PolymorphicEvent) { - c.ch <- entry -} - -func (c *checkSorter) TryAddEntry( - ctx context.Context, entry *model.PolymorphicEvent, -) (bool, error) { - select { - case c.ch <- entry: - return true, nil - default: - return false, nil - } -} - -func (c *checkSorter) Output() <-chan *model.PolymorphicEvent { - return c.ch -} - -func TestSorterResolvedTsLessEqualBarrierTs(t *testing.T) { - t.Parallel() - sch := make(chan *model.PolymorphicEvent, 1) - s := &checkSorter{ch: sch} - sn := newSorterNode("tableName", 1, 1, nil, nil, &config.ReplicaConfig{ - Consistent: &config.ConsistentConfig{}, - }) - sn.sorter = s - - ch := make(chan pipeline.Message, 1) - require.EqualValues(t, 1, sn.ResolvedTs()) - - // Resolved ts must not regress even if there is no barrier ts message. - resolvedTs1 := pipeline.PolymorphicEventMessage(model.NewResolvedPolymorphicEvent(0, 1)) - nctx := pipeline.NewNodeContext( - cdcContext.NewContext(context.Background(), nil), resolvedTs1, ch) - err := sn.Receive(nctx) - require.Nil(t, err) - require.EqualValues(t, model.NewResolvedPolymorphicEvent(0, 1), <-sch) - - // Advance barrier ts. - nctx = pipeline.NewNodeContext( - cdcContext.NewContext(context.Background(), nil), - pipeline.BarrierMessage(2), - ch, - ) - err = sn.Receive(nctx) - require.Nil(t, err) - require.EqualValues(t, 2, sn.barrierTs) - // Barrier message must be passed to the next node. - require.EqualValues(t, pipeline.BarrierMessage(2), <-ch) - - resolvedTs2 := pipeline.PolymorphicEventMessage(model.NewResolvedPolymorphicEvent(0, 2)) - nctx = pipeline.NewNodeContext( - cdcContext.NewContext(context.Background(), nil), resolvedTs2, nil) - err = sn.Receive(nctx) - require.Nil(t, err) - require.EqualValues(t, resolvedTs2.PolymorphicEvent, <-s.Output()) - - resolvedTs3 := pipeline.PolymorphicEventMessage(model.NewResolvedPolymorphicEvent(0, 3)) - nctx = pipeline.NewNodeContext( - cdcContext.NewContext(context.Background(), nil), resolvedTs3, nil) - err = sn.Receive(nctx) - require.Nil(t, err) - require.EqualValues(t, resolvedTs2.PolymorphicEvent, <-s.Output()) - - resolvedTs4 := pipeline.PolymorphicEventMessage(model.NewResolvedPolymorphicEvent(0, 4)) - sn.replConfig.Consistent.Level = string(redo.ConsistentLevelEventual) - nctx = pipeline.NewNodeContext( - cdcContext.NewContext(context.Background(), nil), resolvedTs4, nil) - err = sn.Receive(nctx) - require.Nil(t, err) - resolvedTs4 = pipeline.PolymorphicEventMessage(model.NewResolvedPolymorphicEvent(0, 4)) - require.EqualValues(t, resolvedTs4.PolymorphicEvent, <-s.Output()) -} diff --git a/cdc/cdc/processor/pipeline/system/system.go b/cdc/cdc/processor/pipeline/system/system.go deleted file mode 100644 index 6429be03..00000000 --- a/cdc/cdc/processor/pipeline/system/system.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package system - -import ( - "context" - "fmt" - "sync" - - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/actor" -) - -// System manages table pipeline global resource. -type System struct { - tableActorSystem *actor.System - tableActorRouter *actor.Router - - // actorIDMap store all allocated ID for changefeed-table -> ID pair - actorIDMap map[string]uint64 - actorIDGeneratorLck sync.Mutex - lastID uint64 -} - -// NewSystem returns a system. -func NewSystem() *System { - return &System{ - actorIDMap: map[string]uint64{}, - lastID: 1, - } -} - -// Start starts a system. -func (s *System) Start(ctx context.Context) error { - // todo: make the table actor system configurable - s.tableActorSystem, s.tableActorRouter = actor.NewSystemBuilder("table").Build() - s.tableActorSystem.Start(ctx) - return nil -} - -// Stop stops a system. -func (s *System) Stop() error { - return s.tableActorSystem.Stop() -} - -func (s *System) Router() *actor.Router { - return s.tableActorRouter -} - -func (s *System) System() *actor.System { - return s.tableActorSystem -} - -// ActorID returns an ActorID correspond with tableID. -func (s *System) ActorID(changefeedID string, tableID model.TableID) actor.ID { - s.actorIDGeneratorLck.Lock() - defer s.actorIDGeneratorLck.Unlock() - - key := fmt.Sprintf("%s-%d", changefeedID, tableID) - id, ok := s.actorIDMap[key] - if !ok { - s.lastID++ - id = s.lastID - s.actorIDMap[key] = id - } - return actor.ID(id) -} diff --git a/cdc/cdc/processor/pipeline/system/system_test.go b/cdc/cdc/processor/pipeline/system/system_test.go deleted file mode 100644 index 74e6ad5d..00000000 --- a/cdc/cdc/processor/pipeline/system/system_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package system - -import ( - "context" - "math" - "testing" - - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/actor" -) - -func TestStartAndStopSystem(t *testing.T) { - t.Parallel() - - s := NewSystem() - require.Nil(t, s.Start(context.TODO())) - require.Nil(t, s.Stop()) -} - -func TestActorID(t *testing.T) { - sys := NewSystem() - type table struct { - changeFeed string - tableID model.TableID - } - cases := []table{ - {"abc", 1}, - {"", -1}, - {"", 0}, - {"", math.MaxInt64}, - {"afddeefessssssss", math.MaxInt64}, - {"afddeefessssssss", 0}, - {"afddeefessssssss", 1}, - } - ids := make(map[actor.ID]bool) - for _, c := range cases { - id1 := sys.ActorID(c.changeFeed, c.tableID) - for i := 0; i < 10; i++ { - require.Equal(t, id1, sys.ActorID(c.changeFeed, c.tableID)) - } - require.False(t, ids[id1]) - ids[id1] = true - } -} diff --git a/cdc/cdc/processor/pipeline/table.go b/cdc/cdc/processor/pipeline/table.go deleted file mode 100644 index cf99e79f..00000000 --- a/cdc/cdc/processor/pipeline/table.go +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package pipeline - -import ( - "context" - "time" - - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/entry" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/redo" - "github.com/tikv/migration/cdc/cdc/sink" - "github.com/tikv/migration/cdc/cdc/sink/common" - serverConfig "github.com/tikv/migration/cdc/pkg/config" - cdcContext "github.com/tikv/migration/cdc/pkg/context" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/pipeline" - "go.uber.org/zap" -) - -const ( - // TODO determine a reasonable default value - // This is part of sink performance optimization - resolvedTsInterpolateInterval = 200 * time.Millisecond -) - -// TablePipeline is a pipeline which capture the change log from tikv in a table -type TablePipeline interface { - // ID returns the ID of source table and mark table - ID() (tableID, markTableID int64) - // Name returns the quoted schema and table name - Name() string - // ResolvedTs returns the resolved ts in this table pipeline - ResolvedTs() model.Ts - // CheckpointTs returns the checkpoint ts in this table pipeline - CheckpointTs() model.Ts - // UpdateBarrierTs updates the barrier ts in this table pipeline - UpdateBarrierTs(ts model.Ts) - // AsyncStop tells the pipeline to stop, and returns true is the pipeline is already stopped. - AsyncStop(targetTs model.Ts) bool - // Workload returns the workload of this table - Workload() model.WorkloadInfo - // Status returns the status of this table pipeline - Status() TableStatus - // Cancel stops this table pipeline immediately and destroy all resources created by this table pipeline - Cancel() - // Wait waits for table pipeline destroyed - Wait() -} - -type tablePipelineImpl struct { - p *pipeline.Pipeline - - tableID int64 - markTableID int64 - tableName string // quoted schema and table, used in metircs only - - sorterNode *sorterNode - sinkNode *sinkNode - cancel context.CancelFunc - - replConfig *serverConfig.ReplicaConfig -} - -// TODO find a better name or avoid using an interface -// We use an interface here for ease in unit testing. -type tableFlowController interface { - Consume(commitTs uint64, size uint64, blockCallBack func() error) error - Release(resolvedTs uint64) - Abort() - GetConsumption() uint64 -} - -// ResolvedTs returns the resolved ts in this table pipeline -func (t *tablePipelineImpl) ResolvedTs() model.Ts { - // TODO: after TiCDC introduces p2p based resolved ts mechanism, TiCDC nodes - // will be able to cooperate replication status directly. Then we will add - // another replication barrier for consistent replication instead of reusing - // the global resolved-ts. - if redo.IsConsistentEnabled(t.replConfig.Consistent.Level) { - return t.sinkNode.ResolvedTs() - } - return t.sorterNode.ResolvedTs() -} - -// CheckpointTs returns the checkpoint ts in this table pipeline -func (t *tablePipelineImpl) CheckpointTs() model.Ts { - return t.sinkNode.CheckpointTs() -} - -// UpdateBarrierTs updates the barrier ts in this table pipeline -func (t *tablePipelineImpl) UpdateBarrierTs(ts model.Ts) { - err := t.p.SendToFirstNode(pipeline.BarrierMessage(ts)) - if err != nil && !cerror.ErrSendToClosedPipeline.Equal(err) && !cerror.ErrPipelineTryAgain.Equal(err) { - log.Panic("unexpect error from send to first node", zap.Error(err)) - } -} - -// AsyncStop tells the pipeline to stop, and returns true is the pipeline is already stopped. -func (t *tablePipelineImpl) AsyncStop(targetTs model.Ts) bool { - err := t.p.SendToFirstNode(pipeline.CommandMessage(&pipeline.Command{ - Tp: pipeline.CommandTypeStop, - })) - log.Info("send async stop signal to table", zap.Int64("tableID", t.tableID), zap.Uint64("targetTs", targetTs)) - if err != nil { - if cerror.ErrPipelineTryAgain.Equal(err) { - return false - } - if cerror.ErrSendToClosedPipeline.Equal(err) { - return true - } - log.Panic("unexpect error from send to first node", zap.Error(err)) - } - return true -} - -var workload = model.WorkloadInfo{Workload: 1} - -// Workload returns the workload of this table -func (t *tablePipelineImpl) Workload() model.WorkloadInfo { - // TODO(leoppro) calculate the workload of this table - // We temporarily set the value to constant 1 - return workload -} - -// Status returns the status of this table pipeline -func (t *tablePipelineImpl) Status() TableStatus { - return t.sinkNode.Status() -} - -// ID returns the ID of source table and mark table -func (t *tablePipelineImpl) ID() (tableID, markTableID int64) { - return t.tableID, t.markTableID -} - -// Name returns the quoted schema and table name -func (t *tablePipelineImpl) Name() string { - return t.tableName -} - -// Cancel stops this table pipeline immediately and destroy all resources created by this table pipeline -func (t *tablePipelineImpl) Cancel() { - t.cancel() -} - -// Wait waits for table pipeline destroyed -func (t *tablePipelineImpl) Wait() { - t.p.Wait() -} - -// Assume 1KB per row in upstream TiDB, it takes about 250 MB (1024*4*64) for -// replicating 1024 tables in the worst case. -const defaultOutputChannelSize = 64 - -// There are 4 or 5 runners in table pipeline: header, puller, sorter, -// sink, cyclic if cyclic replication is enabled -const defaultRunnersSize = 4 - -// NewTablePipeline creates a table pipeline -// TODO(leoppro): implement a mock kvclient to test the table pipeline -func NewTablePipeline(ctx cdcContext.Context, - mounter entry.Mounter, - tableID model.TableID, - tableName string, - replicaInfo *model.TableReplicaInfo, - sink sink.Sink, - targetTs model.Ts) TablePipeline { - ctx, cancel := cdcContext.WithCancel(ctx) - replConfig := ctx.ChangefeedVars().Info.Config - tablePipeline := &tablePipelineImpl{ - tableID: tableID, - markTableID: replicaInfo.MarkTableID, - tableName: tableName, - cancel: cancel, - replConfig: replConfig, - } - - perTableMemoryQuota := serverConfig.GetGlobalServerConfig().PerTableMemoryQuota - log.Debug("creating table flow controller", - zap.String("changefeed-id", ctx.ChangefeedVars().ID), - zap.String("table-name", tableName), - zap.Int64("table-id", tableID), - zap.Uint64("quota", perTableMemoryQuota)) - flowController := common.NewTableFlowController(perTableMemoryQuota) - config := ctx.ChangefeedVars().Info.Config - cyclicEnabled := config.Cyclic != nil && config.Cyclic.IsEnabled() - runnerSize := defaultRunnersSize - if cyclicEnabled { - runnerSize++ - } - - p := pipeline.NewPipeline(ctx, 500*time.Millisecond, runnerSize, defaultOutputChannelSize) - sorterNode := - newSorterNode(tableName, tableID, replicaInfo.StartTs, flowController, mounter, replConfig) - sinkNode := newSinkNode(tableID, sink, replicaInfo.StartTs, targetTs, flowController) - - p.AppendNode(ctx, "puller", newPullerNode(tableID, replicaInfo, tableName)) - p.AppendNode(ctx, "sorter", sorterNode) - if cyclicEnabled { - p.AppendNode(ctx, "cyclic", newCyclicMarkNode(replicaInfo.MarkTableID)) - } - p.AppendNode(ctx, "sink", sinkNode) - - tablePipeline.p = p - tablePipeline.sorterNode = sorterNode - tablePipeline.sinkNode = sinkNode - return tablePipeline -} diff --git a/cdc/cdc/processor/processor.go b/cdc/cdc/processor/processor.go index 9ac7c195..32764fb6 100644 --- a/cdc/cdc/processor/processor.go +++ b/cdc/cdc/processor/processor.go @@ -18,7 +18,6 @@ import ( "fmt" "io" "math" - "strconv" "sync" "time" @@ -27,45 +26,25 @@ import ( "github.com/pingcap/log" "github.com/prometheus/client_golang/prometheus" "github.com/tikv/client-go/v2/oracle" - "github.com/tikv/migration/cdc/cdc/entry" - "github.com/tikv/migration/cdc/cdc/kv" "github.com/tikv/migration/cdc/cdc/model" - tablepipeline "github.com/tikv/migration/cdc/cdc/processor/pipeline" - "github.com/tikv/migration/cdc/cdc/puller" - "github.com/tikv/migration/cdc/cdc/redo" + keyspanpipeline "github.com/tikv/migration/cdc/cdc/processor/pipeline" "github.com/tikv/migration/cdc/cdc/sink" - "github.com/tikv/migration/cdc/cdc/sorter/memory" "github.com/tikv/migration/cdc/pkg/config" cdcContext "github.com/tikv/migration/cdc/pkg/context" - "github.com/tikv/migration/cdc/pkg/cyclic/mark" cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/filter" "github.com/tikv/migration/cdc/pkg/orchestrator" - "github.com/tikv/migration/cdc/pkg/regionspan" - "github.com/tikv/migration/cdc/pkg/retry" "github.com/tikv/migration/cdc/pkg/util" "go.uber.org/zap" ) -const ( - backoffBaseDelayInMs = 5 - maxTries = 3 -) - type processor struct { changefeedID model.ChangeFeedID captureInfo *model.CaptureInfo changefeed *orchestrator.ChangefeedReactorState - tables map[model.TableID]tablepipeline.TablePipeline + keyspans map[model.KeySpanID]keyspanpipeline.KeySpanPipeline - schemaStorage entry.SchemaStorage - lastSchemaTs model.Ts - - filter *filter.Filter - mounter entry.Mounter sinkManager *sink.Manager - redoManager redo.LogManager lastRedoFlush time.Time initialized bool @@ -73,9 +52,9 @@ type processor struct { cancel context.CancelFunc wg sync.WaitGroup - lazyInit func(ctx cdcContext.Context) error - createTablePipeline func(ctx cdcContext.Context, tableID model.TableID, replicaInfo *model.TableReplicaInfo) (tablepipeline.TablePipeline, error) - newAgent func(ctx cdcContext.Context) (processorAgent, error) + lazyInit func(ctx cdcContext.Context) error + createKeySpanPipeline func(ctx cdcContext.Context, keyspanID model.KeySpanID, replicaInfo *model.KeySpanReplicaInfo) (keyspanpipeline.KeySpanPipeline, error) + newAgent func(ctx cdcContext.Context) (processorAgent, error) // fields for integration with Scheduler(V2). newSchedulerEnabled bool @@ -83,13 +62,12 @@ type processor struct { checkpointTs model.Ts resolvedTs model.Ts - metricResolvedTsGauge prometheus.Gauge - metricResolvedTsLagGauge prometheus.Gauge - metricCheckpointTsGauge prometheus.Gauge - metricCheckpointTsLagGauge prometheus.Gauge - metricSyncTableNumGauge prometheus.Gauge - metricSchemaStorageGcTsGauge prometheus.Gauge - metricProcessorErrorCounter prometheus.Counter + metricResolvedTsGauge prometheus.Gauge + metricResolvedTsLagGauge prometheus.Gauge + metricCheckpointTsGauge prometheus.Gauge + metricCheckpointTsLagGauge prometheus.Gauge + metricSyncKeySpanNumGauge prometheus.Gauge + metricProcessorErrorCounter prometheus.Counter } // checkReadyForMessages checks whether all necessary Etcd keys have been established. @@ -97,125 +75,130 @@ func (p *processor) checkReadyForMessages() bool { return p.changefeed != nil && p.changefeed.Status != nil } -// AddTable implements TableExecutor interface. -func (p *processor) AddTable(ctx cdcContext.Context, tableID model.TableID) (bool, error) { +// AddKeySpan implements KeySpanExecutor interface. +func (p *processor) AddKeySpan(ctx cdcContext.Context, keyspanID model.KeySpanID, start []byte, end []byte) (bool, error) { if !p.checkReadyForMessages() { return false, nil } - log.Info("adding table", - zap.Int64("table-id", tableID), + log.Info("adding keyspan", + zap.Uint64("keyspan-id", keyspanID), cdcContext.ZapFieldChangefeed(ctx)) - err := p.addTable(ctx, tableID, &model.TableReplicaInfo{}) + + keyspanReplicaInfo := &model.KeySpanReplicaInfo{ + Start: start, + End: end, + } + err := p.addKeySpan(ctx, keyspanID, keyspanReplicaInfo) if err != nil { return false, errors.Trace(err) } return true, nil } -// RemoveTable implements TableExecutor interface. -func (p *processor) RemoveTable(ctx cdcContext.Context, tableID model.TableID) (bool, error) { +// RemoveKeySpan implements KeySpanExecutor interface. +func (p *processor) RemoveKeySpan(ctx cdcContext.Context, keyspanID model.KeySpanID) (bool, error) { if !p.checkReadyForMessages() { return false, nil } - table, ok := p.tables[tableID] + keyspan, ok := p.keyspans[keyspanID] if !ok { - log.Warn("table which will be deleted is not found", - cdcContext.ZapFieldChangefeed(ctx), zap.Int64("tableID", tableID)) + log.Warn("keyspan which will be deleted is not found", + cdcContext.ZapFieldChangefeed(ctx), zap.Uint64("keyspanID", keyspanID)) return true, nil } boundaryTs := p.changefeed.Status.CheckpointTs - if !table.AsyncStop(boundaryTs) { + if !keyspan.AsyncStop(boundaryTs) { // We use a Debug log because it is conceivable for the pipeline to block for a legitimate reason, // and we do not want to alarm the user. log.Debug("AsyncStop has failed, possible due to a full pipeline", cdcContext.ZapFieldChangefeed(ctx), - zap.Uint64("checkpointTs", table.CheckpointTs()), - zap.Int64("tableID", tableID)) + zap.Uint64("checkpointTs", keyspan.CheckpointTs()), + zap.Uint64("keyspanID", keyspanID)) return false, nil } return true, nil } -// IsAddTableFinished implements TableExecutor interface. -func (p *processor) IsAddTableFinished(ctx cdcContext.Context, tableID model.TableID) bool { +// IsAddKeySpanFinished implements KeySpanExecutor interface. +func (p *processor) IsAddKeySpanFinished(ctx cdcContext.Context, keyspanID model.KeySpanID) bool { if !p.checkReadyForMessages() { return false } - table, exist := p.tables[tableID] + keyspan, exist := p.keyspans[keyspanID] if !exist { - log.Panic("table which was added is not found", + log.Panic("keyspan which was added is not found", cdcContext.ZapFieldChangefeed(ctx), - zap.Int64("tableID", tableID)) + zap.Uint64("keyspanID", keyspanID)) } localResolvedTs := p.resolvedTs globalResolvedTs := p.changefeed.Status.ResolvedTs localCheckpointTs := p.agent.GetLastSentCheckpointTs() globalCheckpointTs := p.changefeed.Status.CheckpointTs - // These two conditions are used to determine if the table's pipeline has finished + // These two conditions are used to determine if the keyspan's pipeline has finished // initializing and all invariants have been preserved. // // The processor needs to make sure all reasonable invariants about the checkpoint-ts and // the resolved-ts are preserved before communicating with the Owner. // // These conditions are similar to those in the legacy implementation of the Owner/Processor. - if table.CheckpointTs() < localCheckpointTs || localCheckpointTs < globalCheckpointTs { + if keyspan.CheckpointTs() < localCheckpointTs || localCheckpointTs < globalCheckpointTs { return false } - if table.ResolvedTs() < localResolvedTs || localResolvedTs < globalResolvedTs { + if keyspan.ResolvedTs() < localResolvedTs || localResolvedTs < globalResolvedTs { return false } - log.Info("Add Table finished", + log.Info("Add KeySpan finished", cdcContext.ZapFieldChangefeed(ctx), - zap.Int64("tableID", tableID)) + zap.Uint64("keyspanID", keyspanID)) return true } -// IsRemoveTableFinished implements TableExecutor interface. -func (p *processor) IsRemoveTableFinished(ctx cdcContext.Context, tableID model.TableID) bool { +// IsRemoveKeySpanFinished implements KeySpanExecutor interface. +func (p *processor) IsRemoveKeySpanFinished(ctx cdcContext.Context, keyspanID model.KeySpanID) bool { if !p.checkReadyForMessages() { return false } - table, exist := p.tables[tableID] + keyspan, exist := p.keyspans[keyspanID] if !exist { - log.Panic("table which was deleted is not found", + log.Panic("keyspan which was deleted is not found", cdcContext.ZapFieldChangefeed(ctx), - zap.Int64("tableID", tableID)) + zap.Uint64("keyspanID", keyspanID)) return true } - if table.Status() != tablepipeline.TableStatusStopped { - log.Debug("the table is still not stopped", + if keyspan.Status() != keyspanpipeline.KeySpanStatusStopped { + log.Debug("the keyspan is still not stopped", cdcContext.ZapFieldChangefeed(ctx), - zap.Uint64("checkpointTs", table.CheckpointTs()), - zap.Int64("tableID", tableID)) + zap.Uint64("checkpointTs", keyspan.CheckpointTs()), + zap.Uint64("keyspanID", keyspanID)) return false } - table.Cancel() - table.Wait() - delete(p.tables, tableID) - log.Info("Remove Table finished", + keyspan.Cancel() + keyspan.Wait() + delete(p.keyspans, keyspanID) + log.Info("Remove KeySpan finished", cdcContext.ZapFieldChangefeed(ctx), - zap.Int64("tableID", tableID)) + zap.Uint64("keyspanID", keyspanID)) return true } -// GetAllCurrentTables implements TableExecutor interface. -func (p *processor) GetAllCurrentTables() []model.TableID { - ret := make([]model.TableID, 0, len(p.tables)) - for tableID := range p.tables { - ret = append(ret, tableID) +// GetAllCurrentKeySpans implements KeySpanExecutor interface. +func (p *processor) GetAllCurrentKeySpans() []model.KeySpanID { + ret := make([]model.KeySpanID, 0, len(p.keyspans)) + for keyspanID := range p.keyspans { + ret = append(ret, keyspanID) } return ret } -// GetCheckpoint implements TableExecutor interface. +// GetCheckpoint implements KeySpanExecutor interface. func (p *processor) GetCheckpoint() (checkpointTs, resolvedTs model.Ts) { return p.checkpointTs, p.resolvedTs } @@ -226,7 +209,7 @@ func newProcessor(ctx cdcContext.Context) *processor { advertiseAddr := ctx.GlobalVars().CaptureInfo.AdvertiseAddr conf := config.GetGlobalServerConfig() p := &processor{ - tables: make(map[model.TableID]tablepipeline.TablePipeline), + keyspans: make(map[model.KeySpanID]keyspanpipeline.KeySpanPipeline), errCh: make(chan error, 1), changefeedID: changefeedID, captureInfo: ctx.GlobalVars().CaptureInfo, @@ -235,15 +218,15 @@ func newProcessor(ctx cdcContext.Context) *processor { newSchedulerEnabled: conf.Debug.EnableNewScheduler, - metricResolvedTsGauge: resolvedTsGauge.WithLabelValues(changefeedID, advertiseAddr), - metricResolvedTsLagGauge: resolvedTsLagGauge.WithLabelValues(changefeedID, advertiseAddr), - metricCheckpointTsGauge: checkpointTsGauge.WithLabelValues(changefeedID, advertiseAddr), - metricCheckpointTsLagGauge: checkpointTsLagGauge.WithLabelValues(changefeedID, advertiseAddr), - metricSyncTableNumGauge: syncTableNumGauge.WithLabelValues(changefeedID, advertiseAddr), - metricProcessorErrorCounter: processorErrorCounter.WithLabelValues(changefeedID, advertiseAddr), - metricSchemaStorageGcTsGauge: processorSchemaStorageGcTsGauge.WithLabelValues(changefeedID, advertiseAddr), + metricResolvedTsGauge: resolvedTsGauge.WithLabelValues(changefeedID, advertiseAddr), + metricResolvedTsLagGauge: resolvedTsLagGauge.WithLabelValues(changefeedID, advertiseAddr), + metricCheckpointTsGauge: checkpointTsGauge.WithLabelValues(changefeedID, advertiseAddr), + metricCheckpointTsLagGauge: checkpointTsLagGauge.WithLabelValues(changefeedID, advertiseAddr), + metricSyncKeySpanNumGauge: syncKeySpanNumGauge.WithLabelValues(changefeedID, advertiseAddr), + metricProcessorErrorCounter: processorErrorCounter.WithLabelValues(changefeedID, advertiseAddr), + // metricSchemaStorageGcTsGauge: processorSchemaStorageGcTsGauge.WithLabelValues(changefeedID, advertiseAddr), } - p.createTablePipeline = p.createTablePipelineImpl + p.createKeySpanPipeline = p.createKeySpanPipelineImpl p.lazyInit = p.lazyInitImpl p.newAgent = p.newAgentImpl return p @@ -274,7 +257,7 @@ func isProcessorIgnorableError(err error) bool { // Tick implements the `orchestrator.State` interface // the `state` parameter is sent by the etcd worker, the `state` must be a snapshot of KVs in etcd -// The main logic of processor is in this function, including the calculation of many kinds of ts, maintain table pipeline, error handling, etc. +// The main logic of processor is in this function, including the calculation of many kinds of ts, maintain keyspan pipeline, error handling, etc. func (p *processor) Tick(ctx cdcContext.Context, state *orchestrator.ChangefeedReactorState) (orchestrator.ReactorState, error) { p.changefeed = state state.CheckCaptureAlive(ctx.GlobalVars().CaptureInfo.ID) @@ -333,32 +316,28 @@ func (p *processor) tick(ctx cdcContext.Context, state *orchestrator.ChangefeedR } // sink manager will return this checkpointTs to sink node if sink node resolvedTs flush failed p.sinkManager.UpdateChangeFeedCheckpointTs(state.Info.GetCheckpointTs(state.Status)) - if err := p.handleTableOperation(ctx); err != nil { + if err := p.handleKeySpanOperation(ctx); err != nil { return nil, errors.Trace(err) } - if err := p.checkTablesNum(ctx); err != nil { + if err := p.checkKeySpansNum(ctx); err != nil { return nil, errors.Trace(err) } - if err := p.flushRedoLogMeta(ctx); err != nil { - return nil, err - } // it is no need to check the err here, because we will use // local time when an error return, which is acceptable pdTime, _ := ctx.GlobalVars().TimeAcquirer.CurrentTimeFromCached() p.handlePosition(oracle.GetPhysical(pdTime)) - p.pushResolvedTs2Table() + p.pushResolvedTs2KeySpan() // The workload key does not contain extra information and // will not be used in the new scheduler. If we wrote to the - // key while there are many tables (>10000), we would risk burdening Etcd. + // key while there are many keyspans (>10000), we would risk burdening Etcd. // // The keys will still exist but will no longer be written to // if we do not call handleWorkload. if !p.newSchedulerEnabled { p.handleWorkload() } - p.doGCSchemaStorage(ctx) if p.newSchedulerEnabled { if err := p.agent.Tick(ctx); err != nil { @@ -434,53 +413,41 @@ func (p *processor) lazyInitImpl(ctx cdcContext.Context) error { }() var err error - p.filter, err = filter.NewFilter(p.changefeed.Info.Config) - if err != nil { - return errors.Trace(err) - } - - p.schemaStorage, err = p.createAndDriveSchemaStorage(ctx) - if err != nil { - return errors.Trace(err) - } + /* + p.filter, err = filter.NewFilter(p.changefeed.Info.Config) + if err != nil { + return errors.Trace(err) + } + */ stdCtx := util.PutChangefeedIDInCtx(ctx, p.changefeed.ID) stdCtx = util.PutCaptureAddrInCtx(stdCtx, p.captureInfo.AdvertiseAddr) - p.mounter = entry.NewMounter(p.schemaStorage, p.changefeed.Info.Config.Mounter.WorkerNum, p.changefeed.Info.Config.EnableOldValue) - p.wg.Add(1) - go func() { - defer p.wg.Done() - p.sendError(p.mounter.Run(stdCtx)) - }() - opts := make(map[string]string, len(p.changefeed.Info.Opts)+2) for k, v := range p.changefeed.Info.Opts { opts[k] = v } // TODO(neil) find a better way to let sink know cyclic is enabled. - if p.changefeed.Info.Config.Cyclic.IsEnabled() { - cyclicCfg, err := p.changefeed.Info.Config.Cyclic.Marshal() - if err != nil { - return errors.Trace(err) + // TODO: it's useless for tikv cdc. + /* + if p.changefeed.Info.Config.Cyclic.IsEnabled() { + cyclicCfg, err := p.changefeed.Info.Config.Cyclic.Marshal() + if err != nil { + return errors.Trace(err) + } + opts[mark.OptCyclicConfig] = cyclicCfg } - opts[mark.OptCyclicConfig] = cyclicCfg - } + */ opts[sink.OptChangefeedID] = p.changefeed.ID opts[sink.OptCaptureAddr] = ctx.GlobalVars().CaptureInfo.AdvertiseAddr - s, err := sink.New(stdCtx, p.changefeed.ID, p.changefeed.Info.SinkURI, p.filter, p.changefeed.Info.Config, opts, errCh) + s, err := sink.New(stdCtx, p.changefeed.ID, p.changefeed.Info.SinkURI, p.changefeed.Info.Config, opts, errCh) if err != nil { return errors.Trace(err) } checkpointTs := p.changefeed.Info.GetCheckpointTs(p.changefeed.Status) captureAddr := ctx.GlobalVars().CaptureInfo.AdvertiseAddr p.sinkManager = sink.NewManager(stdCtx, s, errCh, checkpointTs, captureAddr, p.changefeedID) - redoManagerOpts := &redo.ManagerOptions{EnableBgRunner: true, ErrCh: errCh} - p.redoManager, err = redo.NewManager(stdCtx, p.changefeed.Info.Config.Consistent, redoManagerOpts) - if err != nil { - return err - } if p.newSchedulerEnabled { p.agent, err = p.newAgent(ctx) @@ -523,21 +490,21 @@ func (p *processor) handleErrorCh(ctx cdcContext.Context) error { return cerror.ErrReactorFinished } -// handleTableOperation handles the operation of `TaskStatus`(add table operation and remove table operation) -func (p *processor) handleTableOperation(ctx cdcContext.Context) error { +// handleKeySpanOperation handles the operation of `TaskStatus`(add keyspan operation and remove keyspan operation) +func (p *processor) handleKeySpanOperation(ctx cdcContext.Context) error { if p.newSchedulerEnabled { return nil } - patchOperation := func(tableID model.TableID, fn func(operation *model.TableOperation) error) { + patchOperation := func(keyspanID model.KeySpanID, fn func(operation *model.KeySpanOperation) error) { p.changefeed.PatchTaskStatus(p.captureInfo.ID, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { if status == nil || status.Operation == nil { - log.Error("Operation not found, may be remove by other patch", zap.Int64("tableID", tableID), zap.Any("status", status)) + log.Error("Operation not found, may be remove by other patch", zap.Uint64("keyspanID", keyspanID), zap.Any("status", status)) return nil, false, cerror.ErrTaskStatusNotExists.GenWithStackByArgs() } - opt := status.Operation[tableID] + opt := status.Operation[keyspanID] if opt == nil { - log.Error("Operation not found, may be remove by other patch", zap.Int64("tableID", tableID), zap.Any("status", status)) + log.Error("Operation not found, may be remove by other patch", zap.Uint64("keyspanID", keyspanID), zap.Any("status", status)) return nil, false, cerror.ErrTaskStatusNotExists.GenWithStackByArgs() } if err := fn(opt); err != nil { @@ -547,17 +514,17 @@ func (p *processor) handleTableOperation(ctx cdcContext.Context) error { }) } taskStatus := p.changefeed.TaskStatuses[p.captureInfo.ID] - for tableID, opt := range taskStatus.Operation { - if opt.TableApplied() { + for keyspanID, opt := range taskStatus.Operation { + if opt.KeySpanApplied() { continue } globalCheckpointTs := p.changefeed.Status.CheckpointTs if opt.Delete { - table, exist := p.tables[tableID] + keyspan, exist := p.keyspans[keyspanID] if !exist { - log.Warn("table which will be deleted is not found", - cdcContext.ZapFieldChangefeed(ctx), zap.Int64("tableID", tableID)) - patchOperation(tableID, func(operation *model.TableOperation) error { + log.Warn("keyspan which will be deleted is not found", + cdcContext.ZapFieldChangefeed(ctx), zap.Uint64("keyspanID", keyspanID)) + patchOperation(keyspanID, func(operation *model.KeySpanOperation) error { operation.Status = model.OperFinished return nil }) @@ -566,33 +533,33 @@ func (p *processor) handleTableOperation(ctx cdcContext.Context) error { switch opt.Status { case model.OperDispatched: if opt.BoundaryTs < globalCheckpointTs { - log.Warn("the BoundaryTs of remove table operation is smaller than global checkpoint ts", zap.Uint64("globalCheckpointTs", globalCheckpointTs), zap.Any("operation", opt)) + log.Warn("the BoundaryTs of remove keyspan operation is smaller than global checkpoint ts", zap.Uint64("globalCheckpointTs", globalCheckpointTs), zap.Any("operation", opt)) } - if !table.AsyncStop(opt.BoundaryTs) { + if !keyspan.AsyncStop(opt.BoundaryTs) { // We use a Debug log because it is conceivable for the pipeline to block for a legitimate reason, // and we do not want to alarm the user. log.Debug("AsyncStop has failed, possible due to a full pipeline", - zap.Uint64("checkpointTs", table.CheckpointTs()), zap.Int64("tableID", tableID)) + zap.Uint64("checkpointTs", keyspan.CheckpointTs()), zap.Uint64("keyspanID", keyspanID)) continue } - patchOperation(tableID, func(operation *model.TableOperation) error { + patchOperation(keyspanID, func(operation *model.KeySpanOperation) error { operation.Status = model.OperProcessed return nil }) case model.OperProcessed: - if table.Status() != tablepipeline.TableStatusStopped { - log.Debug("the table is still not stopped", zap.Uint64("checkpointTs", table.CheckpointTs()), zap.Int64("tableID", tableID)) + if keyspan.Status() != keyspanpipeline.KeySpanStatusStopped { + log.Debug("the keyspan is still not stopped", zap.Uint64("checkpointTs", keyspan.CheckpointTs()), zap.Uint64("keyspanID", keyspanID)) continue } - patchOperation(tableID, func(operation *model.TableOperation) error { - operation.BoundaryTs = table.CheckpointTs() + patchOperation(keyspanID, func(operation *model.KeySpanOperation) error { + operation.BoundaryTs = keyspan.CheckpointTs() operation.Status = model.OperFinished return nil }) - p.removeTable(table, tableID) + p.removeKeySpan(keyspan, keyspanID) log.Debug("Operation done signal received", cdcContext.ZapFieldChangefeed(ctx), - zap.Int64("tableID", tableID), + zap.Uint64("keyspanID", keyspanID), zap.Reflect("operation", opt)) default: log.Panic("unreachable") @@ -600,27 +567,27 @@ func (p *processor) handleTableOperation(ctx cdcContext.Context) error { } else { switch opt.Status { case model.OperDispatched: - replicaInfo, exist := taskStatus.Tables[tableID] + replicaInfo, exist := taskStatus.KeySpans[keyspanID] if !exist { - return cerror.ErrProcessorTableNotFound.GenWithStack("replicaInfo of table(%d)", tableID) + return cerror.ErrProcessorKeySpanNotFound.GenWithStack("replicaInfo of keyspan(%d)", keyspanID) } if replicaInfo.StartTs != opt.BoundaryTs { - log.Warn("the startTs and BoundaryTs of add table operation should be always equaled", zap.Any("replicaInfo", replicaInfo)) + log.Warn("the startTs and BoundaryTs of add keyspan operation should be always equaled", zap.Any("replicaInfo", replicaInfo)) } - err := p.addTable(ctx, tableID, replicaInfo) + err := p.addKeySpan(ctx, keyspanID, replicaInfo) if err != nil { return errors.Trace(err) } - patchOperation(tableID, func(operation *model.TableOperation) error { + patchOperation(keyspanID, func(operation *model.KeySpanOperation) error { operation.Status = model.OperProcessed return nil }) case model.OperProcessed: - table, exist := p.tables[tableID] + keyspan, exist := p.keyspans[keyspanID] if !exist { - log.Warn("table which was added is not found", - cdcContext.ZapFieldChangefeed(ctx), zap.Int64("tableID", tableID)) - patchOperation(tableID, func(operation *model.TableOperation) error { + log.Warn("keyspan which was added is not found", + cdcContext.ZapFieldChangefeed(ctx), zap.Uint64("keyspanID", keyspanID)) + patchOperation(keyspanID, func(operation *model.KeySpanOperation) error { operation.Status = model.OperDispatched return nil }) @@ -628,14 +595,14 @@ func (p *processor) handleTableOperation(ctx cdcContext.Context) error { } localResolvedTs := p.changefeed.TaskPositions[p.captureInfo.ID].ResolvedTs globalResolvedTs := p.changefeed.Status.ResolvedTs - if table.ResolvedTs() >= localResolvedTs && localResolvedTs >= globalResolvedTs { - patchOperation(tableID, func(operation *model.TableOperation) error { + if keyspan.ResolvedTs() >= localResolvedTs && localResolvedTs >= globalResolvedTs { + patchOperation(keyspanID, func(operation *model.KeySpanOperation) error { operation.Status = model.OperFinished return nil }) log.Debug("Operation done signal received", cdcContext.ZapFieldChangefeed(ctx), - zap.Int64("tableID", tableID), + zap.Uint64("keyspanID", keyspanID), zap.Reflect("operation", opt)) } default: @@ -646,67 +613,6 @@ func (p *processor) handleTableOperation(ctx cdcContext.Context) error { return nil } -func (p *processor) createAndDriveSchemaStorage(ctx cdcContext.Context) (entry.SchemaStorage, error) { - kvStorage := ctx.GlobalVars().KVStorage - ddlspans := []regionspan.Span{regionspan.GetDDLSpan(), regionspan.GetAddIndexDDLSpan()} - checkpointTs := p.changefeed.Info.GetCheckpointTs(p.changefeed.Status) - stdCtx := util.PutTableInfoInCtx(ctx, -1, puller.DDLPullerTableName) - stdCtx = util.PutChangefeedIDInCtx(stdCtx, ctx.ChangefeedVars().ID) - ddlPuller := puller.NewPuller( - stdCtx, - ctx.GlobalVars().PDClient, - ctx.GlobalVars().GrpcPool, - ctx.GlobalVars().RegionCache, - ctx.GlobalVars().KVStorage, - checkpointTs, ddlspans, false) - meta, err := kv.GetSnapshotMeta(kvStorage, checkpointTs) - if err != nil { - return nil, errors.Trace(err) - } - schemaStorage, err := entry.NewSchemaStorage(meta, checkpointTs, p.filter, p.changefeed.Info.Config.ForceReplicate) - if err != nil { - return nil, errors.Trace(err) - } - p.wg.Add(1) - go func() { - defer p.wg.Done() - p.sendError(ddlPuller.Run(stdCtx)) - }() - ddlRawKVCh := memory.SortOutput(ctx, ddlPuller.Output()) - p.wg.Add(1) - go func() { - defer p.wg.Done() - var ddlRawKV *model.RawKVEntry - for { - select { - case <-ctx.Done(): - return - case ddlRawKV = <-ddlRawKVCh: - } - if ddlRawKV == nil { - continue - } - failpoint.Inject("processorDDLResolved", nil) - if ddlRawKV.OpType == model.OpTypeResolved { - schemaStorage.AdvanceResolvedTs(ddlRawKV.CRTs) - } - job, err := entry.UnmarshalDDL(ddlRawKV) - if err != nil { - p.sendError(errors.Trace(err)) - return - } - if job == nil { - continue - } - if err := schemaStorage.HandleDDLJob(job); err != nil { - p.sendError(errors.Trace(err)) - return - } - } - }() - return schemaStorage, nil -} - func (p *processor) sendError(err error) { if err == nil { return @@ -720,51 +626,51 @@ func (p *processor) sendError(err error) { } } -// checkTablesNum if the number of table pipelines is equal to the number of TaskStatus in etcd state. -// if the table number is not right, create or remove the odd tables. -func (p *processor) checkTablesNum(ctx cdcContext.Context) error { +// checkKeySpansNum if the number of keyspan pipelines is equal to the number of TaskStatus in etcd state. +// if the keyspan number is not right, create or remove the odd keyspans. +func (p *processor) checkKeySpansNum(ctx cdcContext.Context) error { if p.newSchedulerEnabled { // No need to check this for the new scheduler. return nil } taskStatus := p.changefeed.TaskStatuses[p.captureInfo.ID] - if len(p.tables) == len(taskStatus.Tables) { + if len(p.keyspans) == len(taskStatus.KeySpans) { return nil } - // check if a table should be listen but not + // check if a keyspan should be listen but not // this only could be happened in the first tick. - for tableID, replicaInfo := range taskStatus.Tables { - if _, exist := p.tables[tableID]; exist { + for keyspanID, replicaInfo := range taskStatus.KeySpans { + if _, exist := p.keyspans[keyspanID]; exist { continue } opt := taskStatus.Operation // TODO(leoppro): check if the operation is a undone add operation - if opt != nil && opt[tableID] != nil { + if opt != nil && opt[keyspanID] != nil { continue } - log.Info("start to listen to the table immediately", zap.Int64("tableID", tableID), zap.Any("replicaInfo", replicaInfo)) + log.Info("start to listen to the keyspan immediately", zap.Uint64("keyspanID", keyspanID), zap.Any("replicaInfo", replicaInfo)) if replicaInfo.StartTs < p.changefeed.Status.CheckpointTs { replicaInfo.StartTs = p.changefeed.Status.CheckpointTs } - err := p.addTable(ctx, tableID, replicaInfo) + err := p.addKeySpan(ctx, keyspanID, replicaInfo) if err != nil { return errors.Trace(err) } } - // check if a table should be removed but still exist + // check if a keyspan should be removed but still exist // this shouldn't be happened in any time. - for tableID, tablePipeline := range p.tables { - if _, exist := taskStatus.Tables[tableID]; exist { + for keyspanID, keyspanPipeline := range p.keyspans { + if _, exist := taskStatus.KeySpans[keyspanID]; exist { continue } opt := taskStatus.Operation - if opt != nil && opt[tableID] != nil && opt[tableID].Delete { - // table will be removed by normal logic + if opt != nil && opt[keyspanID] != nil && opt[keyspanID].Delete { + // keyspan will be removed by normal logic continue } - p.removeTable(tablePipeline, tableID) - log.Warn("the table was forcibly deleted", zap.Int64("tableID", tableID), zap.Any("taskStatus", taskStatus)) + p.removeKeySpan(keyspanPipeline, keyspanID) + log.Warn("the keyspan was forcibly deleted", zap.Uint64("keyspanID", keyspanID), zap.Any("taskStatus", taskStatus)) } return nil } @@ -772,19 +678,17 @@ func (p *processor) checkTablesNum(ctx cdcContext.Context) error { // handlePosition calculates the local resolved ts and local checkpoint ts func (p *processor) handlePosition(currentTs int64) { minResolvedTs := uint64(math.MaxUint64) - if p.schemaStorage != nil { - minResolvedTs = p.schemaStorage.ResolvedTs() - } - for _, table := range p.tables { - ts := table.ResolvedTs() + + for _, keyspan := range p.keyspans { + ts := keyspan.ResolvedTs() if ts < minResolvedTs { minResolvedTs = ts } } minCheckpointTs := minResolvedTs - for _, table := range p.tables { - ts := table.CheckpointTs() + for _, keyspan := range p.keyspans { + ts := keyspan.CheckpointTs() if ts < minCheckpointTs { minCheckpointTs = ts } @@ -804,7 +708,7 @@ func (p *processor) handlePosition(currentTs int64) { return } - // minResolvedTs and minCheckpointTs may less than global resolved ts and global checkpoint ts when a new table added, the startTs of the new table is less than global checkpoint ts. + // minResolvedTs and minCheckpointTs may less than global resolved ts and global checkpoint ts when a new keyspan added, the startTs of the new keyspan is less than global checkpoint ts. if minResolvedTs != p.changefeed.TaskPositions[p.captureInfo.ID].ResolvedTs || minCheckpointTs != p.changefeed.TaskPositions[p.captureInfo.ID].CheckPointTs { p.changefeed.PatchTaskPosition(p.captureInfo.ID, func(position *model.TaskPosition) (*model.TaskPosition, bool, error) { @@ -822,22 +726,22 @@ func (p *processor) handlePosition(currentTs int64) { } } -// handleWorkload calculates the workload of all tables +// handleWorkload calculates the workload of all keyspans func (p *processor) handleWorkload() { p.changefeed.PatchTaskWorkload(p.captureInfo.ID, func(workloads model.TaskWorkload) (model.TaskWorkload, bool, error) { changed := false if workloads == nil { workloads = make(model.TaskWorkload) } - for tableID := range workloads { - if _, exist := p.tables[tableID]; !exist { - delete(workloads, tableID) + for keyspanID := range workloads { + if _, exist := p.keyspans[keyspanID]; !exist { + delete(workloads, keyspanID) changed = true } } - for tableID, table := range p.tables { - if workloads[tableID] != table.Workload() { - workloads[tableID] = table.Workload() + for keyspanID, keyspan := range p.keyspans { + if workloads[keyspanID] != keyspan.Workload() { + workloads[keyspanID] = keyspan.Workload() changed = true } } @@ -845,34 +749,26 @@ func (p *processor) handleWorkload() { }) } -// pushResolvedTs2Table sends global resolved ts to all the table pipelines. -func (p *processor) pushResolvedTs2Table() { +// pushResolvedTs2KeySpan sends global resolved ts to all the keyspan pipelines. +func (p *processor) pushResolvedTs2KeySpan() { resolvedTs := p.changefeed.Status.ResolvedTs - schemaResolvedTs := p.schemaStorage.ResolvedTs() - if schemaResolvedTs < resolvedTs { - // Do not update barrier ts that is larger than - // DDL puller's resolved ts. - // When DDL puller stall, resolved events that outputted by sorter - // may pile up in memory, as they have to wait DDL. - resolvedTs = schemaResolvedTs - } - for _, table := range p.tables { - table.UpdateBarrierTs(resolvedTs) + for _, keyspan := range p.keyspans { + keyspan.UpdateBarrierTs(resolvedTs) } } -// addTable creates a new table pipeline and adds it to the `p.tables` -func (p *processor) addTable(ctx cdcContext.Context, tableID model.TableID, replicaInfo *model.TableReplicaInfo) error { +// addKeySpan creates a new keyspan pipeline and adds it to the `p.keyspans` +func (p *processor) addKeySpan(ctx cdcContext.Context, keyspanID model.KeySpanID, replicaInfo *model.KeySpanReplicaInfo) error { if replicaInfo.StartTs == 0 { replicaInfo.StartTs = p.changefeed.Status.CheckpointTs } - if table, ok := p.tables[tableID]; ok { - if table.Status() == tablepipeline.TableStatusStopped { - log.Warn("The same table exists but is stopped. Cancel it and continue.", cdcContext.ZapFieldChangefeed(ctx), zap.Int64("ID", tableID)) - p.removeTable(table, tableID) + if keyspan, ok := p.keyspans[keyspanID]; ok { + if keyspan.Status() == keyspanpipeline.KeySpanStatusStopped { + log.Warn("The same keyspan exists but is stopped. Cancel it and continue.", cdcContext.ZapFieldChangefeed(ctx), zap.Uint64("ID", keyspanID)) + p.removeKeySpan(keyspan, keyspanID) } else { - log.Warn("Ignore existing table", cdcContext.ZapFieldChangefeed(ctx), zap.Int64("ID", tableID)) + log.Warn("Ignore existing keyspan", cdcContext.ZapFieldChangefeed(ctx), zap.Uint64("ID", keyspanID)) return nil } } @@ -880,172 +776,85 @@ func (p *processor) addTable(ctx cdcContext.Context, tableID model.TableID, repl globalCheckpointTs := p.changefeed.Status.CheckpointTs if replicaInfo.StartTs < globalCheckpointTs { - log.Warn("addTable: startTs < checkpoint", + log.Warn("addKeySpan: startTs < checkpoint", cdcContext.ZapFieldChangefeed(ctx), - zap.Int64("tableID", tableID), + zap.Uint64("keyspanID", keyspanID), zap.Uint64("checkpoint", globalCheckpointTs), zap.Uint64("startTs", replicaInfo.StartTs)) } - table, err := p.createTablePipeline(ctx, tableID, replicaInfo) + keyspan, err := p.createKeySpanPipeline(ctx, keyspanID, replicaInfo) if err != nil { return errors.Trace(err) } - p.tables[tableID] = table + p.keyspans[keyspanID] = keyspan return nil } -func (p *processor) createTablePipelineImpl(ctx cdcContext.Context, tableID model.TableID, replicaInfo *model.TableReplicaInfo) (tablepipeline.TablePipeline, error) { +func (p *processor) createKeySpanPipelineImpl(ctx cdcContext.Context, keyspanID model.KeySpanID, replicaInfo *model.KeySpanReplicaInfo) (keyspanpipeline.KeySpanPipeline, error) { ctx = cdcContext.WithErrorHandler(ctx, func(err error) error { - if cerror.ErrTableProcessorStoppedSafely.Equal(err) || + if cerror.ErrKeySpanProcessorStoppedSafely.Equal(err) || errors.Cause(errors.Cause(err)) == context.Canceled { return nil } p.sendError(err) return nil }) - var tableName *model.TableName - retry.Do(ctx, func() error { //nolint:errcheck - if name, ok := p.schemaStorage.GetLastSnapshot().GetTableNameByID(tableID); ok { - tableName = &name - return nil - } - return errors.Errorf("failed to get table name, fallback to use table id: %d", tableID) - }, retry.WithBackoffBaseDelay(backoffBaseDelayInMs), retry.WithMaxTries(maxTries), retry.WithIsRetryableErr(cerror.IsRetryableError)) - if p.changefeed.Info.Config.Cyclic.IsEnabled() { - // Retry to find mark table ID - var markTableID model.TableID - err := retry.Do(context.Background(), func() error { - if tableName == nil { - name, exist := p.schemaStorage.GetLastSnapshot().GetTableNameByID(tableID) - if !exist { - return cerror.ErrProcessorTableNotFound.GenWithStack("normal table(%s)", tableID) - } - tableName = &name - } - markTableSchemaName, markTableTableName := mark.GetMarkTableName(tableName.Schema, tableName.Table) - tableInfo, exist := p.schemaStorage.GetLastSnapshot().GetTableByName(markTableSchemaName, markTableTableName) - if !exist { - return cerror.ErrProcessorTableNotFound.GenWithStack("normal table(%s) and mark table not match", tableName.String()) - } - markTableID = tableInfo.ID - return nil - }, retry.WithBackoffBaseDelay(50), retry.WithBackoffMaxDelay(60*1000), retry.WithMaxTries(20)) - if err != nil { - return nil, errors.Trace(err) - } - replicaInfo.MarkTableID = markTableID - } - var tableNameStr string - if tableName == nil { - log.Warn("failed to get table name for metric") - tableNameStr = strconv.Itoa(int(tableID)) - } else { - tableNameStr = tableName.QuoteString() - } - sink := p.sinkManager.CreateTableSink(tableID, replicaInfo.StartTs, p.redoManager) - table := tablepipeline.NewTablePipeline( + // sink := p.sinkManager.CreateKeySpanSink(keyspanID, replicaInfo.StartTs, p.redoManager) + sink := p.sinkManager.CreateKeySpanSink(keyspanID, replicaInfo.StartTs) + keyspan := keyspanpipeline.NewKeySpanPipeline( ctx, - p.mounter, - tableID, - tableNameStr, + // p.mounter, + keyspanID, replicaInfo, sink, p.changefeed.Info.GetTargetTs(), ) p.wg.Add(1) - p.metricSyncTableNumGauge.Inc() + p.metricSyncKeySpanNumGauge.Inc() go func() { - table.Wait() + keyspan.Wait() p.wg.Done() - p.metricSyncTableNumGauge.Dec() - log.Debug("Table pipeline exited", zap.Int64("tableID", tableID), + p.metricSyncKeySpanNumGauge.Dec() + log.Debug("KeySpan pipeline exited", zap.Uint64("keyspanID", keyspanID), cdcContext.ZapFieldChangefeed(ctx), - zap.String("name", table.Name()), + zap.String("name", keyspan.Name()), zap.Any("replicaInfo", replicaInfo)) }() - if p.redoManager.Enabled() { - p.redoManager.AddTable(tableID, replicaInfo.StartTs) - } - - log.Info("Add table pipeline", zap.Int64("tableID", tableID), + log.Info("Add keyspan pipeline", zap.Uint64("keyspanID", keyspanID), cdcContext.ZapFieldChangefeed(ctx), - zap.String("name", table.Name()), + zap.String("name", keyspan.Name()), zap.Any("replicaInfo", replicaInfo), zap.Uint64("globalResolvedTs", p.changefeed.Status.ResolvedTs)) - return table, nil + return keyspan, nil } -func (p *processor) removeTable(table tablepipeline.TablePipeline, tableID model.TableID) { - table.Cancel() - table.Wait() - delete(p.tables, tableID) - if p.redoManager.Enabled() { - p.redoManager.RemoveTable(tableID) - } -} - -// doGCSchemaStorage trigger the schema storage GC -func (p *processor) doGCSchemaStorage(ctx cdcContext.Context) { - if p.schemaStorage == nil { - // schemaStorage is nil only in test - return - } - - if p.changefeed.Status == nil { - // This could happen if Etcd data is not complete. - return - } - - // Please refer to `unmarshalAndMountRowChanged` in cdc/entry/mounter.go - // for why we need -1. - lastSchemaTs := p.schemaStorage.DoGC(p.changefeed.Status.CheckpointTs - 1) - if p.lastSchemaTs == lastSchemaTs { - return - } - p.lastSchemaTs = lastSchemaTs - - log.Debug("finished gc in schema storage", - zap.Uint64("gcTs", lastSchemaTs), - cdcContext.ZapFieldChangefeed(ctx)) - lastSchemaPhysicalTs := oracle.ExtractPhysical(lastSchemaTs) - p.metricSchemaStorageGcTsGauge.Set(float64(lastSchemaPhysicalTs)) -} - -// flushRedoLogMeta flushes redo log meta, including resolved-ts and checkpoint-ts -func (p *processor) flushRedoLogMeta(ctx context.Context) error { - if p.redoManager.Enabled() && - time.Since(p.lastRedoFlush).Milliseconds() > p.changefeed.Info.Config.Consistent.FlushIntervalInMs { - st := p.changefeed.Status - err := p.redoManager.FlushResolvedAndCheckpointTs(ctx, st.ResolvedTs, st.CheckpointTs) - if err != nil { - return err - } - p.lastRedoFlush = time.Now() - } - return nil +func (p *processor) removeKeySpan(keyspan keyspanpipeline.KeySpanPipeline, keyspanID model.KeySpanID) { + keyspan.Cancel() + keyspan.Wait() + delete(p.keyspans, keyspanID) } func (p *processor) Close() error { - for _, tbl := range p.tables { + for _, tbl := range p.keyspans { tbl.Cancel() } - for _, tbl := range p.tables { + for _, tbl := range p.keyspans { tbl.Wait() } p.cancel() p.wg.Wait() - // mark tables share the same cdcContext with its original table, don't need to cancel + // mark keyspans share the same cdcContext with its original keyspan, don't need to cancel failpoint.Inject("processorStopDelay", nil) resolvedTsGauge.DeleteLabelValues(p.changefeedID, p.captureInfo.AdvertiseAddr) resolvedTsLagGauge.DeleteLabelValues(p.changefeedID, p.captureInfo.AdvertiseAddr) checkpointTsGauge.DeleteLabelValues(p.changefeedID, p.captureInfo.AdvertiseAddr) checkpointTsLagGauge.DeleteLabelValues(p.changefeedID, p.captureInfo.AdvertiseAddr) - syncTableNumGauge.DeleteLabelValues(p.changefeedID, p.captureInfo.AdvertiseAddr) + syncKeySpanNumGauge.DeleteLabelValues(p.changefeedID, p.captureInfo.AdvertiseAddr) processorErrorCounter.DeleteLabelValues(p.changefeedID, p.captureInfo.AdvertiseAddr) - processorSchemaStorageGcTsGauge.DeleteLabelValues(p.changefeedID, p.captureInfo.AdvertiseAddr) + // processorSchemaStorageGcTsGauge.DeleteLabelValues(p.changefeedID, p.captureInfo.AdvertiseAddr) if p.sinkManager != nil { // pass a canceled context is ok here, since we don't need to wait Close ctx, cancel := context.WithCancel(context.Background()) @@ -1070,8 +879,8 @@ func (p *processor) Close() error { // WriteDebugInfo write the debug info to Writer func (p *processor) WriteDebugInfo(w io.Writer) { fmt.Fprintf(w, "%+v\n", *p.changefeed) - for tableID, tablePipeline := range p.tables { - fmt.Fprintf(w, "tableID: %d, tableName: %s, resolvedTs: %d, checkpointTs: %d, status: %s\n", - tableID, tablePipeline.Name(), tablePipeline.ResolvedTs(), tablePipeline.CheckpointTs(), tablePipeline.Status()) + for keyspanID, keyspanPipeline := range p.keyspans { + fmt.Fprintf(w, "keyspanID: %d, keyspanName: %s, resolvedTs: %d, checkpointTs: %d, status: %s\n", + keyspanID, keyspanPipeline.Name(), keyspanPipeline.ResolvedTs(), keyspanPipeline.CheckpointTs(), keyspanPipeline.Status()) } } diff --git a/cdc/cdc/processor/processor_test.go b/cdc/cdc/processor/processor_test.go index cd66b1ec..c225d79a 100644 --- a/cdc/cdc/processor/processor_test.go +++ b/cdc/cdc/processor/processor_test.go @@ -17,17 +17,13 @@ import ( "context" "encoding/json" "fmt" - "math" - "sync/atomic" "testing" "github.com/pingcap/check" "github.com/pingcap/errors" "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/entry" "github.com/tikv/migration/cdc/cdc/model" - tablepipeline "github.com/tikv/migration/cdc/cdc/processor/pipeline" - "github.com/tikv/migration/cdc/cdc/redo" + keyspanpipeline "github.com/tikv/migration/cdc/cdc/processor/pipeline" "github.com/tikv/migration/cdc/cdc/scheduler" "github.com/tikv/migration/cdc/cdc/sink" cdcContext "github.com/tikv/migration/cdc/pkg/context" @@ -43,131 +39,110 @@ type processorSuite struct{} var _ = check.Suite(&processorSuite{}) -// processor needs to implement TableExecutor. -var _ scheduler.TableExecutor = (*processor)(nil) +// processor needs to implement KeySpanExecutor. +var _ scheduler.KeySpanExecutor = (*processor)(nil) func newProcessor4Test( ctx cdcContext.Context, - c *check.C, - createTablePipeline func(ctx cdcContext.Context, tableID model.TableID, replicaInfo *model.TableReplicaInfo) (tablepipeline.TablePipeline, error), + _ *check.C, + createKeySpanPipeline func(ctx cdcContext.Context, keyspanID model.KeySpanID, replicaInfo *model.KeySpanReplicaInfo) (keyspanpipeline.KeySpanPipeline, error), ) *processor { p := newProcessor(ctx) // disable new scheduler to pass old test cases // TODO refactor the test cases so that new scheduler can be enabled - p.newSchedulerEnabled = false + // p.newSchedulerEnabled = true p.lazyInit = func(ctx cdcContext.Context) error { return nil } p.sinkManager = &sink.Manager{} - p.redoManager = redo.NewDisabledManager() - p.createTablePipeline = createTablePipeline - p.schemaStorage = &mockSchemaStorage{c: c, resolvedTs: math.MaxUint64} + p.agent = &mockAgent{executor: p} + p.createKeySpanPipeline = createKeySpanPipeline return p } func initProcessor4Test(ctx cdcContext.Context, c *check.C) (*processor, *orchestrator.ReactorStateTester) { - p := newProcessor4Test(ctx, c, func(ctx cdcContext.Context, tableID model.TableID, replicaInfo *model.TableReplicaInfo) (tablepipeline.TablePipeline, error) { - return &mockTablePipeline{ - tableID: tableID, - name: fmt.Sprintf("`test`.`table%d`", tableID), - status: tablepipeline.TableStatusRunning, + p := newProcessor4Test(ctx, c, func(ctx cdcContext.Context, keyspanID model.KeySpanID, replicaInfo *model.KeySpanReplicaInfo) (keyspanpipeline.KeySpanPipeline, error) { + return &mockKeySpanPipeline{ + keyspanID: keyspanID, + name: fmt.Sprintf("`test`.`keyspan%d`", keyspanID), + status: keyspanpipeline.KeySpanStatusRunning, resolvedTs: replicaInfo.StartTs, checkpointTs: replicaInfo.StartTs, }, nil }) p.changefeed = orchestrator.NewChangefeedReactorState(ctx.ChangefeedVars().ID) return p, orchestrator.NewReactorStateTester(c, p.changefeed, map[string]string{ - "/tidb/cdc/capture/" + ctx.GlobalVars().CaptureInfo.ID: `{"id":"` + ctx.GlobalVars().CaptureInfo.ID + `","address":"127.0.0.1:8300"}`, - "/tidb/cdc/changefeed/info/" + ctx.ChangefeedVars().ID: `{"sink-uri":"blackhole://","opts":{},"create-time":"2020-02-02T00:00:00.000000+00:00","start-ts":0,"target-ts":0,"admin-job-type":0,"sort-engine":"memory","sort-dir":".","config":{"case-sensitive":true,"enable-old-value":false,"force-replicate":false,"check-gc-safe-point":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"open-protocol"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"table-number","polling-time":-1}},"state":"normal","history":null,"error":null,"sync-point-enabled":false,"sync-point-interval":600000000000}`, - "/tidb/cdc/job/" + ctx.ChangefeedVars().ID: `{"resolved-ts":0,"checkpoint-ts":0,"admin-job-type":0}`, - "/tidb/cdc/task/status/" + ctx.GlobalVars().CaptureInfo.ID + "/" + ctx.ChangefeedVars().ID: `{"tables":{},"operation":null,"admin-job-type":0}`, + "/tikv/cdc/capture/" + ctx.GlobalVars().CaptureInfo.ID: `{"id":"` + ctx.GlobalVars().CaptureInfo.ID + `","address":"127.0.0.1:8300"}`, + "/tikv/cdc/changefeed/info/" + ctx.ChangefeedVars().ID: `{"sink-uri":"blackhole://","opts":{},"create-time":"2020-02-02T00:00:00.000000+00:00","start-ts":0,"target-ts":0,"admin-job-type":0,"sort-engine":"memory","sort-dir":".","config":{"case-sensitive":true,"enable-old-value":false,"force-replicate":false,"check-gc-safe-point":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"open-protocol"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"keyspan-number","polling-time":-1}},"state":"normal","history":null,"error":null,"sync-point-enabled":false,"sync-point-interval":600000000000}`, + "/tikv/cdc/job/" + ctx.ChangefeedVars().ID: `{"resolved-ts":0,"checkpoint-ts":0,"admin-job-type":0}`, + "/tikv/cdc/task/status/" + ctx.GlobalVars().CaptureInfo.ID + "/" + ctx.ChangefeedVars().ID: `{"keyspans":{},"operation":null,"admin-job-type":0}`, }) } -type mockTablePipeline struct { - tableID model.TableID +type mockKeySpanPipeline struct { + keyspanID model.KeySpanID name string resolvedTs model.Ts checkpointTs model.Ts barrierTs model.Ts stopTs model.Ts - status tablepipeline.TableStatus + status keyspanpipeline.KeySpanStatus canceled bool } -func (m *mockTablePipeline) ID() (tableID int64, markTableID int64) { - return m.tableID, 0 +func (m *mockKeySpanPipeline) ID() (keyspanID uint64) { + return m.keyspanID } -func (m *mockTablePipeline) Name() string { +func (m *mockKeySpanPipeline) Name() string { return m.name } -func (m *mockTablePipeline) ResolvedTs() model.Ts { +func (m *mockKeySpanPipeline) ResolvedTs() model.Ts { return m.resolvedTs } -func (m *mockTablePipeline) CheckpointTs() model.Ts { +func (m *mockKeySpanPipeline) CheckpointTs() model.Ts { return m.checkpointTs } -func (m *mockTablePipeline) UpdateBarrierTs(ts model.Ts) { +func (m *mockKeySpanPipeline) UpdateBarrierTs(ts model.Ts) { m.barrierTs = ts } -func (m *mockTablePipeline) AsyncStop(targetTs model.Ts) bool { +func (m *mockKeySpanPipeline) AsyncStop(targetTs model.Ts) bool { m.stopTs = targetTs return true } -func (m *mockTablePipeline) Workload() model.WorkloadInfo { +func (m *mockKeySpanPipeline) Workload() model.WorkloadInfo { return model.WorkloadInfo{Workload: 1} } -func (m *mockTablePipeline) Status() tablepipeline.TableStatus { +func (m *mockKeySpanPipeline) Status() keyspanpipeline.KeySpanStatus { return m.status } -func (m *mockTablePipeline) Cancel() { +func (m *mockKeySpanPipeline) Cancel() { if m.canceled { - log.Panic("cancel a canceled table pipeline") + log.Panic("cancel a canceled keyspan pipeline") } m.canceled = true } -func (m *mockTablePipeline) Wait() { +func (m *mockKeySpanPipeline) Wait() { // do nothing } -type mockSchemaStorage struct { - // dummy to provide default versions of unimplemented interface methods, - // as we only need ResolvedTs() and DoGC() in unit tests. - entry.SchemaStorage - - c *check.C - lastGcTs uint64 - resolvedTs uint64 -} - -func (s *mockSchemaStorage) ResolvedTs() uint64 { - return s.resolvedTs -} - -func (s *mockSchemaStorage) DoGC(ts uint64) uint64 { - s.c.Assert(s.lastGcTs, check.LessEqual, ts) - atomic.StoreUint64(&s.lastGcTs, ts) - return ts -} - type mockAgent struct { // dummy to satisfy the interface processorAgent - executor scheduler.TableExecutor + executor scheduler.KeySpanExecutor lastCheckpointTs model.Ts isClosed bool } func (a *mockAgent) Tick(_ cdcContext.Context) error { - if len(a.executor.GetAllCurrentTables()) == 0 { + if len(a.executor.GetAllCurrentKeySpans()) == 0 { return nil } a.lastCheckpointTs, _ = a.executor.GetCheckpoint() @@ -183,7 +158,7 @@ func (a *mockAgent) Close() error { return nil } -func (s *processorSuite) TestCheckTablesNum(c *check.C) { +func (s *processorSuite) TestCheckKeySpansNum(c *check.C) { defer testleak.AfterTest(c)() ctx := cdcContext.NewBackendContext4Test(true) p, tester := initProcessor4Test(ctx, c) @@ -214,7 +189,7 @@ func (s *processorSuite) TestCheckTablesNum(c *check.C) { }) } -func (s *processorSuite) TestHandleTableOperation4SingleTable(c *check.C) { +func (s *processorSuite) TestHandleKeySpanOperation4SingleKeySpan(c *check.C) { defer testleak.AfterTest(c)() ctx := cdcContext.NewBackendContext4Test(true) p, tester := initProcessor4Test(ctx, c) @@ -239,10 +214,10 @@ func (s *processorSuite) TestHandleTableOperation4SingleTable(c *check.C) { c.Assert(err, check.IsNil) tester.MustApplyPatches() - // add table, in processing - // in current implementation of owner, the startTs and BoundaryTs of add table operation should be always equaled. + // add keyspan, in processing + // in current implementation of owner, the startTs and BoundaryTs of add keyspan operation should be always equaled. p.changefeed.PatchTaskStatus(p.captureInfo.ID, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { - status.AddTable(66, &model.TableReplicaInfo{StartTs: 60}, 60) + status.AddKeySpan(66, &model.KeySpanReplicaInfo{StartTs: 60}, 60) return status, true, nil }) tester.MustApplyPatches() @@ -250,38 +225,38 @@ func (s *processorSuite) TestHandleTableOperation4SingleTable(c *check.C) { c.Assert(err, check.IsNil) tester.MustApplyPatches() c.Assert(p.changefeed.TaskStatuses[p.captureInfo.ID], check.DeepEquals, &model.TaskStatus{ - Tables: map[int64]*model.TableReplicaInfo{ + KeySpans: map[uint64]*model.KeySpanReplicaInfo{ 66: {StartTs: 60}, }, - Operation: map[int64]*model.TableOperation{ + Operation: map[uint64]*model.KeySpanOperation{ 66: {Delete: false, BoundaryTs: 60, Status: model.OperProcessed}, }, }) - // add table, not finished + // add keyspan, not finished _, err = p.Tick(ctx, p.changefeed) c.Assert(err, check.IsNil) tester.MustApplyPatches() c.Assert(p.changefeed.TaskStatuses[p.captureInfo.ID], check.DeepEquals, &model.TaskStatus{ - Tables: map[int64]*model.TableReplicaInfo{ + KeySpans: map[uint64]*model.KeySpanReplicaInfo{ 66: {StartTs: 60}, }, - Operation: map[int64]*model.TableOperation{ + Operation: map[uint64]*model.KeySpanOperation{ 66: {Delete: false, BoundaryTs: 60, Status: model.OperProcessed}, }, }) - // add table, push the resolvedTs - table66 := p.tables[66].(*mockTablePipeline) - table66.resolvedTs = 101 + // add keyspan, push the resolvedTs + keyspan66 := p.keyspans[66].(*mockKeySpanPipeline) + keyspan66.resolvedTs = 101 _, err = p.Tick(ctx, p.changefeed) c.Assert(err, check.IsNil) tester.MustApplyPatches() c.Assert(p.changefeed.TaskStatuses[p.captureInfo.ID], check.DeepEquals, &model.TaskStatus{ - Tables: map[int64]*model.TableReplicaInfo{ + KeySpans: map[uint64]*model.KeySpanReplicaInfo{ 66: {StartTs: 60}, }, - Operation: map[int64]*model.TableOperation{ + Operation: map[uint64]*model.KeySpanOperation{ 66: {Delete: false, BoundaryTs: 60, Status: model.OperProcessed}, }, }) @@ -292,10 +267,10 @@ func (s *processorSuite) TestHandleTableOperation4SingleTable(c *check.C) { c.Assert(err, check.IsNil) tester.MustApplyPatches() c.Assert(p.changefeed.TaskStatuses[p.captureInfo.ID], check.DeepEquals, &model.TaskStatus{ - Tables: map[int64]*model.TableReplicaInfo{ + KeySpans: map[uint64]*model.KeySpanReplicaInfo{ 66: {StartTs: 60}, }, - Operation: map[int64]*model.TableOperation{ + Operation: map[uint64]*model.KeySpanOperation{ 66: {Delete: false, BoundaryTs: 60, Status: model.OperFinished}, }, }) @@ -303,9 +278,9 @@ func (s *processorSuite) TestHandleTableOperation4SingleTable(c *check.C) { // clear finished operations cleanUpFinishedOpOperation(p.changefeed, p.captureInfo.ID, tester) - // remove table, in processing + // remove keyspan, in processing p.changefeed.PatchTaskStatus(p.captureInfo.ID, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { - status.RemoveTable(66, 120, false) + status.RemoveKeySpan(66, 120, false) return status, true, nil }) tester.MustApplyPatches() @@ -313,41 +288,41 @@ func (s *processorSuite) TestHandleTableOperation4SingleTable(c *check.C) { c.Assert(err, check.IsNil) tester.MustApplyPatches() c.Assert(p.changefeed.TaskStatuses[p.captureInfo.ID], check.DeepEquals, &model.TaskStatus{ - Tables: map[int64]*model.TableReplicaInfo{}, - Operation: map[int64]*model.TableOperation{ + KeySpans: map[uint64]*model.KeySpanReplicaInfo{}, + Operation: map[uint64]*model.KeySpanOperation{ 66: {Delete: true, BoundaryTs: 120, Status: model.OperProcessed}, }, }) - c.Assert(table66.stopTs, check.Equals, uint64(120)) + c.Assert(keyspan66.stopTs, check.Equals, uint64(120)) - // remove table, not finished + // remove keyspan, not finished _, err = p.Tick(ctx, p.changefeed) c.Assert(err, check.IsNil) tester.MustApplyPatches() c.Assert(p.changefeed.TaskStatuses[p.captureInfo.ID], check.DeepEquals, &model.TaskStatus{ - Tables: map[int64]*model.TableReplicaInfo{}, - Operation: map[int64]*model.TableOperation{ + KeySpans: map[uint64]*model.KeySpanReplicaInfo{}, + Operation: map[uint64]*model.KeySpanOperation{ 66: {Delete: true, BoundaryTs: 120, Status: model.OperProcessed}, }, }) - // remove table, finished - table66.status = tablepipeline.TableStatusStopped - table66.checkpointTs = 121 + // remove keyspan, finished + keyspan66.status = keyspanpipeline.KeySpanStatusStopped + keyspan66.checkpointTs = 121 _, err = p.Tick(ctx, p.changefeed) c.Assert(err, check.IsNil) tester.MustApplyPatches() c.Assert(p.changefeed.TaskStatuses[p.captureInfo.ID], check.DeepEquals, &model.TaskStatus{ - Tables: map[int64]*model.TableReplicaInfo{}, - Operation: map[int64]*model.TableOperation{ + KeySpans: map[uint64]*model.KeySpanReplicaInfo{}, + Operation: map[uint64]*model.KeySpanOperation{ 66: {Delete: true, BoundaryTs: 121, Status: model.OperFinished}, }, }) - c.Assert(table66.canceled, check.IsTrue) - c.Assert(p.tables[66], check.IsNil) + c.Assert(keyspan66.canceled, check.IsTrue) + c.Assert(p.keyspans[66], check.IsNil) } -func (s *processorSuite) TestHandleTableOperation4MultiTable(c *check.C) { +func (s *processorSuite) TestHandleKeySpanOperation4MultiKeySpan(c *check.C) { defer testleak.AfterTest(c)() ctx := cdcContext.NewBackendContext4Test(true) p, tester := initProcessor4Test(ctx, c) @@ -373,13 +348,13 @@ func (s *processorSuite) TestHandleTableOperation4MultiTable(c *check.C) { c.Assert(err, check.IsNil) tester.MustApplyPatches() - // add table, in processing - // in current implementation of owner, the startTs and BoundaryTs of add table operation should be always equaled. + // add keyspan, in processing + // in current implementation of owner, the startTs and BoundaryTs of add keyspan operation should be always equaled. p.changefeed.PatchTaskStatus(p.captureInfo.ID, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { - status.AddTable(1, &model.TableReplicaInfo{StartTs: 60}, 60) - status.AddTable(2, &model.TableReplicaInfo{StartTs: 50}, 50) - status.AddTable(3, &model.TableReplicaInfo{StartTs: 40}, 40) - status.Tables[4] = &model.TableReplicaInfo{StartTs: 30} + status.AddKeySpan(1, &model.KeySpanReplicaInfo{StartTs: 60}, 60) + status.AddKeySpan(2, &model.KeySpanReplicaInfo{StartTs: 50}, 50) + status.AddKeySpan(3, &model.KeySpanReplicaInfo{StartTs: 40}, 40) + status.KeySpans[4] = &model.KeySpanReplicaInfo{StartTs: 30} return status, true, nil }) tester.MustApplyPatches() @@ -387,34 +362,34 @@ func (s *processorSuite) TestHandleTableOperation4MultiTable(c *check.C) { c.Assert(err, check.IsNil) tester.MustApplyPatches() c.Assert(p.changefeed.TaskStatuses[p.captureInfo.ID], check.DeepEquals, &model.TaskStatus{ - Tables: map[int64]*model.TableReplicaInfo{ + KeySpans: map[uint64]*model.KeySpanReplicaInfo{ 1: {StartTs: 60}, 2: {StartTs: 50}, 3: {StartTs: 40}, 4: {StartTs: 30}, }, - Operation: map[int64]*model.TableOperation{ + Operation: map[uint64]*model.KeySpanOperation{ 1: {Delete: false, BoundaryTs: 60, Status: model.OperProcessed}, 2: {Delete: false, BoundaryTs: 50, Status: model.OperProcessed}, 3: {Delete: false, BoundaryTs: 40, Status: model.OperProcessed}, }, }) - c.Assert(p.tables, check.HasLen, 4) + c.Assert(p.keyspans, check.HasLen, 4) c.Assert(p.changefeed.TaskPositions[p.captureInfo.ID].CheckPointTs, check.Equals, uint64(30)) c.Assert(p.changefeed.TaskPositions[p.captureInfo.ID].ResolvedTs, check.Equals, uint64(30)) - // add table, push the resolvedTs, finished add table - table1 := p.tables[1].(*mockTablePipeline) - table2 := p.tables[2].(*mockTablePipeline) - table3 := p.tables[3].(*mockTablePipeline) - table4 := p.tables[4].(*mockTablePipeline) - table1.resolvedTs = 101 - table2.resolvedTs = 101 - table3.resolvedTs = 102 - table4.resolvedTs = 103 - // removed table 3 + // add keyspan, push the resolvedTs, finished add keyspan + keyspan1 := p.keyspans[1].(*mockKeySpanPipeline) + keyspan2 := p.keyspans[2].(*mockKeySpanPipeline) + keyspan3 := p.keyspans[3].(*mockKeySpanPipeline) + keyspan4 := p.keyspans[4].(*mockKeySpanPipeline) + keyspan1.resolvedTs = 101 + keyspan2.resolvedTs = 101 + keyspan3.resolvedTs = 102 + keyspan4.resolvedTs = 103 + // removed keyspan 3 p.changefeed.PatchTaskStatus(p.captureInfo.ID, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { - status.RemoveTable(3, 60, false) + status.RemoveKeySpan(3, 60, false) return status, true, nil }) tester.MustApplyPatches() @@ -422,51 +397,51 @@ func (s *processorSuite) TestHandleTableOperation4MultiTable(c *check.C) { c.Assert(err, check.IsNil) tester.MustApplyPatches() c.Assert(p.changefeed.TaskStatuses[p.captureInfo.ID], check.DeepEquals, &model.TaskStatus{ - Tables: map[int64]*model.TableReplicaInfo{ + KeySpans: map[uint64]*model.KeySpanReplicaInfo{ 1: {StartTs: 60}, 2: {StartTs: 50}, 4: {StartTs: 30}, }, - Operation: map[int64]*model.TableOperation{ + Operation: map[uint64]*model.KeySpanOperation{ 1: {Delete: false, BoundaryTs: 60, Status: model.OperFinished}, 2: {Delete: false, BoundaryTs: 50, Status: model.OperFinished}, 3: {Delete: true, BoundaryTs: 60, Status: model.OperProcessed}, }, }) - c.Assert(p.tables, check.HasLen, 4) - c.Assert(table3.canceled, check.IsFalse) - c.Assert(table3.stopTs, check.Equals, uint64(60)) + c.Assert(p.keyspans, check.HasLen, 4) + c.Assert(keyspan3.canceled, check.IsFalse) + c.Assert(keyspan3.stopTs, check.Equals, uint64(60)) c.Assert(p.changefeed.TaskPositions[p.captureInfo.ID].ResolvedTs, check.Equals, uint64(101)) // finish remove operations - table3.status = tablepipeline.TableStatusStopped - table3.checkpointTs = 65 + keyspan3.status = keyspanpipeline.KeySpanStatusStopped + keyspan3.checkpointTs = 65 _, err = p.Tick(ctx, p.changefeed) c.Assert(err, check.IsNil) tester.MustApplyPatches() c.Assert(p.changefeed.TaskStatuses[p.captureInfo.ID], check.DeepEquals, &model.TaskStatus{ - Tables: map[int64]*model.TableReplicaInfo{ + KeySpans: map[uint64]*model.KeySpanReplicaInfo{ 1: {StartTs: 60}, 2: {StartTs: 50}, 4: {StartTs: 30}, }, - Operation: map[int64]*model.TableOperation{ + Operation: map[uint64]*model.KeySpanOperation{ 1: {Delete: false, BoundaryTs: 60, Status: model.OperFinished}, 2: {Delete: false, BoundaryTs: 50, Status: model.OperFinished}, 3: {Delete: true, BoundaryTs: 65, Status: model.OperFinished}, }, }) - c.Assert(p.tables, check.HasLen, 3) - c.Assert(table3.canceled, check.IsTrue) + c.Assert(p.keyspans, check.HasLen, 3) + c.Assert(keyspan3.canceled, check.IsTrue) // clear finished operations cleanUpFinishedOpOperation(p.changefeed, p.captureInfo.ID, tester) - // remove table, in processing + // remove keyspan, in processing p.changefeed.PatchTaskStatus(p.captureInfo.ID, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { - status.RemoveTable(1, 120, false) - status.RemoveTable(4, 120, false) - delete(status.Tables, 2) + status.RemoveKeySpan(1, 120, false) + status.RemoveKeySpan(4, 120, false) + delete(status.KeySpans, 2) return status, true, nil }) tester.MustApplyPatches() @@ -474,50 +449,50 @@ func (s *processorSuite) TestHandleTableOperation4MultiTable(c *check.C) { c.Assert(err, check.IsNil) tester.MustApplyPatches() c.Assert(p.changefeed.TaskStatuses[p.captureInfo.ID], check.DeepEquals, &model.TaskStatus{ - Tables: map[int64]*model.TableReplicaInfo{}, - Operation: map[int64]*model.TableOperation{ + KeySpans: map[uint64]*model.KeySpanReplicaInfo{}, + Operation: map[uint64]*model.KeySpanOperation{ 1: {Delete: true, BoundaryTs: 120, Status: model.OperProcessed}, 4: {Delete: true, BoundaryTs: 120, Status: model.OperProcessed}, }, }) - c.Assert(table1.stopTs, check.Equals, uint64(120)) - c.Assert(table4.stopTs, check.Equals, uint64(120)) - c.Assert(table2.canceled, check.IsTrue) - c.Assert(p.tables, check.HasLen, 2) + c.Assert(keyspan1.stopTs, check.Equals, uint64(120)) + c.Assert(keyspan4.stopTs, check.Equals, uint64(120)) + c.Assert(keyspan2.canceled, check.IsTrue) + c.Assert(p.keyspans, check.HasLen, 2) - // remove table, not finished + // remove keyspan, not finished _, err = p.Tick(ctx, p.changefeed) c.Assert(err, check.IsNil) tester.MustApplyPatches() c.Assert(p.changefeed.TaskStatuses[p.captureInfo.ID], check.DeepEquals, &model.TaskStatus{ - Tables: map[int64]*model.TableReplicaInfo{}, - Operation: map[int64]*model.TableOperation{ + KeySpans: map[uint64]*model.KeySpanReplicaInfo{}, + Operation: map[uint64]*model.KeySpanOperation{ 1: {Delete: true, BoundaryTs: 120, Status: model.OperProcessed}, 4: {Delete: true, BoundaryTs: 120, Status: model.OperProcessed}, }, }) - // remove table, finished - table1.status = tablepipeline.TableStatusStopped - table1.checkpointTs = 121 - table4.status = tablepipeline.TableStatusStopped - table4.checkpointTs = 122 + // remove keyspan, finished + keyspan1.status = keyspanpipeline.KeySpanStatusStopped + keyspan1.checkpointTs = 121 + keyspan4.status = keyspanpipeline.KeySpanStatusStopped + keyspan4.checkpointTs = 122 _, err = p.Tick(ctx, p.changefeed) c.Assert(err, check.IsNil) tester.MustApplyPatches() c.Assert(p.changefeed.TaskStatuses[p.captureInfo.ID], check.DeepEquals, &model.TaskStatus{ - Tables: map[int64]*model.TableReplicaInfo{}, - Operation: map[int64]*model.TableOperation{ + KeySpans: map[uint64]*model.KeySpanReplicaInfo{}, + Operation: map[uint64]*model.KeySpanOperation{ 1: {Delete: true, BoundaryTs: 121, Status: model.OperFinished}, 4: {Delete: true, BoundaryTs: 122, Status: model.OperFinished}, }, }) - c.Assert(table1.canceled, check.IsTrue) - c.Assert(table4.canceled, check.IsTrue) - c.Assert(p.tables, check.HasLen, 0) + c.Assert(keyspan1.canceled, check.IsTrue) + c.Assert(keyspan4.canceled, check.IsTrue) + c.Assert(p.keyspans, check.HasLen, 0) } -func (s *processorSuite) TestTableExecutor(c *check.C) { +func (s *processorSuite) TestKeySpanExecutor(c *check.C) { defer testleak.AfterTest(c)() ctx := cdcContext.NewBackendContext4Test(true) p, tester := initProcessor4Test(ctx, c) @@ -549,71 +524,71 @@ func (s *processorSuite) TestTableExecutor(c *check.C) { c.Assert(err, check.IsNil) tester.MustApplyPatches() - ok, err := p.AddTable(ctx, 1) + ok, err := p.AddKeySpan(ctx, 1, []byte{1}, []byte{2}) c.Check(err, check.IsNil) c.Check(ok, check.IsTrue) - ok, err = p.AddTable(ctx, 2) + ok, err = p.AddKeySpan(ctx, 2, []byte{2}, []byte{3}) c.Check(err, check.IsNil) c.Check(ok, check.IsTrue) - ok, err = p.AddTable(ctx, 3) + ok, err = p.AddKeySpan(ctx, 3, []byte{3}, []byte{4}) c.Check(err, check.IsNil) c.Check(ok, check.IsTrue) - ok, err = p.AddTable(ctx, 4) + ok, err = p.AddKeySpan(ctx, 4, []byte{5}, []byte{6}) c.Check(err, check.IsNil) c.Check(ok, check.IsTrue) - c.Assert(p.tables, check.HasLen, 4) + c.Assert(p.keyspans, check.HasLen, 4) checkpointTs := p.agent.GetLastSentCheckpointTs() c.Assert(checkpointTs, check.Equals, uint64(0)) - done := p.IsAddTableFinished(ctx, 1) + done := p.IsAddKeySpanFinished(ctx, 1) c.Check(done, check.IsFalse) - done = p.IsAddTableFinished(ctx, 2) + done = p.IsAddKeySpanFinished(ctx, 2) c.Check(done, check.IsFalse) - done = p.IsAddTableFinished(ctx, 3) + done = p.IsAddKeySpanFinished(ctx, 3) c.Check(done, check.IsFalse) - done = p.IsAddTableFinished(ctx, 4) + done = p.IsAddKeySpanFinished(ctx, 4) c.Check(done, check.IsFalse) - c.Assert(p.tables, check.HasLen, 4) + c.Assert(p.keyspans, check.HasLen, 4) _, err = p.Tick(ctx, p.changefeed) c.Assert(err, check.IsNil) tester.MustApplyPatches() - // add table, push the resolvedTs, finished add table - table1 := p.tables[1].(*mockTablePipeline) - table2 := p.tables[2].(*mockTablePipeline) - table3 := p.tables[3].(*mockTablePipeline) - table4 := p.tables[4].(*mockTablePipeline) - table1.resolvedTs = 101 - table2.resolvedTs = 101 - table3.resolvedTs = 102 - table4.resolvedTs = 103 + // add keyspan, push the resolvedTs, finished add keyspan + keyspan1 := p.keyspans[1].(*mockKeySpanPipeline) + keyspan2 := p.keyspans[2].(*mockKeySpanPipeline) + keyspan3 := p.keyspans[3].(*mockKeySpanPipeline) + keyspan4 := p.keyspans[4].(*mockKeySpanPipeline) + keyspan1.resolvedTs = 101 + keyspan2.resolvedTs = 101 + keyspan3.resolvedTs = 102 + keyspan4.resolvedTs = 103 - table1.checkpointTs = 30 - table2.checkpointTs = 30 - table3.checkpointTs = 30 - table4.checkpointTs = 30 + keyspan1.checkpointTs = 30 + keyspan2.checkpointTs = 30 + keyspan3.checkpointTs = 30 + keyspan4.checkpointTs = 30 - done = p.IsAddTableFinished(ctx, 1) + done = p.IsAddKeySpanFinished(ctx, 1) c.Check(done, check.IsTrue) - done = p.IsAddTableFinished(ctx, 2) + done = p.IsAddKeySpanFinished(ctx, 2) c.Check(done, check.IsTrue) - done = p.IsAddTableFinished(ctx, 3) + done = p.IsAddKeySpanFinished(ctx, 3) c.Check(done, check.IsTrue) - done = p.IsAddTableFinished(ctx, 4) + done = p.IsAddKeySpanFinished(ctx, 4) c.Check(done, check.IsTrue) _, err = p.Tick(ctx, p.changefeed) c.Assert(err, check.IsNil) tester.MustApplyPatches() - table1.checkpointTs = 75 - table2.checkpointTs = 75 - table3.checkpointTs = 60 - table4.checkpointTs = 75 + keyspan1.checkpointTs = 75 + keyspan2.checkpointTs = 75 + keyspan3.checkpointTs = 60 + keyspan4.checkpointTs = 75 _, err = p.Tick(ctx, p.changefeed) c.Assert(err, check.IsNil) @@ -628,7 +603,7 @@ func (s *processorSuite) TestTableExecutor(c *check.C) { c.Assert(err, check.IsNil) tester.MustApplyPatches() - ok, err = p.RemoveTable(ctx, 3) + ok, err = p.RemoveKeySpan(ctx, 3) c.Check(err, check.IsNil) c.Check(ok, check.IsTrue) @@ -636,11 +611,11 @@ func (s *processorSuite) TestTableExecutor(c *check.C) { c.Assert(err, check.IsNil) tester.MustApplyPatches() - c.Assert(p.tables, check.HasLen, 4) - c.Assert(table3.canceled, check.IsFalse) - c.Assert(table3.stopTs, check.Equals, uint64(60)) + c.Assert(p.keyspans, check.HasLen, 4) + c.Assert(keyspan3.canceled, check.IsFalse) + c.Assert(keyspan3.stopTs, check.Equals, uint64(60)) - done = p.IsRemoveTableFinished(ctx, 3) + done = p.IsRemoveKeySpanFinished(ctx, 3) c.Assert(done, check.IsFalse) _, err = p.Tick(ctx, p.changefeed) @@ -651,21 +626,21 @@ func (s *processorSuite) TestTableExecutor(c *check.C) { c.Assert(checkpointTs, check.Equals, uint64(60)) // finish remove operations - table3.status = tablepipeline.TableStatusStopped - table3.checkpointTs = 65 + keyspan3.status = keyspanpipeline.KeySpanStatusStopped + keyspan3.checkpointTs = 65 _, err = p.Tick(ctx, p.changefeed) c.Assert(err, check.IsNil) tester.MustApplyPatches() - c.Assert(p.tables, check.HasLen, 4) - c.Assert(table3.canceled, check.IsFalse) + c.Assert(p.keyspans, check.HasLen, 4) + c.Assert(keyspan3.canceled, check.IsFalse) - done = p.IsRemoveTableFinished(ctx, 3) + done = p.IsRemoveKeySpanFinished(ctx, 3) c.Assert(done, check.IsTrue) - c.Assert(p.tables, check.HasLen, 3) - c.Assert(table3.canceled, check.IsTrue) + c.Assert(p.keyspans, check.HasLen, 3) + c.Assert(keyspan3.canceled, check.IsTrue) _, err = p.Tick(ctx, p.changefeed) c.Assert(err, check.IsNil) @@ -679,7 +654,7 @@ func (s *processorSuite) TestTableExecutor(c *check.C) { c.Assert(p.agent, check.IsNil) } -func (s *processorSuite) TestInitTable(c *check.C) { +func (s *processorSuite) TestInitKeySpan(c *check.C) { defer testleak.AfterTest(c)() ctx := cdcContext.NewBackendContext4Test(true) p, tester := initProcessor4Test(ctx, c) @@ -690,16 +665,16 @@ func (s *processorSuite) TestInitTable(c *check.C) { tester.MustApplyPatches() p.changefeed.PatchTaskStatus(p.captureInfo.ID, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { - status.Tables[1] = &model.TableReplicaInfo{StartTs: 20} - status.Tables[2] = &model.TableReplicaInfo{StartTs: 30} + status.KeySpans[1] = &model.KeySpanReplicaInfo{StartTs: 20} + status.KeySpans[2] = &model.KeySpanReplicaInfo{StartTs: 30} return status, true, nil }) tester.MustApplyPatches() _, err = p.Tick(ctx, p.changefeed) c.Assert(err, check.IsNil) tester.MustApplyPatches() - c.Assert(p.tables[1], check.Not(check.IsNil)) - c.Assert(p.tables[2], check.Not(check.IsNil)) + c.Assert(p.keyspans[1], check.Not(check.IsNil)) + c.Assert(p.keyspans[2], check.Not(check.IsNil)) } func (s *processorSuite) TestProcessorError(c *check.C) { @@ -779,10 +754,10 @@ func (s *processorSuite) TestProcessorClose(c *check.C) { c.Assert(err, check.IsNil) tester.MustApplyPatches() - // add tables + // add keyspans p.changefeed.PatchTaskStatus(p.captureInfo.ID, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { - status.Tables[1] = &model.TableReplicaInfo{StartTs: 20} - status.Tables[2] = &model.TableReplicaInfo{StartTs: 30} + status.KeySpans[1] = &model.KeySpanReplicaInfo{StartTs: 20} + status.KeySpans[2] = &model.KeySpanReplicaInfo{StartTs: 30} return status, true, nil }) tester.MustApplyPatches() @@ -796,10 +771,10 @@ func (s *processorSuite) TestProcessorClose(c *check.C) { return status, true, nil }) tester.MustApplyPatches() - p.tables[1].(*mockTablePipeline).resolvedTs = 110 - p.tables[2].(*mockTablePipeline).resolvedTs = 90 - p.tables[1].(*mockTablePipeline).checkpointTs = 90 - p.tables[2].(*mockTablePipeline).checkpointTs = 95 + p.keyspans[1].(*mockKeySpanPipeline).resolvedTs = 110 + p.keyspans[2].(*mockKeySpanPipeline).resolvedTs = 90 + p.keyspans[1].(*mockKeySpanPipeline).checkpointTs = 90 + p.keyspans[2].(*mockKeySpanPipeline).checkpointTs = 95 _, err = p.Tick(ctx, p.changefeed) c.Assert(err, check.IsNil) tester.MustApplyPatches() @@ -809,14 +784,14 @@ func (s *processorSuite) TestProcessorClose(c *check.C) { Error: nil, }) c.Assert(p.changefeed.TaskStatuses[p.captureInfo.ID], check.DeepEquals, &model.TaskStatus{ - Tables: map[int64]*model.TableReplicaInfo{1: {StartTs: 20}, 2: {StartTs: 30}}, + KeySpans: map[uint64]*model.KeySpanReplicaInfo{1: {StartTs: 20}, 2: {StartTs: 30}}, }) c.Assert(p.changefeed.Workloads[p.captureInfo.ID], check.DeepEquals, model.TaskWorkload{1: {Workload: 1}, 2: {Workload: 1}}) c.Assert(p.Close(), check.IsNil) tester.MustApplyPatches() - c.Assert(p.tables[1].(*mockTablePipeline).canceled, check.IsTrue) - c.Assert(p.tables[2].(*mockTablePipeline).canceled, check.IsTrue) + c.Assert(p.keyspans[1].(*mockKeySpanPipeline).canceled, check.IsTrue) + c.Assert(p.keyspans[2].(*mockKeySpanPipeline).canceled, check.IsTrue) p, tester = initProcessor4Test(ctx, c) // init tick @@ -824,10 +799,10 @@ func (s *processorSuite) TestProcessorClose(c *check.C) { c.Assert(err, check.IsNil) tester.MustApplyPatches() - // add tables + // add keyspans p.changefeed.PatchTaskStatus(p.captureInfo.ID, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { - status.Tables[1] = &model.TableReplicaInfo{StartTs: 20} - status.Tables[2] = &model.TableReplicaInfo{StartTs: 30} + status.KeySpans[1] = &model.KeySpanReplicaInfo{StartTs: 20} + status.KeySpans[2] = &model.KeySpanReplicaInfo{StartTs: 30} return status, true, nil }) tester.MustApplyPatches() @@ -848,8 +823,8 @@ func (s *processorSuite) TestProcessorClose(c *check.C) { Code: "CDC:ErrSinkURIInvalid", Message: "[CDC:ErrSinkURIInvalid]sink uri invalid", }) - c.Assert(p.tables[1].(*mockTablePipeline).canceled, check.IsTrue) - c.Assert(p.tables[2].(*mockTablePipeline).canceled, check.IsTrue) + c.Assert(p.keyspans[1].(*mockKeySpanPipeline).canceled, check.IsTrue) + c.Assert(p.keyspans[2].(*mockKeySpanPipeline).canceled, check.IsTrue) } func (s *processorSuite) TestPositionDeleted(c *check.C) { @@ -857,8 +832,8 @@ func (s *processorSuite) TestPositionDeleted(c *check.C) { ctx := cdcContext.NewBackendContext4Test(true) p, tester := initProcessor4Test(ctx, c) p.changefeed.PatchTaskStatus(p.captureInfo.ID, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { - status.Tables[1] = &model.TableReplicaInfo{StartTs: 30} - status.Tables[2] = &model.TableReplicaInfo{StartTs: 40} + status.KeySpans[1] = &model.KeySpanReplicaInfo{StartTs: 30} + status.KeySpans[2] = &model.KeySpanReplicaInfo{StartTs: 40} return status, true, nil }) var err error @@ -900,40 +875,14 @@ func (s *processorSuite) TestPositionDeleted(c *check.C) { }) } -func (s *processorSuite) TestSchemaGC(c *check.C) { - defer testleak.AfterTest(c)() - ctx := cdcContext.NewBackendContext4Test(true) - p, tester := initProcessor4Test(ctx, c) - p.changefeed.PatchTaskStatus(p.captureInfo.ID, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { - status.Tables[1] = &model.TableReplicaInfo{StartTs: 30} - status.Tables[2] = &model.TableReplicaInfo{StartTs: 40} - return status, true, nil - }) - - var err error - // init tick - _, err = p.Tick(ctx, p.changefeed) - c.Assert(err, check.IsNil) - tester.MustApplyPatches() - - updateChangeFeedPosition(c, tester, "changefeed-id-test", 50, 50) - _, err = p.Tick(ctx, p.changefeed) - c.Assert(err, check.IsNil) - tester.MustApplyPatches() - - // GC Ts should be (checkpoint - 1). - c.Assert(p.schemaStorage.(*mockSchemaStorage).lastGcTs, check.Equals, uint64(49)) - c.Assert(p.lastSchemaTs, check.Equals, uint64(49)) -} - func cleanUpFinishedOpOperation(state *orchestrator.ChangefeedReactorState, captureID model.CaptureID, tester *orchestrator.ReactorStateTester) { state.PatchTaskStatus(captureID, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { if status == nil || status.Operation == nil { return status, false, nil } - for tableID, opt := range status.Operation { + for keyspanID, opt := range status.Operation { if opt.Status == model.OperFinished { - delete(status.Operation, tableID) + delete(status.Operation, keyspanID) } } return status, true, nil @@ -970,54 +919,10 @@ func (s *processorSuite) TestIgnorableError(c *check.C) { {cerror.ErrReactorFinished.GenWithStackByArgs(), true}, {cerror.ErrRedoWriterStopped.GenWithStackByArgs(), true}, {errors.Trace(context.Canceled), true}, - {cerror.ErrProcessorTableNotFound.GenWithStackByArgs(), false}, + {cerror.ErrProcessorKeySpanNotFound.GenWithStackByArgs(), false}, {errors.New("test error"), false}, } for _, tc := range testCases { c.Assert(isProcessorIgnorableError(tc.err), check.Equals, tc.ignorable) } } - -func (s *processorSuite) TestUpdateBarrierTs(c *check.C) { - defer testleak.AfterTest(c)() - ctx := cdcContext.NewBackendContext4Test(true) - p, tester := initProcessor4Test(ctx, c) - p.changefeed.PatchStatus(func(status *model.ChangeFeedStatus) (*model.ChangeFeedStatus, bool, error) { - status.CheckpointTs = 5 - status.ResolvedTs = 10 - return status, true, nil - }) - p.changefeed.PatchTaskStatus(p.captureInfo.ID, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { - status.AddTable(1, &model.TableReplicaInfo{StartTs: 5}, 5) - return status, true, nil - }) - p.schemaStorage.(*mockSchemaStorage).resolvedTs = 10 - - // init tick, add table OperDispatched. - _, err := p.Tick(ctx, p.changefeed) - c.Assert(err, check.IsNil) - tester.MustApplyPatches() - // tick again, add table OperProcessed. - _, err = p.Tick(ctx, p.changefeed) - c.Assert(err, check.IsNil) - tester.MustApplyPatches() - - // Global resolved ts has advanced while schema storage stalls. - p.changefeed.PatchStatus(func(status *model.ChangeFeedStatus) (*model.ChangeFeedStatus, bool, error) { - status.ResolvedTs = 20 - return status, true, nil - }) - _, err = p.Tick(ctx, p.changefeed) - c.Assert(err, check.IsNil) - tester.MustApplyPatches() - tb := p.tables[model.TableID(1)].(*mockTablePipeline) - c.Assert(tb.barrierTs, check.Equals, uint64(10)) - - // Schema storage has advanced too. - p.schemaStorage.(*mockSchemaStorage).resolvedTs = 15 - _, err = p.Tick(ctx, p.changefeed) - c.Assert(err, check.IsNil) - tester.MustApplyPatches() - tb = p.tables[model.TableID(1)].(*mockTablePipeline) - c.Assert(tb.barrierTs, check.Equals, uint64(15)) -} diff --git a/cdc/cdc/puller/metrics.go b/cdc/cdc/puller/metrics.go index 50c0a339..99a168a1 100644 --- a/cdc/cdc/puller/metrics.go +++ b/cdc/cdc/puller/metrics.go @@ -20,28 +20,28 @@ import ( var ( kvEventCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "puller", Name: "kv_event_count", Help: "The number of events received from kv client event channel", }, []string{"capture", "changefeed", "type"}) txnCollectCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "puller", Name: "txn_collect_event_count", Help: "The number of events received from txn collector", }, []string{"capture", "changefeed", "type"}) pullerResolvedTsGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "puller", Name: "resolved_ts", Help: "puller forward resolved ts", }, []string{"capture", "changefeed"}) outputChanSizeHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "puller", Name: "output_chan_size", Help: "Puller entry buffer size", @@ -49,14 +49,14 @@ var ( }, []string{"capture", "changefeed"}) memBufferSizeGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "puller", Name: "mem_buffer_size", Help: "Puller in memory buffer size", }, []string{"capture", "changefeed"}) eventChanSizeHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "puller", Name: "event_chan_size", Help: "Puller event channel size", diff --git a/cdc/cdc/puller/puller.go b/cdc/cdc/puller/puller.go index 8f637502..1cc392ce 100644 --- a/cdc/cdc/puller/puller.go +++ b/cdc/cdc/puller/puller.go @@ -34,9 +34,6 @@ import ( "golang.org/x/sync/errgroup" ) -// DDLPullerTableName is the fake table name for ddl puller. -const DDLPullerTableName = "DDL_PULLER" - const ( defaultPullerEventChanSize = 128 defaultPullerOutputChanSize = 128 @@ -120,7 +117,7 @@ func (p *pullerImpl) Run(ctx context.Context) error { captureAddr := util.CaptureAddrFromCtx(ctx) changefeedID := util.ChangefeedIDFromCtx(ctx) - tableID, _ := util.TableIDFromCtx(ctx) + keyspanID, _ := util.KeySpanIDFromCtx(ctx) metricOutputChanSize := outputChanSizeHistogram.WithLabelValues(captureAddr, changefeedID) metricEventChanSize := eventChanSizeHistogram.WithLabelValues(captureAddr, changefeedID) metricPullerResolvedTs := pullerResolvedTsGauge.WithLabelValues(captureAddr, changefeedID) @@ -162,9 +159,10 @@ func (p *pullerImpl) Run(ctx context.Context) error { zap.Reflect("row", raw), zap.Uint64("CRTs", raw.CRTs), zap.Uint64("resolvedTs", p.resolvedTs), - zap.Int64("tableID", tableID)) + zap.Int64("keyspanID", keyspanID)) return nil } + select { case <-ctx.Done(): return errors.Trace(ctx.Err()) @@ -182,6 +180,11 @@ func (p *pullerImpl) Run(ctx context.Context) error { case <-ctx.Done(): return errors.Trace(ctx.Err()) } + + log.Debug("revcive region feed event", + zap.String("RawKVEntry Key", string(e.Val.Key)), + zap.Uint64("ResolvedTS", e.Resolved.ResolvedTs)) + if e.Val != nil { metricTxnCollectCounterKv.Inc() if err := output(e.Val); err != nil { @@ -192,7 +195,7 @@ func (p *pullerImpl) Run(ctx context.Context) error { if !regionspan.IsSubSpan(e.Resolved.Span, p.spans...) { log.Panic("the resolved span is not in the total span", zap.Reflect("resolved", e.Resolved), - zap.Int64("tableID", tableID), + zap.Int64("keyspanID", keyspanID), zap.Reflect("spans", p.spans), ) } @@ -212,7 +215,7 @@ func (p *pullerImpl) Run(ctx context.Context) error { log.Info("puller is initialized", zap.Duration("duration", time.Since(start)), zap.String("changefeed", changefeedID), - zap.Int64("tableID", tableID), + zap.Int64("keyspanID", keyspanID), zap.Strings("spans", spans), zap.Uint64("resolvedTs", resolvedTs)) } diff --git a/cdc/cdc/redo/applier.go b/cdc/cdc/redo/applier.go deleted file mode 100644 index ca7fe125..00000000 --- a/cdc/cdc/redo/applier.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package redo - -import ( - "context" - - "github.com/tikv/migration/cdc/cdc/redo/reader" - cerror "github.com/tikv/migration/cdc/pkg/errors" -) - -// NewRedoReader creates a new redo log reader -func NewRedoReader(ctx context.Context, storage string, cfg *reader.LogReaderConfig) (rd reader.RedoLogReader, err error) { - switch consistentStorage(storage) { - case consistentStorageBlackhole: - rd = reader.NewBlackHoleReader() - case consistentStorageLocal, consistentStorageNFS, consistentStorageS3: - rd, err = reader.NewLogReader(ctx, cfg) - default: - err = cerror.ErrConsistentStorage.GenWithStackByArgs(storage) - } - return -} diff --git a/cdc/cdc/redo/common/redo.go b/cdc/cdc/redo/common/redo.go deleted file mode 100644 index 2ba2feb2..00000000 --- a/cdc/cdc/redo/common/redo.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:generate msgp - -package common - -const ( - // MinSectorSize is minimum sector size used when flushing log so that log can safely - // distinguish between torn writes and ordinary data corruption. - MinSectorSize = 512 -) - -const ( - // TmpEXT is the file ext of log file before safely wrote to disk - TmpEXT = ".tmp" - // LogEXT is the file ext of log file after safely wrote to disk - LogEXT = ".log" - // MetaEXT is the meta file ext of meta file after safely wrote to disk - MetaEXT = ".meta" - // MetaTmpEXT is the meta file ext of meta file before safely wrote to disk - MetaTmpEXT = ".mtmp" - // SortLogEXT is the sorted log file ext of log file after safely wrote to disk - SortLogEXT = ".sort" -) - -const ( - // DefaultFileMode is the default mode when operation files - DefaultFileMode = 0o644 - // DefaultDirMode is the default mode when operation dir - DefaultDirMode = 0o755 -) - -const ( - // DefaultMetaFileType is the default file type of meta file - DefaultMetaFileType = "meta" - // DefaultRowLogFileType is the default file type of row log file - DefaultRowLogFileType = "row" - // DefaultDDLLogFileType is the default file type of ddl log file - DefaultDDLLogFileType = "ddl" -) - -// LogMeta is used for store meta info. -type LogMeta struct { - CheckPointTs uint64 `msg:"checkPointTs"` - ResolvedTs uint64 `msg:"resolvedTs"` - ResolvedTsList map[int64]uint64 `msg:"-"` -} diff --git a/cdc/cdc/redo/common/redo_gen.go b/cdc/cdc/redo/common/redo_gen.go deleted file mode 100644 index bc79d273..00000000 --- a/cdc/cdc/redo/common/redo_gen.go +++ /dev/null @@ -1,135 +0,0 @@ -package common - -// Code generated by github.com/tinylib/msgp DO NOT EDIT. - -import ( - "github.com/tinylib/msgp/msgp" -) - -// DecodeMsg implements msgp.Decodable -func (z *LogMeta) DecodeMsg(dc *msgp.Reader) (err error) { - var field []byte - _ = field - var zb0001 uint32 - zb0001, err = dc.ReadMapHeader() - if err != nil { - err = msgp.WrapError(err) - return - } - for zb0001 > 0 { - zb0001-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - err = msgp.WrapError(err) - return - } - switch msgp.UnsafeString(field) { - case "checkPointTs": - z.CheckPointTs, err = dc.ReadUint64() - if err != nil { - err = msgp.WrapError(err, "CheckPointTs") - return - } - case "resolvedTs": - z.ResolvedTs, err = dc.ReadUint64() - if err != nil { - err = msgp.WrapError(err, "ResolvedTs") - return - } - default: - err = dc.Skip() - if err != nil { - err = msgp.WrapError(err) - return - } - } - } - return -} - -// EncodeMsg implements msgp.Encodable -func (z LogMeta) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 2 - // write "checkPointTs" - err = en.Append(0x82, 0xac, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x54, 0x73) - if err != nil { - return - } - err = en.WriteUint64(z.CheckPointTs) - if err != nil { - err = msgp.WrapError(err, "CheckPointTs") - return - } - // write "resolvedTs" - err = en.Append(0xaa, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x54, 0x73) - if err != nil { - return - } - err = en.WriteUint64(z.ResolvedTs) - if err != nil { - err = msgp.WrapError(err, "ResolvedTs") - return - } - return -} - -// MarshalMsg implements msgp.Marshaler -func (z LogMeta) MarshalMsg(b []byte) (o []byte, err error) { - o = msgp.Require(b, z.Msgsize()) - // map header, size 2 - // string "checkPointTs" - o = append(o, 0x82, 0xac, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x54, 0x73) - o = msgp.AppendUint64(o, z.CheckPointTs) - // string "resolvedTs" - o = append(o, 0xaa, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x54, 0x73) - o = msgp.AppendUint64(o, z.ResolvedTs) - return -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *LogMeta) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zb0001 uint32 - zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - for zb0001 > 0 { - zb0001-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - switch msgp.UnsafeString(field) { - case "checkPointTs": - z.CheckPointTs, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "CheckPointTs") - return - } - case "resolvedTs": - z.ResolvedTs, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "ResolvedTs") - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - } - } - o = bts - return -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z LogMeta) Msgsize() (s int) { - s = 1 + 13 + msgp.Uint64Size + 11 + msgp.Uint64Size - return -} diff --git a/cdc/cdc/redo/common/redo_gen_test.go b/cdc/cdc/redo/common/redo_gen_test.go deleted file mode 100644 index 4364e7bd..00000000 --- a/cdc/cdc/redo/common/redo_gen_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package common - -// Code generated by github.com/tinylib/msgp DO NOT EDIT. - -import ( - "bytes" - "testing" - - "github.com/tinylib/msgp/msgp" -) - -func TestMarshalUnmarshalLogMeta(t *testing.T) { - v := LogMeta{} - bts, err := v.MarshalMsg(nil) - if err != nil { - t.Fatal(err) - } - left, err := v.UnmarshalMsg(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) - } - - left, err = msgp.Skip(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after Skip(): %q", len(left), left) - } -} - -func BenchmarkMarshalMsgLogMeta(b *testing.B) { - v := LogMeta{} - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - v.MarshalMsg(nil) - } -} - -func BenchmarkAppendMsgLogMeta(b *testing.B) { - v := LogMeta{} - bts := make([]byte, 0, v.Msgsize()) - bts, _ = v.MarshalMsg(bts[0:0]) - b.SetBytes(int64(len(bts))) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - bts, _ = v.MarshalMsg(bts[0:0]) - } -} - -func BenchmarkUnmarshalLogMeta(b *testing.B) { - v := LogMeta{} - bts, _ := v.MarshalMsg(nil) - b.ReportAllocs() - b.SetBytes(int64(len(bts))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, err := v.UnmarshalMsg(bts) - if err != nil { - b.Fatal(err) - } - } -} - -func TestEncodeDecodeLogMeta(t *testing.T) { - v := LogMeta{} - var buf bytes.Buffer - msgp.Encode(&buf, &v) - - m := v.Msgsize() - if buf.Len() > m { - t.Log("WARNING: TestEncodeDecodeLogMeta Msgsize() is inaccurate") - } - - vn := LogMeta{} - err := msgp.Decode(&buf, &vn) - if err != nil { - t.Error(err) - } - - buf.Reset() - msgp.Encode(&buf, &v) - err = msgp.NewReader(&buf).Skip() - if err != nil { - t.Error(err) - } -} - -func BenchmarkEncodeLogMeta(b *testing.B) { - v := LogMeta{} - var buf bytes.Buffer - msgp.Encode(&buf, &v) - b.SetBytes(int64(buf.Len())) - en := msgp.NewWriter(msgp.Nowhere) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - v.EncodeMsg(en) - } - en.Flush() -} - -func BenchmarkDecodeLogMeta(b *testing.B) { - v := LogMeta{} - var buf bytes.Buffer - msgp.Encode(&buf, &v) - b.SetBytes(int64(buf.Len())) - rd := msgp.NewEndlessReader(buf.Bytes(), b) - dc := msgp.NewReader(rd) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - err := v.DecodeMsg(dc) - if err != nil { - b.Fatal(err) - } - } -} diff --git a/cdc/cdc/redo/common/util.go b/cdc/cdc/redo/common/util.go deleted file mode 100644 index f33cdf8e..00000000 --- a/cdc/cdc/redo/common/util.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package common - -import ( - "context" - "fmt" - "net/url" - "path/filepath" - "strings" - - "github.com/pingcap/errors" - backuppb "github.com/pingcap/kvproto/pkg/brpb" - "github.com/pingcap/tidb/br/pkg/storage" - cerror "github.com/tikv/migration/cdc/pkg/errors" -) - -// InitS3storage init a storage used for s3, -// s3URI should be like s3URI="s3://logbucket/test-changefeed?endpoint=http://$S3_ENDPOINT/" -var InitS3storage = func(ctx context.Context, uri url.URL) (storage.ExternalStorage, error) { - if len(uri.Host) == 0 { - return nil, cerror.WrapError(cerror.ErrS3StorageInitialize, errors.Errorf("please specify the bucket for s3 in %v", uri)) - } - - prefix := strings.Trim(uri.Path, "/") - s3 := &backuppb.S3{Bucket: uri.Host, Prefix: prefix} - options := &storage.BackendOptions{} - storage.ExtractQueryParameters(&uri, &options.S3) - if err := options.S3.Apply(s3); err != nil { - return nil, cerror.WrapError(cerror.ErrS3StorageInitialize, err) - } - - // we should set this to true, since br set it by default in parseBackend - s3.ForcePathStyle = true - backend := &backuppb.StorageBackend{ - Backend: &backuppb.StorageBackend_S3{S3: s3}, - } - s3storage, err := storage.New(ctx, backend, &storage.ExternalStorageOptions{ - SendCredentials: false, - HTTPClient: nil, - }) - if err != nil { - return nil, cerror.WrapError(cerror.ErrS3StorageInitialize, err) - } - - return s3storage, nil -} - -// ParseLogFileName extract the commitTs, fileType from log fileName -func ParseLogFileName(name string) (uint64, string, error) { - ext := filepath.Ext(name) - if ext == MetaEXT { - return 0, DefaultMetaFileType, nil - } - - // if .sort, the name should be like - // fmt.Sprintf("%s_%s_%d_%s_%d%s", w.cfg.captureID, w.cfg.changeFeedID, w.cfg.createTime.Unix(), w.cfg.fileType, w.commitTS.Load(), LogEXT)+SortLogEXT - if ext == SortLogEXT { - name = strings.TrimSuffix(name, SortLogEXT) - ext = filepath.Ext(name) - } - if ext != LogEXT && ext != TmpEXT { - return 0, "", nil - } - - var commitTs, d1 uint64 - var s1, s2, fileType string - // the log looks like: fmt.Sprintf("%s_%s_%d_%s_%d%s", w.cfg.captureID, w.cfg.changeFeedID, w.cfg.createTime.Unix(), w.cfg.fileType, w.commitTS.Load(), redo.LogEXT) - formatStr := "%s %s %d %s %d" + LogEXT - if ext == TmpEXT { - formatStr += TmpEXT - } - name = strings.ReplaceAll(name, "_", " ") - _, err := fmt.Sscanf(name, formatStr, &s1, &s2, &d1, &fileType, &commitTs) - if err != nil { - return 0, "", errors.Annotatef(err, "bad log name: %s", name) - } - - return commitTs, fileType, nil -} diff --git a/cdc/cdc/redo/common/util_test.go b/cdc/cdc/redo/common/util_test.go deleted file mode 100644 index acd9d92b..00000000 --- a/cdc/cdc/redo/common/util_test.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package common - -import ( - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -func TestParseLogFileName(t *testing.T) { - type arg struct { - name string - } - // the log looks like: fmt.Sprintf("%s_%s_%d_%s_%d%s", w.cfg.captureID, w.cfg.changeFeedID, w.cfg.createTime.Unix(), w.cfg.fileType, w.commitTS.Load(), redo.LogEXT) - tests := []struct { - name string - args arg - wantTs uint64 - wantFileType string - wantErr string - }{ - { - name: "happy row .log", - args: arg{ - name: fmt.Sprintf("%s_%s_%d_%s_%d%s", "cp", "test", time.Now().Unix(), DefaultRowLogFileType, 1, LogEXT), - }, - wantTs: 1, - wantFileType: DefaultRowLogFileType, - }, - { - name: "happy row .tmp", - args: arg{ - name: fmt.Sprintf("%s_%s_%d_%s_%d%s", "cp", "test", time.Now().Unix(), DefaultRowLogFileType, 1, LogEXT) + TmpEXT, - }, - wantTs: 1, - wantFileType: DefaultRowLogFileType, - }, - { - name: "happy ddl .log", - args: arg{ - name: fmt.Sprintf("%s_%s_%d_%s_%d%s", "cp", "test", time.Now().Unix(), DefaultDDLLogFileType, 1, LogEXT), - }, - wantTs: 1, - wantFileType: DefaultDDLLogFileType, - }, - { - name: "happy ddl .sort", - args: arg{ - name: fmt.Sprintf("%s_%s_%d_%s_%d%s", "cp", "test", time.Now().Unix(), DefaultDDLLogFileType, 1, LogEXT) + SortLogEXT, - }, - wantTs: 1, - wantFileType: DefaultDDLLogFileType, - }, - { - name: "happy ddl .tmp", - args: arg{ - name: fmt.Sprintf("%s_%s_%d_%s_%d%s", "cp", "test", time.Now().Unix(), DefaultDDLLogFileType, 1, LogEXT) + TmpEXT, - }, - wantTs: 1, - wantFileType: DefaultDDLLogFileType, - }, - { - name: "happy .meta", - args: arg{ - name: "sdfsdfsf" + MetaEXT, - }, - wantTs: 0, - wantFileType: DefaultMetaFileType, - }, - { - name: "not supported fileType", - args: arg{ - name: "sdfsdfsf.sfsf", - }, - }, - { - name: "err wrong format ddl .tmp", - args: arg{ - name: fmt.Sprintf("%s_%s_%d_%s%d%s", "cp", "test", time.Now().Unix(), DefaultDDLLogFileType, 1, LogEXT) + TmpEXT, - }, - wantErr: ".*bad log name*.", - }, - } - for _, tt := range tests { - ts, fileType, err := ParseLogFileName(tt.args.name) - if tt.wantErr != "" { - require.Regexp(t, tt.wantErr, err, tt.name) - } else { - require.Nil(t, err, tt.name) - require.EqualValues(t, tt.wantTs, ts, tt.name) - require.Equal(t, tt.wantFileType, fileType, tt.name) - } - } -} diff --git a/cdc/cdc/redo/convert.go b/cdc/cdc/redo/convert.go deleted file mode 100644 index a89c97e0..00000000 --- a/cdc/cdc/redo/convert.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package redo - -import ( - "bytes" - - pmodel "github.com/pingcap/tidb/parser/model" - "github.com/tikv/migration/cdc/cdc/model" -) - -// RowToRedo converts row changed event to redo log row -func RowToRedo(row *model.RowChangedEvent) *model.RedoRowChangedEvent { - redoLog := &model.RedoRowChangedEvent{ - Row: row, - Columns: make([]*model.RedoColumn, 0, len(row.Columns)), - PreColumns: make([]*model.RedoColumn, 0, len(row.PreColumns)), - } - for _, column := range row.Columns { - var redoColumn *model.RedoColumn - if column != nil { - // workaround msgp issue(Decode replaces empty slices with nil https://github.com/tinylib/msgp/issues/247) - // if []byte("") send with RowChangedEvent after UnmarshalMsg, - // the value will become nil, which is unexpected. - switch v := column.Value.(type) { - case []byte: - if bytes.Equal(v, []byte("")) { - column.Value = "" - } - } - redoColumn = &model.RedoColumn{Column: column, Flag: uint64(column.Flag)} - } - redoLog.Columns = append(redoLog.Columns, redoColumn) - } - for _, column := range row.PreColumns { - var redoColumn *model.RedoColumn - if column != nil { - switch v := column.Value.(type) { - case []byte: - if bytes.Equal(v, []byte("")) { - column.Value = "" - } - } - redoColumn = &model.RedoColumn{Column: column, Flag: uint64(column.Flag)} - } - redoLog.PreColumns = append(redoLog.PreColumns, redoColumn) - } - return redoLog -} - -// LogToRow converts redo log row to row changed event -func LogToRow(redoLog *model.RedoRowChangedEvent) *model.RowChangedEvent { - row := redoLog.Row - row.Columns = make([]*model.Column, 0, len(redoLog.Columns)) - row.PreColumns = make([]*model.Column, 0, len(redoLog.PreColumns)) - for _, column := range redoLog.PreColumns { - if column == nil { - row.PreColumns = append(row.PreColumns, nil) - continue - } - column.Column.Flag = model.ColumnFlagType(column.Flag) - row.PreColumns = append(row.PreColumns, column.Column) - } - for _, column := range redoLog.Columns { - if column == nil { - row.Columns = append(row.Columns, nil) - continue - } - column.Column.Flag = model.ColumnFlagType(column.Flag) - row.Columns = append(row.Columns, column.Column) - } - return row -} - -// DDLToRedo converts ddl event to redo log ddl -func DDLToRedo(ddl *model.DDLEvent) *model.RedoDDLEvent { - redoDDL := &model.RedoDDLEvent{ - DDL: ddl, - Type: byte(ddl.Type), - } - return redoDDL -} - -// LogToDDL converts redo log ddl to ddl event -func LogToDDL(redoDDL *model.RedoDDLEvent) *model.DDLEvent { - redoDDL.DDL.Type = pmodel.ActionType(redoDDL.Type) - return redoDDL.DDL -} diff --git a/cdc/cdc/redo/convert_test.go b/cdc/cdc/redo/convert_test.go deleted file mode 100644 index 4e1e438c..00000000 --- a/cdc/cdc/redo/convert_test.go +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package redo - -import ( - "testing" - - timodel "github.com/pingcap/tidb/parser/model" - "github.com/pingcap/tidb/parser/mysql" - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" -) - -func TestRowRedoConvert(t *testing.T) { - t.Parallel() - row := &model.RowChangedEvent{ - StartTs: 100, - CommitTs: 120, - Table: &model.TableName{Schema: "test", Table: "table1", TableID: 57}, - PreColumns: []*model.Column{{ - Name: "a1", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.MultipleKeyFlag | model.HandleKeyFlag, - Value: int64(1), - }, { - Name: "a2", - Type: mysql.TypeVarchar, - Value: "char", - }, { - Name: "a3", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.MultipleKeyFlag | model.HandleKeyFlag, - Value: int64(1), - }, nil}, - Columns: []*model.Column{{ - Name: "a1", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.MultipleKeyFlag | model.HandleKeyFlag, - Value: int64(2), - }, { - Name: "a2", - Type: mysql.TypeVarchar, - Value: "char-updated", - }, { - Name: "a3", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.MultipleKeyFlag | model.HandleKeyFlag, - Value: int64(2), - }, nil}, - IndexColumns: [][]int{{1, 3}}, - } - rowRedo := RowToRedo(row) - require.Equal(t, 4, len(rowRedo.PreColumns)) - require.Equal(t, 4, len(rowRedo.Columns)) - - redoLog := &model.RedoLog{ - RedoRow: rowRedo, - Type: model.RedoLogTypeRow, - } - data, err := redoLog.MarshalMsg(nil) - require.Nil(t, err) - redoLog2 := &model.RedoLog{} - _, err = redoLog2.UnmarshalMsg(data) - require.Nil(t, err) - require.Equal(t, row, LogToRow(redoLog2.RedoRow)) -} - -func TestRowRedoConvertWithEmptySlice(t *testing.T) { - t.Parallel() - row := &model.RowChangedEvent{ - StartTs: 100, - CommitTs: 120, - Table: &model.TableName{Schema: "test", Table: "table1", TableID: 57}, - PreColumns: []*model.Column{{ - Name: "a1", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.MultipleKeyFlag | model.HandleKeyFlag, - Value: int64(1), - }, { - Name: "a2", - Type: mysql.TypeVarchar, - Value: []byte(""), // empty slice should be marshal and unmarshal safely - }}, - Columns: []*model.Column{{ - Name: "a1", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.MultipleKeyFlag | model.HandleKeyFlag, - Value: int64(2), - }, { - Name: "a2", - Type: mysql.TypeVarchar, - Value: []byte(""), - }}, - IndexColumns: [][]int{{1}}, - } - rowRedo := RowToRedo(row) - redoLog := &model.RedoLog{ - RedoRow: rowRedo, - Type: model.RedoLogTypeRow, - } - data, err := redoLog.MarshalMsg(nil) - require.Nil(t, err) - - redoLog2 := &model.RedoLog{} - _, err = redoLog2.UnmarshalMsg(data) - require.Nil(t, err) - require.Equal(t, row, LogToRow(redoLog2.RedoRow)) -} - -func TestDDLRedoConvert(t *testing.T) { - t.Parallel() - ddl := &model.DDLEvent{ - StartTs: 1020, - CommitTs: 1030, - TableInfo: &model.SimpleTableInfo{ - Schema: "test", - Table: "t2", - }, - Type: timodel.ActionAddColumn, - Query: "ALTER TABLE test.t1 ADD COLUMN a int", - } - redoDDL := DDLToRedo(ddl) - - redoLog := &model.RedoLog{ - RedoDDL: redoDDL, - Type: model.RedoLogTypeDDL, - } - data, err := redoLog.MarshalMsg(nil) - require.Nil(t, err) - redoLog2 := &model.RedoLog{} - _, err = redoLog2.UnmarshalMsg(data) - require.Nil(t, err) - require.Equal(t, ddl, LogToDDL(redoLog2.RedoDDL)) -} diff --git a/cdc/cdc/redo/doc.go b/cdc/cdc/redo/doc.go deleted file mode 100644 index 5968846c..00000000 --- a/cdc/cdc/redo/doc.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -/* -Package redo provide a redo log for cdc. - -There are three types of log file: meta log file, row log file, ddl log file. -meta file used to store common.LogMeta info (CheckPointTs, ResolvedTs), atomic updated is guaranteed. A rotated file writer is used for other log files. -All files will flush to disk or upload to s3 if enabled every defaultFlushIntervalInMs 1000ms or file size larger than defaultMaxLogSize 64 MB by default. -The log file name is formatted as CaptureID_ChangeFeedID_CreateTime_FileType_MaxCommitTSOfAllEventInTheFile.log if safely wrote or end up with .log.tmp is not. -meta file name is like CaptureID_ChangeFeedID_meta.meta - -Each log file contains batch of model.RedoRowChangedEvent or model.RedoDDLEvent records wrote into different file with defaultMaxLogSize 64 MB. -If larger than 64 MB will auto rotated to a new file. -A record has a length field and a logical Log data. The length field is a 64-bit packed structure holding the length of the remaining logical Log data in its lower -56 bits and its physical padding in the first three bits of the most significant byte. Each record is 8-byte aligned so that the length field is never torn. - -When apply redo log from cli, will select files in the specific dir to open base on the startTs, endTs send from cli or download logs from s3 first is enabled, -then sort the event records in each file base on commitTs, after sorted, the new sort file name should be as CaptureID_ChangeFeedID_CreateTime_FileType_MaxCommitTSOfAllEventInTheFile.log.sort. - -*/ -package redo diff --git a/cdc/cdc/redo/manager.go b/cdc/cdc/redo/manager.go deleted file mode 100644 index e4631da0..00000000 --- a/cdc/cdc/redo/manager.go +++ /dev/null @@ -1,380 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package redo - -import ( - "context" - "math" - "path/filepath" - "sort" - "sync" - "sync/atomic" - "time" - - "github.com/pingcap/log" - "github.com/pingcap/tidb/br/pkg/storage" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/redo/writer" - "github.com/tikv/migration/cdc/pkg/config" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/util" - "go.uber.org/zap" -) - -var updateRtsInterval = time.Second - -// ConsistentLevelType is the level of redo log consistent level. -type ConsistentLevelType string - -const ( - // ConsistentLevelNone no consistent guarantee. - ConsistentLevelNone ConsistentLevelType = "none" - // ConsistentLevelEventual eventual consistent. - ConsistentLevelEventual ConsistentLevelType = "eventual" -) - -type consistentStorage string - -const ( - consistentStorageLocal consistentStorage = "local" - consistentStorageNFS consistentStorage = "nfs" - consistentStorageS3 consistentStorage = "s3" - consistentStorageBlackhole consistentStorage = "blackhole" -) - -const ( - // supposing to replicate 10k tables, each table has one cached changce averagely. - logBufferChanSize = 10000 - logBufferTimeout = time.Minute * 10 -) - -// IsValidConsistentLevel checks whether a give consistent level is valid -func IsValidConsistentLevel(level string) bool { - switch ConsistentLevelType(level) { - case ConsistentLevelNone, ConsistentLevelEventual: - return true - default: - return false - } -} - -// IsValidConsistentStorage checks whether a give consistent storage is valid -func IsValidConsistentStorage(storage string) bool { - switch consistentStorage(storage) { - case consistentStorageLocal, consistentStorageNFS, - consistentStorageS3, consistentStorageBlackhole: - return true - default: - return false - } -} - -// IsConsistentEnabled returns whether the consistent feature is enabled -func IsConsistentEnabled(level string) bool { - return IsValidConsistentLevel(level) && ConsistentLevelType(level) != ConsistentLevelNone -} - -// IsS3StorageEnabled returns whether s3 storage is enabled -func IsS3StorageEnabled(storage string) bool { - return consistentStorage(storage) == consistentStorageS3 -} - -// LogManager defines an interface that is used to manage redo log -type LogManager interface { - // Enabled returns whether the log manager is enabled - Enabled() bool - - // The following 5 APIs are called from processor only - EmitRowChangedEvents(ctx context.Context, tableID model.TableID, rows ...*model.RowChangedEvent) error - FlushLog(ctx context.Context, tableID model.TableID, resolvedTs uint64) error - AddTable(tableID model.TableID, startTs uint64) - RemoveTable(tableID model.TableID) - GetMinResolvedTs() uint64 - - // EmitDDLEvent and FlushResolvedAndCheckpointTs are called from owner only - EmitDDLEvent(ctx context.Context, ddl *model.DDLEvent) error - FlushResolvedAndCheckpointTs(ctx context.Context, resolvedTs, checkpointTs uint64) (err error) - - // Cleanup removes all redo logs - Cleanup(ctx context.Context) error -} - -// ManagerOptions defines options for redo log manager -type ManagerOptions struct { - // whether to run background goroutine to fetch table resolved ts - EnableBgRunner bool - ErrCh chan<- error -} - -type cacheRows struct { - tableID model.TableID - rows []*model.RowChangedEvent -} - -// ManagerImpl manages redo log writer, buffers un-persistent redo logs, calculates -// redo log resolved ts. It implements LogManager interface. -type ManagerImpl struct { - enabled bool - level ConsistentLevelType - storageType consistentStorage - - logBuffer chan cacheRows - writer writer.RedoLogWriter - - minResolvedTs uint64 - tableIDs []model.TableID - rtsMap map[model.TableID]uint64 - rtsMapMu sync.RWMutex - - // record whether there exists a table being flushing resolved ts - flushing int64 -} - -// NewManager creates a new Manager -func NewManager(ctx context.Context, cfg *config.ConsistentConfig, opts *ManagerOptions) (*ManagerImpl, error) { - // return a disabled Manager if no consistent config or normal consistent level - if cfg == nil || ConsistentLevelType(cfg.Level) == ConsistentLevelNone { - return &ManagerImpl{enabled: false}, nil - } - uri, err := storage.ParseRawURL(cfg.Storage) - if err != nil { - return nil, err - } - m := &ManagerImpl{ - enabled: true, - level: ConsistentLevelType(cfg.Level), - storageType: consistentStorage(uri.Scheme), - rtsMap: make(map[model.TableID]uint64), - logBuffer: make(chan cacheRows, logBufferChanSize), - } - - switch m.storageType { - case consistentStorageBlackhole: - m.writer = writer.NewBlackHoleWriter() - case consistentStorageLocal, consistentStorageNFS, consistentStorageS3: - globalConf := config.GetGlobalServerConfig() - changeFeedID := util.ChangefeedIDFromCtx(ctx) - // We use a temporary dir to storage redo logs before flushing to other backends, such as S3 - redoDir := filepath.Join(globalConf.DataDir, config.DefaultRedoDir, changeFeedID) - if m.storageType == consistentStorageLocal || m.storageType == consistentStorageNFS { - // When using local or nfs as backend, store redo logs to redoDir directly. - redoDir = uri.Path - } - - writerCfg := &writer.LogWriterConfig{ - Dir: redoDir, - CaptureID: util.CaptureAddrFromCtx(ctx), - ChangeFeedID: changeFeedID, - CreateTime: time.Now(), - MaxLogSize: cfg.MaxLogSize, - FlushIntervalInMs: cfg.FlushIntervalInMs, - S3Storage: m.storageType == consistentStorageS3, - } - if writerCfg.S3Storage { - writerCfg.S3URI = *uri - } - writer, err := writer.NewLogWriter(ctx, writerCfg) - if err != nil { - return nil, err - } - m.writer = writer - default: - return nil, cerror.ErrConsistentStorage.GenWithStackByArgs(m.storageType) - } - - if opts.EnableBgRunner { - go m.bgUpdateResolvedTs(ctx, opts.ErrCh) - go m.bgWriteLog(ctx, opts.ErrCh) - } - return m, nil -} - -// NewDisabledManager returns a disabled log manger instance, used in test only -func NewDisabledManager() *ManagerImpl { - return &ManagerImpl{enabled: false} -} - -// Enabled returns whether this log manager is enabled -func (m *ManagerImpl) Enabled() bool { - return m.enabled -} - -// EmitRowChangedEvents sends row changed events to a log buffer, the log buffer -// will be consumed by a background goroutine, which converts row changed events -// to redo logs and sends to log writer. Note this function is non-blocking if -// the channel is not full, otherwise if the channel is always full after timeout, -// error ErrBufferLogTimeout will be returned. -// TODO: if the API is truly non-blocking, we should return an error immediately -// when the log buffer channel is full. -func (m *ManagerImpl) EmitRowChangedEvents( - ctx context.Context, - tableID model.TableID, - rows ...*model.RowChangedEvent, -) error { - timer := time.NewTimer(logBufferTimeout) - defer timer.Stop() - select { - case <-ctx.Done(): - return nil - case <-timer.C: - return cerror.ErrBufferLogTimeout.GenWithStackByArgs() - case m.logBuffer <- cacheRows{ - tableID: tableID, - // Because the pipeline sink doesn't hold slice memory after calling - // EmitRowChangedEvents, we copy to a new slice to manage memory - // in redo manager itself, which is the same behavior as sink manager. - rows: append(make([]*model.RowChangedEvent, 0, len(rows)), rows...), - }: - } - return nil -} - -// FlushLog emits resolved ts of a single table -func (m *ManagerImpl) FlushLog( - ctx context.Context, - tableID model.TableID, - resolvedTs uint64, -) error { - // Use flushing as a lightweight lock to reduce log contention in log writer. - if !atomic.CompareAndSwapInt64(&m.flushing, 0, 1) { - return nil - } - defer atomic.StoreInt64(&m.flushing, 0) - return m.writer.FlushLog(ctx, tableID, resolvedTs) -} - -// EmitDDLEvent sends DDL event to redo log writer -func (m *ManagerImpl) EmitDDLEvent(ctx context.Context, ddl *model.DDLEvent) error { - return m.writer.SendDDL(ctx, DDLToRedo(ddl)) -} - -// GetMinResolvedTs returns the minimum resolved ts of all tables in this redo log manager -func (m *ManagerImpl) GetMinResolvedTs() uint64 { - return atomic.LoadUint64(&m.minResolvedTs) -} - -// FlushResolvedAndCheckpointTs flushes resolved-ts and checkpoint-ts to redo log writer -func (m *ManagerImpl) FlushResolvedAndCheckpointTs(ctx context.Context, resolvedTs, checkpointTs uint64) (err error) { - err = m.writer.EmitResolvedTs(ctx, resolvedTs) - if err != nil { - return - } - err = m.writer.EmitCheckpointTs(ctx, checkpointTs) - return -} - -// AddTable adds a new table in redo log manager -func (m *ManagerImpl) AddTable(tableID model.TableID, startTs uint64) { - m.rtsMapMu.Lock() - defer m.rtsMapMu.Unlock() - i := sort.Search(len(m.tableIDs), func(i int) bool { - return m.tableIDs[i] >= tableID - }) - if i < len(m.tableIDs) && m.tableIDs[i] == tableID { - log.Warn("add duplicated table in redo log manager", zap.Int64("table-id", tableID)) - return - } - if i == len(m.tableIDs) { - m.tableIDs = append(m.tableIDs, tableID) - } else { - m.tableIDs = append(m.tableIDs[:i+1], m.tableIDs[i:]...) - m.tableIDs[i] = tableID - } - m.rtsMap[tableID] = startTs -} - -// RemoveTable removes a table from redo log manager -func (m *ManagerImpl) RemoveTable(tableID model.TableID) { - m.rtsMapMu.Lock() - defer m.rtsMapMu.Unlock() - i := sort.Search(len(m.tableIDs), func(i int) bool { - return m.tableIDs[i] >= tableID - }) - if i < len(m.tableIDs) && m.tableIDs[i] == tableID { - copy(m.tableIDs[i:], m.tableIDs[i+1:]) - m.tableIDs = m.tableIDs[:len(m.tableIDs)-1] - delete(m.rtsMap, tableID) - } else { - log.Warn("remove a table not maintained in redo log manager", zap.Int64("table-id", tableID)) - } -} - -// Cleanup removes all redo logs of this manager, it is called when changefeed is removed -func (m *ManagerImpl) Cleanup(ctx context.Context) error { - return m.writer.DeleteAllLogs(ctx) -} - -// updateTableResolvedTs reads rtsMap from redo log writer and calculate the minimum -// resolved ts of all maintaining tables. -func (m *ManagerImpl) updateTableResolvedTs(ctx context.Context) error { - m.rtsMapMu.Lock() - defer m.rtsMapMu.Unlock() - rtsMap, err := m.writer.GetCurrentResolvedTs(ctx, m.tableIDs) - if err != nil { - return err - } - minResolvedTs := uint64(math.MaxUint64) - for tableID, rts := range rtsMap { - m.rtsMap[tableID] = rts - if rts < minResolvedTs { - minResolvedTs = rts - } - } - atomic.StoreUint64(&m.minResolvedTs, minResolvedTs) - return nil -} - -func (m *ManagerImpl) bgUpdateResolvedTs(ctx context.Context, errCh chan<- error) { - ticker := time.NewTicker(updateRtsInterval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - err := m.updateTableResolvedTs(ctx) - if err != nil { - select { - case errCh <- err: - default: - log.Error("err channel is full", zap.Error(err)) - } - return - } - } - } -} - -func (m *ManagerImpl) bgWriteLog(ctx context.Context, errCh chan<- error) { - for { - select { - case <-ctx.Done(): - return - case cache := <-m.logBuffer: - logs := make([]*model.RedoRowChangedEvent, 0, len(cache.rows)) - for _, row := range cache.rows { - logs = append(logs, RowToRedo(row)) - } - _, err := m.writer.WriteLog(ctx, cache.tableID, logs) - if err != nil { - select { - case errCh <- err: - default: - log.Error("err channel is full", zap.Error(err)) - } - return - } - } - } -} diff --git a/cdc/cdc/redo/manager_test.go b/cdc/cdc/redo/manager_test.go deleted file mode 100644 index 6b8eab1f..00000000 --- a/cdc/cdc/redo/manager_test.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package redo - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/config" -) - -func TestConsistentConfig(t *testing.T) { - t.Parallel() - levelCases := []struct { - level string - valid bool - }{ - {"none", true}, - {"eventual", true}, - {"NONE", false}, - {"", false}, - } - for _, lc := range levelCases { - require.Equal(t, lc.valid, IsValidConsistentLevel(lc.level)) - } - - levelEnableCases := []struct { - level string - consistent bool - }{ - {"invalid-level", false}, - {"none", false}, - {"eventual", true}, - } - for _, lc := range levelEnableCases { - require.Equal(t, lc.consistent, IsConsistentEnabled(lc.level)) - } - - storageCases := []struct { - storage string - valid bool - }{ - {"local", true}, - {"nfs", true}, - {"s3", true}, - {"blackhole", true}, - {"Local", false}, - {"", false}, - } - for _, sc := range storageCases { - require.Equal(t, sc.valid, IsValidConsistentStorage(sc.storage)) - } - - s3StorageCases := []struct { - storage string - s3Enabled bool - }{ - {"local", false}, - {"nfs", false}, - {"s3", true}, - {"blackhole", false}, - } - for _, sc := range s3StorageCases { - require.Equal(t, sc.s3Enabled, IsS3StorageEnabled(sc.storage)) - } -} - -// TestLogManagerInProcessor tests how redo log manager is used in processor, -// where the redo log manager needs to handle DMLs and redo log meta data -func TestLogManagerInProcessor(t *testing.T) { - t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - checkResovledTs := func(mgr LogManager, expectedRts uint64) { - time.Sleep(time.Millisecond*200 + updateRtsInterval) - resolvedTs := mgr.GetMinResolvedTs() - require.Equal(t, expectedRts, resolvedTs) - } - - cfg := &config.ConsistentConfig{ - Level: string(ConsistentLevelEventual), - Storage: "blackhole://", - } - errCh := make(chan error, 1) - opts := &ManagerOptions{ - EnableBgRunner: true, - ErrCh: errCh, - } - logMgr, err := NewManager(ctx, cfg, opts) - require.Nil(t, err) - - // check emit row changed events can move forward resolved ts - tables := []model.TableID{53, 55, 57, 59} - startTs := uint64(100) - for _, tableID := range tables { - logMgr.AddTable(tableID, startTs) - } - testCases := []struct { - tableID model.TableID - rows []*model.RowChangedEvent - }{ - { - tableID: 53, - rows: []*model.RowChangedEvent{ - {CommitTs: 120, Table: &model.TableName{TableID: 53}}, - {CommitTs: 125, Table: &model.TableName{TableID: 53}}, - {CommitTs: 130, Table: &model.TableName{TableID: 53}}, - }, - }, - { - tableID: 55, - rows: []*model.RowChangedEvent{ - {CommitTs: 130, Table: &model.TableName{TableID: 55}}, - {CommitTs: 135, Table: &model.TableName{TableID: 55}}, - }, - }, - { - tableID: 57, - rows: []*model.RowChangedEvent{ - {CommitTs: 130, Table: &model.TableName{TableID: 57}}, - }, - }, - { - tableID: 59, - rows: []*model.RowChangedEvent{ - {CommitTs: 128, Table: &model.TableName{TableID: 59}}, - {CommitTs: 130, Table: &model.TableName{TableID: 59}}, - {CommitTs: 133, Table: &model.TableName{TableID: 59}}, - }, - }, - } - for _, tc := range testCases { - err := logMgr.EmitRowChangedEvents(ctx, tc.tableID, tc.rows...) - require.Nil(t, err) - } - checkResovledTs(logMgr, uint64(130)) - - // check FlushLog can move forward the resolved ts when there is not row event. - flushResolvedTs := uint64(150) - for _, tableID := range tables { - err := logMgr.FlushLog(ctx, tableID, flushResolvedTs) - require.Nil(t, err) - } - checkResovledTs(logMgr, flushResolvedTs) - - // check remove table can work normally - removeTable := tables[len(tables)-1] - tables = tables[:len(tables)-1] - logMgr.RemoveTable(removeTable) - flushResolvedTs = uint64(200) - for _, tableID := range tables { - err := logMgr.FlushLog(ctx, tableID, flushResolvedTs) - require.Nil(t, err) - } - checkResovledTs(logMgr, flushResolvedTs) - - err = logMgr.FlushResolvedAndCheckpointTs(ctx, 200 /*resolvedTs*/, 120 /*CheckPointTs*/) - require.Nil(t, err) -} - -// TestLogManagerInOwner tests how redo log manager is used in owner, -// where the redo log manager needs to handle DDL event only. -func TestLogManagerInOwner(t *testing.T) { - t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - cfg := &config.ConsistentConfig{ - Level: string(ConsistentLevelEventual), - Storage: "blackhole://", - } - opts := &ManagerOptions{ - EnableBgRunner: false, - } - logMgr, err := NewManager(ctx, cfg, opts) - require.Nil(t, err) - - ddl := &model.DDLEvent{StartTs: 100, CommitTs: 120, Query: "CREATE TABLE `TEST.T1`"} - err = logMgr.EmitDDLEvent(ctx, ddl) - require.Nil(t, err) - - err = logMgr.writer.DeleteAllLogs(ctx) - require.Nil(t, err) -} diff --git a/cdc/cdc/redo/reader/blackhole_reader.go b/cdc/cdc/redo/reader/blackhole_reader.go deleted file mode 100644 index dc171a9a..00000000 --- a/cdc/cdc/redo/reader/blackhole_reader.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package reader - -import ( - "context" - - "github.com/tikv/migration/cdc/cdc/model" -) - -// BlackHoleReader is a blockHole storage which implements LogReader interface -type BlackHoleReader struct{} - -// NewBlackHoleReader creates a new BlackHoleReader -func NewBlackHoleReader() *BlackHoleReader { - return &BlackHoleReader{} -} - -// ResetReader implements LogReader.ReadLog -func (br *BlackHoleReader) ResetReader(ctx context.Context, startTs, endTs uint64) error { - return nil -} - -// ReadNextLog implements LogReader.ReadNextLog -func (br *BlackHoleReader) ReadNextLog(ctx context.Context, maxNumberOfEvents uint64) ([]*model.RedoRowChangedEvent, error) { - return nil, nil -} - -// ReadNextDDL implements LogReader.ReadNextDDL -func (br *BlackHoleReader) ReadNextDDL(ctx context.Context, maxNumberOfEvents uint64) ([]*model.RedoDDLEvent, error) { - return nil, nil -} - -// ReadMeta implements LogReader.ReadMeta -func (br *BlackHoleReader) ReadMeta(ctx context.Context) (checkpointTs, resolvedTs uint64, err error) { - return 0, 1, nil -} - -// Close implement the Close interface -func (br *BlackHoleReader) Close() error { - return nil -} diff --git a/cdc/cdc/redo/reader/file.go b/cdc/cdc/redo/reader/file.go deleted file mode 100644 index ce10dd9e..00000000 --- a/cdc/cdc/redo/reader/file.go +++ /dev/null @@ -1,471 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package reader - -import ( - "bufio" - "container/heap" - "context" - "encoding/binary" - "io" - "io/ioutil" - "math" - "net/url" - "os" - "path/filepath" - "sync" - - "github.com/pingcap/errors" - "github.com/pingcap/log" - "github.com/pingcap/tidb/br/pkg/storage" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/redo/common" - "github.com/tikv/migration/cdc/cdc/redo/writer" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "go.uber.org/multierr" - "go.uber.org/zap" - "golang.org/x/sync/errgroup" -) - -const ( - // frameSizeBytes is frame size in bytes, including record size and padding size. - frameSizeBytes = 8 - - // defaultWorkerNum is the num of workers used to sort the log file to sorted file, - // will load the file to memory first then write the sorted file to disk - // the memory used is defaultWorkerNum * defaultMaxLogSize (64 * megabyte) total - defaultWorkerNum = 50 -) - -//go:generate mockery --name=fileReader --inpackage -type fileReader interface { - io.Closer - // Read return the log from log file - Read(log *model.RedoLog) error -} - -type readerConfig struct { - dir string - fileType string - startTs uint64 - endTs uint64 - s3Storage bool - s3URI url.URL - workerNums int -} - -type reader struct { - cfg *readerConfig - mu sync.Mutex - br *bufio.Reader - fileName string - closer io.Closer - // lastValidOff file offset following the last valid decoded record - lastValidOff int64 -} - -func newReader(ctx context.Context, cfg *readerConfig) ([]fileReader, error) { - if cfg == nil { - return nil, cerror.WrapError(cerror.ErrRedoConfigInvalid, errors.New("readerConfig can not be nil")) - } - - if cfg.s3Storage { - s3storage, err := common.InitS3storage(ctx, cfg.s3URI) - if err != nil { - return nil, err - } - - err = downLoadToLocal(ctx, cfg.dir, s3storage, cfg.fileType) - if err != nil { - return nil, cerror.WrapError(cerror.ErrRedoDownloadFailed, err) - } - } - if cfg.workerNums == 0 { - cfg.workerNums = defaultWorkerNum - } - - rr, err := openSelectedFiles(ctx, cfg.dir, cfg.fileType, cfg.startTs, cfg.workerNums) - if err != nil { - return nil, err - } - - readers := []fileReader{} - for i := range rr { - readers = append(readers, - &reader{ - cfg: cfg, - br: bufio.NewReader(rr[i]), - fileName: rr[i].(*os.File).Name(), - closer: rr[i], - }) - } - - return readers, nil -} - -func selectDownLoadFile(ctx context.Context, s3storage storage.ExternalStorage, fixedType string) ([]string, error) { - files := []string{} - err := s3storage.WalkDir(ctx, &storage.WalkOption{}, func(path string, size int64) error { - fileName := filepath.Base(path) - _, fileType, err := common.ParseLogFileName(fileName) - if err != nil { - return err - } - - if fileType == fixedType { - files = append(files, path) - } - return nil - }) - if err != nil { - return nil, cerror.WrapError(cerror.ErrS3StorageAPI, err) - } - - return files, nil -} - -func downLoadToLocal(ctx context.Context, dir string, s3storage storage.ExternalStorage, fixedType string) error { - files, err := selectDownLoadFile(ctx, s3storage, fixedType) - if err != nil { - return err - } - - eg, eCtx := errgroup.WithContext(ctx) - for _, file := range files { - f := file - eg.Go(func() error { - data, err := s3storage.ReadFile(eCtx, f) - if err != nil { - return cerror.WrapError(cerror.ErrS3StorageAPI, err) - } - - err = os.MkdirAll(dir, common.DefaultDirMode) - if err != nil { - return cerror.WrapError(cerror.ErrRedoFileOp, err) - } - path := filepath.Join(dir, f) - err = ioutil.WriteFile(path, data, common.DefaultFileMode) - return cerror.WrapError(cerror.ErrRedoFileOp, err) - }) - } - - return eg.Wait() -} - -func openSelectedFiles(ctx context.Context, dir, fixedType string, startTs uint64, workerNum int) ([]io.ReadCloser, error) { - files, err := ioutil.ReadDir(dir) - if err != nil { - return nil, cerror.WrapError(cerror.ErrRedoFileOp, errors.Annotatef(err, "can't read log file directory: %s", dir)) - } - - sortedFileList := map[string]bool{} - for _, file := range files { - if filepath.Ext(file.Name()) == common.SortLogEXT { - sortedFileList[file.Name()] = false - } - } - - logFiles := []io.ReadCloser{} - unSortedFile := []string{} - for _, f := range files { - name := f.Name() - ret, err := shouldOpen(startTs, name, fixedType) - if err != nil { - log.Warn("check selected log file fail", - zap.String("log file", name), - zap.Error(err)) - continue - } - - if ret { - sortedName := name - if filepath.Ext(sortedName) != common.SortLogEXT { - sortedName += common.SortLogEXT - } - if opened, ok := sortedFileList[sortedName]; ok { - if opened { - continue - } - } else { - unSortedFile = append(unSortedFile, name) - continue - } - path := filepath.Join(dir, sortedName) - file, err := openReadFile(path) - if err != nil { - return nil, cerror.WrapError(cerror.ErrRedoFileOp, errors.Annotate(err, "can't open redo logfile")) - } - logFiles = append(logFiles, file) - sortedFileList[sortedName] = true - } - } - - sortFiles, err := createSortedFiles(ctx, dir, unSortedFile, workerNum) - if err != nil { - return nil, err - } - logFiles = append(logFiles, sortFiles...) - return logFiles, nil -} - -func openReadFile(name string) (*os.File, error) { - return os.OpenFile(name, os.O_RDONLY, common.DefaultFileMode) -} - -func readFile(file *os.File) (logHeap, error) { - r := &reader{ - br: bufio.NewReader(file), - fileName: file.Name(), - closer: file, - } - defer r.Close() - - h := logHeap{} - for { - rl := &model.RedoLog{} - err := r.Read(rl) - if err != nil { - if err != io.EOF { - return nil, err - } - break - } - h = append(h, &logWithIdx{data: rl}) - } - - return h, nil -} - -// writFile if not safely closed, the sorted file will end up with .sort.tmp as the file name suffix -func writFile(ctx context.Context, dir, name string, h logHeap) error { - cfg := &writer.FileWriterConfig{ - Dir: dir, - MaxLogSize: math.MaxInt32, - } - w, err := writer.NewWriter(ctx, cfg, writer.WithLogFileName(func() string { return name })) - if err != nil { - return err - } - - for h.Len() != 0 { - item := heap.Pop(&h).(*logWithIdx).data - data, err := item.MarshalMsg(nil) - if err != nil { - return cerror.WrapError(cerror.ErrMarshalFailed, err) - } - _, err = w.Write(data) - if err != nil { - return err - } - } - - return w.Close() -} - -func createSortedFiles(ctx context.Context, dir string, names []string, workerNum int) ([]io.ReadCloser, error) { - logFiles := []io.ReadCloser{} - errCh := make(chan error) - retCh := make(chan io.ReadCloser) - - var errs error - i := 0 - for i != len(names) { - nn := []string{} - for i < len(names) { - if len(nn) < workerNum { - nn = append(nn, names[i]) - i++ - continue - } - break - } - - for i := 0; i < len(nn); i++ { - go createSortedFile(ctx, dir, nn[i], errCh, retCh) - } - for i := 0; i < len(nn); i++ { - select { - case err := <-errCh: - errs = multierr.Append(errs, err) - case ret := <-retCh: - if ret != nil { - logFiles = append(logFiles, ret) - } - } - } - if errs != nil { - return nil, errs - } - } - - return logFiles, nil -} - -func createSortedFile(ctx context.Context, dir string, name string, errCh chan error, retCh chan io.ReadCloser) { - path := filepath.Join(dir, name) - file, err := openReadFile(path) - if err != nil { - errCh <- cerror.WrapError(cerror.ErrRedoFileOp, errors.Annotate(err, "can't open redo logfile")) - return - } - - h, err := readFile(file) - if err != nil { - errCh <- err - return - } - - heap.Init(&h) - if h.Len() == 0 { - retCh <- nil - return - } - - sortFileName := name + common.SortLogEXT - err = writFile(ctx, dir, sortFileName, h) - if err != nil { - errCh <- err - return - } - - file, err = openReadFile(filepath.Join(dir, sortFileName)) - if err != nil { - errCh <- cerror.WrapError(cerror.ErrRedoFileOp, errors.Annotate(err, "can't open redo logfile")) - return - } - retCh <- file -} - -func shouldOpen(startTs uint64, name, fixedType string) (bool, error) { - // .sort.tmp will return error - commitTs, fileType, err := common.ParseLogFileName(name) - if err != nil { - return false, err - } - if fileType != fixedType { - return false, nil - } - // always open .tmp - if filepath.Ext(name) == common.TmpEXT { - return true, nil - } - // the commitTs=max(ts of log item in the file), if max > startTs then should open, - // filter out ts in (startTs, endTs] for consume - return commitTs > startTs, nil -} - -// Read implement Read interface. -// TODO: more general reader pair with writer in writer pkg -func (r *reader) Read(redoLog *model.RedoLog) error { - r.mu.Lock() - defer r.mu.Unlock() - - lenField, err := readInt64(r.br) - if err != nil { - if err == io.EOF { - return err - } - return cerror.WrapError(cerror.ErrRedoFileOp, err) - } - - recBytes, padBytes := decodeFrameSize(lenField) - data := make([]byte, recBytes+padBytes) - _, err = io.ReadFull(r.br, data) - if err != nil { - if err == io.EOF || err == io.ErrUnexpectedEOF { - log.Warn("read redo log have unexpected io error", - zap.String("fileName", r.fileName), - zap.Error(err)) - return io.EOF - } - return cerror.WrapError(cerror.ErrRedoFileOp, err) - } - - _, err = redoLog.UnmarshalMsg(data[:recBytes]) - if err != nil { - if r.isTornEntry(data) { - // just return io.EOF, since if torn write it is the last redoLog entry - return io.EOF - } - return cerror.WrapError(cerror.ErrUnmarshalFailed, err) - } - - // point last valid offset to the end of redoLog - r.lastValidOff += frameSizeBytes + recBytes + padBytes - return nil -} - -func readInt64(r io.Reader) (int64, error) { - var n int64 - err := binary.Read(r, binary.LittleEndian, &n) - return n, err -} - -// decodeFrameSize pair with encodeFrameSize in writer.file -// the func use code from etcd wal/decoder.go -func decodeFrameSize(lenField int64) (recBytes int64, padBytes int64) { - // the record size is stored in the lower 56 bits of the 64-bit length - recBytes = int64(uint64(lenField) & ^(uint64(0xff) << 56)) - // non-zero padding is indicated by set MSb / a negative length - if lenField < 0 { - // padding is stored in lower 3 bits of length MSB - padBytes = int64((uint64(lenField) >> 56) & 0x7) - } - return recBytes, padBytes -} - -// isTornEntry determines whether the last entry of the Log was partially written -// and corrupted because of a torn write. -// the func use code from etcd wal/decoder.go -// ref: https://github.com/etcd-io/etcd/pull/5250 -func (r *reader) isTornEntry(data []byte) bool { - fileOff := r.lastValidOff + frameSizeBytes - curOff := 0 - chunks := [][]byte{} - // split data on sector boundaries - for curOff < len(data) { - chunkLen := int(common.MinSectorSize - (fileOff % common.MinSectorSize)) - if chunkLen > len(data)-curOff { - chunkLen = len(data) - curOff - } - chunks = append(chunks, data[curOff:curOff+chunkLen]) - fileOff += int64(chunkLen) - curOff += chunkLen - } - - // if any data for a sector chunk is all 0, it's a torn write - for _, sect := range chunks { - isZero := true - for _, v := range sect { - if v != 0 { - isZero = false - break - } - } - if isZero { - return true - } - } - return false -} - -// Close implement the Close interface -func (r *reader) Close() error { - if r == nil || r.closer == nil { - return nil - } - - return cerror.WrapError(cerror.ErrRedoFileOp, r.closer.Close()) -} diff --git a/cdc/cdc/redo/reader/file_test.go b/cdc/cdc/redo/reader/file_test.go deleted file mode 100644 index 876414ff..00000000 --- a/cdc/cdc/redo/reader/file_test.go +++ /dev/null @@ -1,253 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package reader - -import ( - "bufio" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/redo/common" - "github.com/tikv/migration/cdc/cdc/redo/writer" - "github.com/tikv/migration/cdc/pkg/leakutil" - "golang.org/x/net/context" -) - -func TestMain(m *testing.M) { - leakutil.SetUpLeakTest(m) -} - -func TestReaderNewReader(t *testing.T) { - _, err := newReader(context.Background(), nil) - require.NotNil(t, err) - - dir, err := ioutil.TempDir("", "redo-newReader") - require.Nil(t, err) - defer os.RemoveAll(dir) - _, err = newReader(context.Background(), &readerConfig{dir: dir}) - require.Nil(t, err) -} - -func TestReaderRead(t *testing.T) { - dir, err := ioutil.TempDir("", "redo-reader") - require.Nil(t, err) - defer os.RemoveAll(dir) - - cfg := &writer.FileWriterConfig{ - MaxLogSize: 100000, - Dir: dir, - ChangeFeedID: "test-cf", - CaptureID: "cp", - FileType: common.DefaultRowLogFileType, - CreateTime: time.Date(2000, 1, 1, 1, 1, 1, 1, &time.Location{}), - } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - w, err := writer.NewWriter(ctx, cfg) - require.Nil(t, err) - log := &model.RedoLog{ - RedoRow: &model.RedoRowChangedEvent{Row: &model.RowChangedEvent{CommitTs: 1123}}, - } - data, err := log.MarshalMsg(nil) - require.Nil(t, err) - w.AdvanceTs(11) - _, err = w.Write(data) - require.Nil(t, err) - err = w.Close() - require.Nil(t, err) - require.True(t, !w.IsRunning()) - fileName := fmt.Sprintf("%s_%s_%d_%s_%d%s", cfg.CaptureID, cfg.ChangeFeedID, cfg.CreateTime.Unix(), cfg.FileType, 11, common.LogEXT) - path := filepath.Join(cfg.Dir, fileName) - info, err := os.Stat(path) - require.Nil(t, err) - require.Equal(t, fileName, info.Name()) - - r, err := newReader(ctx, &readerConfig{ - dir: dir, - startTs: 1, - endTs: 12, - fileType: common.DefaultRowLogFileType, - }) - require.Nil(t, err) - require.Equal(t, 1, len(r)) - defer r[0].Close() //nolint:errcheck - log = &model.RedoLog{} - err = r[0].Read(log) - require.Nil(t, err) - require.EqualValues(t, 1123, log.RedoRow.Row.CommitTs) - time.Sleep(1001 * time.Millisecond) -} - -func TestReaderOpenSelectedFiles(t *testing.T) { - dir, err := ioutil.TempDir("", "redo-openSelectedFiles") - require.Nil(t, err) - defer os.RemoveAll(dir) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - cfg := &writer.FileWriterConfig{ - MaxLogSize: 100000, - Dir: dir, - } - fileName := fmt.Sprintf("%s_%s_%d_%s_%d%s", "cp", "test-cf", time.Now().Unix(), common.DefaultDDLLogFileType, 11, common.LogEXT+common.TmpEXT) - w, err := writer.NewWriter(ctx, cfg, writer.WithLogFileName(func() string { - return fileName - })) - require.Nil(t, err) - log := &model.RedoLog{ - RedoRow: &model.RedoRowChangedEvent{Row: &model.RowChangedEvent{CommitTs: 11}}, - } - data, err := log.MarshalMsg(nil) - require.Nil(t, err) - _, err = w.Write(data) - require.Nil(t, err) - log = &model.RedoLog{ - RedoRow: &model.RedoRowChangedEvent{Row: &model.RowChangedEvent{CommitTs: 10}}, - } - data, err = log.MarshalMsg(nil) - require.Nil(t, err) - _, err = w.Write(data) - require.Nil(t, err) - err = w.Close() - require.Nil(t, err) - path := filepath.Join(cfg.Dir, fileName) - f, err := os.Open(path) - require.Nil(t, err) - - // no data, wil not open - fileName = fmt.Sprintf("%s_%s_%d_%s_%d%s", "cp", "test-cf11", time.Now().Unix(), common.DefaultDDLLogFileType, 10, common.LogEXT) - path = filepath.Join(dir, fileName) - _, err = os.Create(path) - require.Nil(t, err) - - // SortLogEXT, wil open - fileName = fmt.Sprintf("%s_%s_%d_%s_%d%s", "cp", "test-cf111", time.Now().Unix(), common.DefaultDDLLogFileType, 10, common.LogEXT) + common.SortLogEXT - path = filepath.Join(dir, fileName) - f1, err := os.Create(path) - require.Nil(t, err) - - dir1, err := ioutil.TempDir("", "redo-openSelectedFiles1") - require.Nil(t, err) - defer os.RemoveAll(dir1) //nolint:errcheck - fileName = fmt.Sprintf("%s_%s_%d_%s_%d%s", "cp", "test-cf", time.Now().Unix(), common.DefaultDDLLogFileType, 11, common.LogEXT+"test") - path = filepath.Join(dir1, fileName) - _, err = os.Create(path) - require.Nil(t, err) - - type arg struct { - dir, fixedName string - startTs uint64 - } - - tests := []struct { - name string - args arg - wantRet []io.ReadCloser - wantErr string - }{ - { - name: "dir not exist", - args: arg{ - dir: dir + "test", - fixedName: common.DefaultDDLLogFileType, - startTs: 0, - }, - wantErr: ".*CDC:ErrRedoFileOp*.", - }, - { - name: "happy", - args: arg{ - dir: dir, - fixedName: common.DefaultDDLLogFileType, - startTs: 0, - }, - wantRet: []io.ReadCloser{f, f1}, - }, - { - name: "wrong ts", - args: arg{ - dir: dir, - fixedName: common.DefaultDDLLogFileType, - startTs: 12, - }, - wantRet: []io.ReadCloser{f}, - }, - { - name: "wrong fixedName", - args: arg{ - dir: dir, - fixedName: common.DefaultDDLLogFileType + "test", - startTs: 0, - }, - }, - { - name: "wrong ext", - args: arg{ - dir: dir1, - fixedName: common.DefaultDDLLogFileType, - startTs: 0, - }, - }, - } - - for _, tt := range tests { - ret, err := openSelectedFiles(ctx, tt.args.dir, tt.args.fixedName, tt.args.startTs, 100) - if tt.wantErr == "" { - require.Nil(t, err, tt.name) - require.Equal(t, len(tt.wantRet), len(ret), tt.name) - for _, closer := range tt.wantRet { - name := closer.(*os.File).Name() - if filepath.Ext(name) != common.SortLogEXT { - name += common.SortLogEXT - } - contains := false - for _, r := range ret { - if r.(*os.File).Name() == name { - contains = true - break - } - } - require.Equal(t, true, contains, tt.name) - } - var preTs uint64 = 0 - for _, r := range ret { - r := &reader{ - br: bufio.NewReader(r), - fileName: r.(*os.File).Name(), - closer: r, - } - for { - rl := &model.RedoLog{} - err := r.Read(rl) - if err == io.EOF { - break - } - require.Greater(t, rl.RedoRow.Row.CommitTs, preTs, tt.name) - preTs = rl.RedoRow.Row.CommitTs - } - } - } else { - require.Regexp(t, tt.wantErr, err.Error(), tt.name) - } - } - time.Sleep(1001 * time.Millisecond) -} diff --git a/cdc/cdc/redo/reader/mock_RedoLogReader.go b/cdc/cdc/redo/reader/mock_RedoLogReader.go deleted file mode 100644 index a208967f..00000000 --- a/cdc/cdc/redo/reader/mock_RedoLogReader.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -// Code generated by mockery v0.0.0-dev. DO NOT EDIT. - -package reader - -import ( - context "context" - - mock "github.com/stretchr/testify/mock" - model "github.com/tikv/migration/cdc/cdc/model" -) - -// MockRedoLogReader is an autogenerated mock type for the RedoLogReader type -type MockRedoLogReader struct { - mock.Mock -} - -// Close provides a mock function with given fields: -func (_m *MockRedoLogReader) Close() error { - ret := _m.Called() - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// ReadMeta provides a mock function with given fields: ctx -func (_m *MockRedoLogReader) ReadMeta(ctx context.Context) (uint64, uint64, error) { - ret := _m.Called(ctx) - - var r0 uint64 - if rf, ok := ret.Get(0).(func(context.Context) uint64); ok { - r0 = rf(ctx) - } else { - r0 = ret.Get(0).(uint64) - } - - var r1 uint64 - if rf, ok := ret.Get(1).(func(context.Context) uint64); ok { - r1 = rf(ctx) - } else { - r1 = ret.Get(1).(uint64) - } - - var r2 error - if rf, ok := ret.Get(2).(func(context.Context) error); ok { - r2 = rf(ctx) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -// ReadNextDDL provides a mock function with given fields: ctx, maxNumberOfEvents -func (_m *MockRedoLogReader) ReadNextDDL(ctx context.Context, maxNumberOfEvents uint64) ([]*model.RedoDDLEvent, error) { - ret := _m.Called(ctx, maxNumberOfEvents) - - var r0 []*model.RedoDDLEvent - if rf, ok := ret.Get(0).(func(context.Context, uint64) []*model.RedoDDLEvent); ok { - r0 = rf(ctx, maxNumberOfEvents) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*model.RedoDDLEvent) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, uint64) error); ok { - r1 = rf(ctx, maxNumberOfEvents) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ReadNextLog provides a mock function with given fields: ctx, maxNumberOfEvents -func (_m *MockRedoLogReader) ReadNextLog(ctx context.Context, maxNumberOfEvents uint64) ([]*model.RedoRowChangedEvent, error) { - ret := _m.Called(ctx, maxNumberOfEvents) - - var r0 []*model.RedoRowChangedEvent - if rf, ok := ret.Get(0).(func(context.Context, uint64) []*model.RedoRowChangedEvent); ok { - r0 = rf(ctx, maxNumberOfEvents) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*model.RedoRowChangedEvent) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, uint64) error); ok { - r1 = rf(ctx, maxNumberOfEvents) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ResetReader provides a mock function with given fields: ctx, startTs, endTs -func (_m *MockRedoLogReader) ResetReader(ctx context.Context, startTs uint64, endTs uint64) error { - ret := _m.Called(ctx, startTs, endTs) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, uint64, uint64) error); ok { - r0 = rf(ctx, startTs, endTs) - } else { - r0 = ret.Error(0) - } - - return r0 -} diff --git a/cdc/cdc/redo/reader/mock_fileReader.go b/cdc/cdc/redo/reader/mock_fileReader.go deleted file mode 100644 index e38a5566..00000000 --- a/cdc/cdc/redo/reader/mock_fileReader.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -// Code generated by mockery v0.0.0-dev. DO NOT EDIT. - -package reader - -import ( - mock "github.com/stretchr/testify/mock" - model "github.com/tikv/migration/cdc/cdc/model" -) - -// mockFileReader is an autogenerated mock type for the fileReader type -type mockFileReader struct { - mock.Mock -} - -// Close provides a mock function with given fields: -func (_m *mockFileReader) Close() error { - ret := _m.Called() - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Read provides a mock function with given fields: log -func (_m *mockFileReader) Read(log *model.RedoLog) error { - ret := _m.Called(log) - - var r0 error - if rf, ok := ret.Get(0).(func(*model.RedoLog) error); ok { - r0 = rf(log) - } else { - r0 = ret.Error(0) - } - - return r0 -} diff --git a/cdc/cdc/redo/reader/reader.go b/cdc/cdc/redo/reader/reader.go deleted file mode 100644 index 895fed03..00000000 --- a/cdc/cdc/redo/reader/reader.go +++ /dev/null @@ -1,462 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package reader - -import ( - "container/heap" - "context" - "io" - "io/ioutil" - "net/url" - "os" - "path/filepath" - "sync" - - "github.com/pingcap/errors" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/redo/common" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "go.uber.org/multierr" -) - -//go:generate mockery --name=RedoLogReader --inpackage -// RedoLogReader is a reader abstraction for redo log storage layer -type RedoLogReader interface { - io.Closer - - // ResetReader setup the reader boundary - ResetReader(ctx context.Context, startTs, endTs uint64) error - - // ReadNextLog reads up to `maxNumberOfMessages` messages from current cursor. - // The returned redo logs sorted by commit-ts - ReadNextLog(ctx context.Context, maxNumberOfEvents uint64) ([]*model.RedoRowChangedEvent, error) - - // ReadNextDDL reads `maxNumberOfDDLs` ddl events from redo logs from current cursor - ReadNextDDL(ctx context.Context, maxNumberOfEvents uint64) ([]*model.RedoDDLEvent, error) - - // ReadMeta reads meta from redo logs and returns the latest checkpointTs and resolvedTs - ReadMeta(ctx context.Context) (checkpointTs, resolvedTs uint64, err error) -} - -// LogReaderConfig is the config for LogReader -type LogReaderConfig struct { - // Dir is the folder contains the redo logs need to apply when OP environment or - // the folder used to download redo logs to if s3 enabled - Dir string - S3Storage bool - // S3URI should be like S3URI="s3://logbucket/test-changefeed?endpoint=http://$S3_ENDPOINT/" - S3URI url.URL - // WorkerNums is the num of workers used to sort the log file to sorted file, - // will load the file to memory first then write the sorted file to disk - // the memory used is WorkerNums * defaultMaxLogSize (64 * megabyte) total - WorkerNums int - startTs uint64 - endTs uint64 -} - -// LogReader implement RedoLogReader interface -type LogReader struct { - cfg *LogReaderConfig - rowReader []fileReader - ddlReader []fileReader - rowHeap logHeap - ddlHeap logHeap - meta *common.LogMeta - rowLock sync.Mutex - ddlLock sync.Mutex - metaLock sync.Mutex - sync.Mutex -} - -// NewLogReader creates a LogReader instance. Need the client to guarantee only one LogReader per changefeed -// currently support rewind operation by ResetReader api -// if s3 will download logs first, if OP environment need fetch the redo logs to local dir first -func NewLogReader(ctx context.Context, cfg *LogReaderConfig) (*LogReader, error) { - if cfg == nil { - return nil, cerror.WrapError(cerror.ErrRedoConfigInvalid, errors.New("LogReaderConfig can not be nil")) - } - - logReader := &LogReader{ - cfg: cfg, - } - if cfg.S3Storage { - s3storage, err := common.InitS3storage(ctx, cfg.S3URI) - if err != nil { - return nil, err - } - // remove logs in local dir first, if have logs left belongs to previous changefeed with the same name may have error when apply logs - err = os.RemoveAll(cfg.Dir) - if err != nil { - return nil, cerror.WrapError(cerror.ErrRedoFileOp, err) - } - err = downLoadToLocal(ctx, cfg.Dir, s3storage, common.DefaultMetaFileType) - if err != nil { - return nil, cerror.WrapError(cerror.ErrRedoDownloadFailed, err) - } - } - return logReader, nil -} - -// ResetReader implement ResetReader interface -func (l *LogReader) ResetReader(ctx context.Context, startTs, endTs uint64) error { - select { - case <-ctx.Done(): - return errors.Trace(ctx.Err()) - default: - } - - if l.meta == nil { - _, _, err := l.ReadMeta(ctx) - if err != nil { - return err - } - } - if startTs > endTs || startTs > l.meta.ResolvedTs || endTs <= l.meta.CheckPointTs { - return errors.Errorf( - "startTs, endTs (%d, %d] should match the boundary: (%d, %d]", - startTs, endTs, l.meta.CheckPointTs, l.meta.ResolvedTs) - } - return l.setUpReader(ctx, startTs, endTs) -} - -func (l *LogReader) setUpReader(ctx context.Context, startTs, endTs uint64) error { - l.Lock() - defer l.Unlock() - - var errs error - errs = multierr.Append(errs, l.setUpRowReader(ctx, startTs, endTs)) - errs = multierr.Append(errs, l.setUpDDLReader(ctx, startTs, endTs)) - - return errs -} - -func (l *LogReader) setUpRowReader(ctx context.Context, startTs, endTs uint64) error { - l.rowLock.Lock() - defer l.rowLock.Unlock() - - err := l.closeRowReader() - if err != nil { - return err - } - - rowCfg := &readerConfig{ - dir: l.cfg.Dir, - fileType: common.DefaultRowLogFileType, - startTs: startTs, - endTs: endTs, - s3Storage: l.cfg.S3Storage, - s3URI: l.cfg.S3URI, - workerNums: l.cfg.WorkerNums, - } - l.rowReader, err = newReader(ctx, rowCfg) - if err != nil { - return err - } - - l.rowHeap = logHeap{} - l.cfg.startTs = startTs - l.cfg.endTs = endTs - return nil -} - -func (l *LogReader) setUpDDLReader(ctx context.Context, startTs, endTs uint64) error { - l.ddlLock.Lock() - defer l.ddlLock.Unlock() - - err := l.closeDDLReader() - if err != nil { - return err - } - - ddlCfg := &readerConfig{ - dir: l.cfg.Dir, - fileType: common.DefaultDDLLogFileType, - startTs: startTs, - endTs: endTs, - s3Storage: l.cfg.S3Storage, - s3URI: l.cfg.S3URI, - workerNums: l.cfg.WorkerNums, - } - l.ddlReader, err = newReader(ctx, ddlCfg) - if err != nil { - return err - } - - l.ddlHeap = logHeap{} - l.cfg.startTs = startTs - l.cfg.endTs = endTs - return nil -} - -// ReadNextLog implement ReadNextLog interface -func (l *LogReader) ReadNextLog(ctx context.Context, maxNumberOfEvents uint64) ([]*model.RedoRowChangedEvent, error) { - select { - case <-ctx.Done(): - return nil, errors.Trace(ctx.Err()) - default: - } - - l.rowLock.Lock() - defer l.rowLock.Unlock() - - // init heap - if l.rowHeap.Len() == 0 { - for i := 0; i < len(l.rowReader); i++ { - rl := &model.RedoLog{} - err := l.rowReader[i].Read(rl) - if err != nil { - if err != io.EOF { - return nil, err - } - continue - } - - ld := &logWithIdx{ - data: rl, - idx: i, - } - l.rowHeap = append(l.rowHeap, ld) - } - heap.Init(&l.rowHeap) - } - - ret := []*model.RedoRowChangedEvent{} - var i uint64 - for l.rowHeap.Len() != 0 && i < maxNumberOfEvents { - item := heap.Pop(&l.rowHeap).(*logWithIdx) - if item.data.RedoRow != nil && item.data.RedoRow.Row != nil && - // by design only data (startTs,endTs] is needed, so filter out data may beyond the boundary - item.data.RedoRow.Row.CommitTs > l.cfg.startTs && - item.data.RedoRow.Row.CommitTs <= l.cfg.endTs { - ret = append(ret, item.data.RedoRow) - i++ - } - - rl := &model.RedoLog{} - err := l.rowReader[item.idx].Read(rl) - if err != nil { - if err != io.EOF { - return nil, err - } - continue - } - - ld := &logWithIdx{ - data: rl, - idx: item.idx, - } - heap.Push(&l.rowHeap, ld) - } - - return ret, nil -} - -// ReadNextDDL implement ReadNextDDL interface -func (l *LogReader) ReadNextDDL(ctx context.Context, maxNumberOfEvents uint64) ([]*model.RedoDDLEvent, error) { - select { - case <-ctx.Done(): - return nil, errors.Trace(ctx.Err()) - default: - } - - l.ddlLock.Lock() - defer l.ddlLock.Unlock() - - // init heap - if l.ddlHeap.Len() == 0 { - for i := 0; i < len(l.ddlReader); i++ { - rl := &model.RedoLog{} - err := l.ddlReader[i].Read(rl) - if err != nil { - if err != io.EOF { - return nil, err - } - continue - } - - ld := &logWithIdx{ - data: rl, - idx: i, - } - l.ddlHeap = append(l.ddlHeap, ld) - } - heap.Init(&l.ddlHeap) - } - - ret := []*model.RedoDDLEvent{} - var i uint64 - for l.ddlHeap.Len() != 0 && i < maxNumberOfEvents { - item := heap.Pop(&l.ddlHeap).(*logWithIdx) - if item.data.RedoDDL != nil && item.data.RedoDDL.DDL != nil && - // by design only data (startTs,endTs] is needed, so filter out data may beyond the boundary - item.data.RedoDDL.DDL.CommitTs > l.cfg.startTs && - item.data.RedoDDL.DDL.CommitTs <= l.cfg.endTs { - ret = append(ret, item.data.RedoDDL) - i++ - } - - rl := &model.RedoLog{} - err := l.ddlReader[item.idx].Read(rl) - if err != nil { - if err != io.EOF { - return nil, err - } - continue - } - - ld := &logWithIdx{ - data: rl, - idx: item.idx, - } - heap.Push(&l.ddlHeap, ld) - } - - return ret, nil -} - -// ReadMeta implement ReadMeta interface -func (l *LogReader) ReadMeta(ctx context.Context) (checkpointTs, resolvedTs uint64, err error) { - select { - case <-ctx.Done(): - return 0, 0, errors.Trace(ctx.Err()) - default: - } - - l.metaLock.Lock() - defer l.metaLock.Unlock() - - if l.meta != nil { - return l.meta.CheckPointTs, l.meta.ResolvedTs, nil - } - - files, err := ioutil.ReadDir(l.cfg.Dir) - if err != nil { - return 0, 0, cerror.WrapError(cerror.ErrRedoFileOp, errors.Annotate(err, "can't read log file directory")) - } - - haveMeta := false - metaList := map[uint64]*common.LogMeta{} - var maxCheckPointTs uint64 - for _, file := range files { - if filepath.Ext(file.Name()) == common.MetaEXT { - path := filepath.Join(l.cfg.Dir, file.Name()) - fileData, err := os.ReadFile(path) - if err != nil { - return 0, 0, cerror.WrapError(cerror.ErrRedoFileOp, err) - } - - l.meta = &common.LogMeta{} - _, err = l.meta.UnmarshalMsg(fileData) - if err != nil { - return 0, 0, cerror.WrapError(cerror.ErrRedoFileOp, err) - } - - if !haveMeta { - haveMeta = true - } - metaList[l.meta.CheckPointTs] = l.meta - // since checkPointTs is guaranteed to increase always - if l.meta.CheckPointTs > maxCheckPointTs { - maxCheckPointTs = l.meta.CheckPointTs - } - } - } - if !haveMeta { - return 0, 0, cerror.ErrRedoMetaFileNotFound.GenWithStackByArgs(l.cfg.Dir) - } - - l.meta = metaList[maxCheckPointTs] - return l.meta.CheckPointTs, l.meta.ResolvedTs, nil -} - -func (l *LogReader) closeRowReader() error { - var errs error - for _, r := range l.rowReader { - errs = multierr.Append(errs, r.Close()) - } - return errs -} - -func (l *LogReader) closeDDLReader() error { - var errs error - for _, r := range l.ddlReader { - errs = multierr.Append(errs, r.Close()) - } - return errs -} - -// Close the backing file readers -func (l *LogReader) Close() error { - if l == nil { - return nil - } - - var errs error - - l.rowLock.Lock() - errs = multierr.Append(errs, l.closeRowReader()) - l.rowLock.Unlock() - - l.ddlLock.Lock() - errs = multierr.Append(errs, l.closeDDLReader()) - l.ddlLock.Unlock() - return errs -} - -type logWithIdx struct { - idx int - data *model.RedoLog -} - -type logHeap []*logWithIdx - -func (h logHeap) Len() int { - return len(h) -} - -func (h logHeap) Less(i, j int) bool { - if h[i].data.Type == model.RedoLogTypeDDL { - if h[i].data.RedoDDL == nil || h[i].data.RedoDDL.DDL == nil { - return true - } - if h[j].data.RedoDDL == nil || h[j].data.RedoDDL.DDL == nil { - return false - } - return h[i].data.RedoDDL.DDL.CommitTs < h[j].data.RedoDDL.DDL.CommitTs - } - - if h[i].data.RedoRow == nil || h[i].data.RedoRow.Row == nil { - return true - } - if h[j].data.RedoRow == nil || h[j].data.RedoRow.Row == nil { - return false - } - return h[i].data.RedoRow.Row.CommitTs < h[j].data.RedoRow.Row.CommitTs -} - -func (h logHeap) Swap(i, j int) { - h[i], h[j] = h[j], h[i] -} - -func (h *logHeap) Push(x interface{}) { - *h = append(*h, x.(*logWithIdx)) -} - -func (h *logHeap) Pop() interface{} { - old := *h - n := len(old) - x := old[n-1] - *h = old[0 : n-1] - return x -} diff --git a/cdc/cdc/redo/reader/reader_test.go b/cdc/cdc/redo/reader/reader_test.go deleted file mode 100644 index b828e859..00000000 --- a/cdc/cdc/redo/reader/reader_test.go +++ /dev/null @@ -1,716 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package reader - -import ( - "context" - "fmt" - "io" - "io/ioutil" - "net/url" - "os" - "path/filepath" - "testing" - "time" - - "github.com/golang/mock/gomock" - "github.com/pingcap/errors" - mockstorage "github.com/pingcap/tidb/br/pkg/mock/storage" - "github.com/pingcap/tidb/br/pkg/storage" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/redo/common" - "github.com/tikv/migration/cdc/cdc/redo/writer" - "go.uber.org/multierr" -) - -func TestNewLogReader(t *testing.T) { - _, err := NewLogReader(context.Background(), nil) - require.NotNil(t, err) - - _, err = NewLogReader(context.Background(), &LogReaderConfig{}) - require.Nil(t, err) - - dir, err := ioutil.TempDir("", "redo-NewLogReader") - require.Nil(t, err) - defer os.RemoveAll(dir) - - s3URI, err := url.Parse("s3://logbucket/test-changefeed?endpoint=http://111/") - require.Nil(t, err) - - origin := common.InitS3storage - defer func() { - common.InitS3storage = origin - }() - controller := gomock.NewController(t) - mockStorage := mockstorage.NewMockExternalStorage(controller) - // no file to download - mockStorage.EXPECT().WalkDir(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) - common.InitS3storage = func(ctx context.Context, uri url.URL) (storage.ExternalStorage, error) { - return mockStorage, nil - } - - // after init should rm the dir - _, err = NewLogReader(context.Background(), &LogReaderConfig{ - S3Storage: true, - Dir: dir, - S3URI: *s3URI, - }) - require.Nil(t, err) - _, err = os.Stat(dir) - require.True(t, os.IsNotExist(err)) -} - -func TestLogReaderResetReader(t *testing.T) { - dir, err := ioutil.TempDir("", "redo-ResetReader") - require.Nil(t, err) - defer os.RemoveAll(dir) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - cfg := &writer.FileWriterConfig{ - MaxLogSize: 100000, - Dir: dir, - } - fileName := fmt.Sprintf("%s_%s_%d_%s_%d%s", "cp", "test-cf100", time.Now().Unix(), common.DefaultDDLLogFileType, 100, common.LogEXT) - w, err := writer.NewWriter(ctx, cfg, writer.WithLogFileName(func() string { - return fileName - })) - require.Nil(t, err) - log := &model.RedoLog{ - RedoRow: &model.RedoRowChangedEvent{Row: &model.RowChangedEvent{CommitTs: 11}}, - } - data, err := log.MarshalMsg(nil) - require.Nil(t, err) - _, err = w.Write(data) - require.Nil(t, err) - err = w.Close() - require.Nil(t, err) - - path := filepath.Join(dir, fileName) - f, err := os.Open(path) - require.Nil(t, err) - - fileName = fmt.Sprintf("%s_%s_%d_%s_%d%s", "cp", "test-cf10", time.Now().Unix(), common.DefaultRowLogFileType, 10, common.LogEXT) - w, err = writer.NewWriter(ctx, cfg, writer.WithLogFileName(func() string { - return fileName - })) - require.Nil(t, err) - log = &model.RedoLog{ - RedoRow: &model.RedoRowChangedEvent{Row: &model.RowChangedEvent{CommitTs: 11}}, - } - data, err = log.MarshalMsg(nil) - require.Nil(t, err) - _, err = w.Write(data) - require.Nil(t, err) - err = w.Close() - require.Nil(t, err) - path = filepath.Join(dir, fileName) - f1, err := os.Open(path) - require.Nil(t, err) - - type arg struct { - ctx context.Context - startTs, endTs uint64 - resolvedTs, checkPointTs uint64 - } - tests := []struct { - name string - args arg - readerErr error - wantErr string - wantStartTs, wantEndTs uint64 - rowFleName string - ddlFleName string - }{ - { - name: "happy", - args: arg{ - ctx: context.Background(), - startTs: 1, - endTs: 101, - checkPointTs: 0, - resolvedTs: 200, - }, - wantStartTs: 1, - wantEndTs: 101, - rowFleName: f1.Name(), - ddlFleName: f.Name(), - }, - { - name: "context cancel", - args: arg{ - ctx: context.Background(), - startTs: 1, - endTs: 101, - checkPointTs: 0, - resolvedTs: 200, - }, - wantErr: context.Canceled.Error(), - }, - { - name: "invalid ts", - args: arg{ - ctx: context.Background(), - startTs: 1, - endTs: 0, - checkPointTs: 0, - resolvedTs: 200, - }, - wantErr: ".*should match the boundary*.", - }, - { - name: "invalid ts", - args: arg{ - ctx: context.Background(), - startTs: 201, - endTs: 10, - checkPointTs: 0, - resolvedTs: 200, - }, - wantErr: ".*should match the boundary*.", - }, - { - name: "reader close err", - args: arg{ - ctx: context.Background(), - startTs: 1, - endTs: 10, - checkPointTs: 0, - resolvedTs: 200, - }, - wantErr: "err", - readerErr: errors.New("err"), - }, - } - - for _, tt := range tests { - mockReader := &mockFileReader{} - mockReader.On("Close").Return(tt.readerErr) - r := &LogReader{ - cfg: &LogReaderConfig{Dir: dir}, - rowReader: []fileReader{mockReader}, - ddlReader: []fileReader{mockReader}, - meta: &common.LogMeta{CheckPointTs: tt.args.checkPointTs, ResolvedTs: tt.args.resolvedTs}, - } - - if tt.name == "context cancel" { - ctx, cancel := context.WithCancel(context.Background()) - cancel() - tt.args.ctx = ctx - } else { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - tt.args.ctx = ctx - } - err := r.ResetReader(tt.args.ctx, tt.args.startTs, tt.args.endTs) - if tt.wantErr != "" { - require.Regexp(t, tt.wantErr, err, tt.name) - } else { - require.Nil(t, err, tt.name) - mockReader.AssertNumberOfCalls(t, "Close", 2) - require.Equal(t, tt.rowFleName+common.SortLogEXT, r.rowReader[0].(*reader).fileName, tt.name) - require.Equal(t, tt.ddlFleName+common.SortLogEXT, r.ddlReader[0].(*reader).fileName, tt.name) - require.Equal(t, tt.wantStartTs, r.cfg.startTs, tt.name) - require.Equal(t, tt.wantEndTs, r.cfg.endTs, tt.name) - - } - } - time.Sleep(1001 * time.Millisecond) -} - -func TestLogReaderReadMeta(t *testing.T) { - dir, err := ioutil.TempDir("", "redo-ReadMeta") - require.Nil(t, err) - defer os.RemoveAll(dir) - - fileName := fmt.Sprintf("%s_%s_%d_%s%s", "cp", "test-changefeed", time.Now().Unix(), common.DefaultMetaFileType, common.MetaEXT) - path := filepath.Join(dir, fileName) - f, err := os.Create(path) - require.Nil(t, err) - meta := &common.LogMeta{ - CheckPointTs: 11, - ResolvedTs: 22, - } - data, err := meta.MarshalMsg(nil) - require.Nil(t, err) - _, err = f.Write(data) - require.Nil(t, err) - - fileName = fmt.Sprintf("%s_%s_%d_%s%s", "cp1", "test-changefeed", time.Now().Unix(), common.DefaultMetaFileType, common.MetaEXT) - path = filepath.Join(dir, fileName) - f, err = os.Create(path) - require.Nil(t, err) - meta = &common.LogMeta{ - CheckPointTs: 111, - ResolvedTs: 21, - } - data, err = meta.MarshalMsg(nil) - require.Nil(t, err) - _, err = f.Write(data) - require.Nil(t, err) - - dir1, err := ioutil.TempDir("", "redo-NoReadMeta") - require.Nil(t, err) - defer os.RemoveAll(dir1) - - tests := []struct { - name string - dir string - wantCheckPointTs, wantResolvedTs uint64 - wantErr string - }{ - { - name: "happy", - dir: dir, - wantCheckPointTs: meta.CheckPointTs, - wantResolvedTs: meta.ResolvedTs, - }, - { - name: "no meta file", - dir: dir1, - wantErr: ".*no redo meta file found in dir*.", - }, - { - name: "wrong dir", - dir: "xxx", - wantErr: ".*can't read log file directory*.", - }, - { - name: "context cancel", - dir: dir, - wantCheckPointTs: meta.CheckPointTs, - wantResolvedTs: meta.ResolvedTs, - wantErr: context.Canceled.Error(), - }, - } - for _, tt := range tests { - l := &LogReader{ - cfg: &LogReaderConfig{ - Dir: tt.dir, - }, - } - ctx := context.Background() - if tt.name == "context cancel" { - ctx1, cancel := context.WithCancel(context.Background()) - cancel() - ctx = ctx1 - } - cts, rts, err := l.ReadMeta(ctx) - if tt.wantErr != "" { - require.Regexp(t, tt.wantErr, err, tt.name) - } else { - require.Nil(t, err, tt.name) - require.Equal(t, tt.wantCheckPointTs, cts, tt.name) - require.Equal(t, tt.wantResolvedTs, rts, tt.name) - } - } -} - -func TestLogReaderReadNextLog(t *testing.T) { - type arg struct { - ctx context.Context - maxNum uint64 - } - tests := []struct { - name string - args arg - wantErr error - readerErr error - readerErr1 error - readerRet *model.RedoLog - readerRet1 *model.RedoLog - }{ - { - name: "happy", - args: arg{ - ctx: context.Background(), - maxNum: 3, - }, - readerRet: &model.RedoLog{ - RedoRow: &model.RedoRowChangedEvent{ - Row: &model.RowChangedEvent{ - CommitTs: 15, - RowID: 1, - }, - }, - }, - readerRet1: &model.RedoLog{ - RedoRow: &model.RedoRowChangedEvent{ - Row: &model.RowChangedEvent{ - CommitTs: 6, - RowID: 2, - }, - }, - }, - }, - { - name: "context cancel", - args: arg{ - ctx: context.Background(), - maxNum: 3, - }, - readerRet: &model.RedoLog{ - RedoRow: &model.RedoRowChangedEvent{ - Row: &model.RowChangedEvent{ - CommitTs: 5, - RowID: 1, - }, - }, - }, - readerRet1: &model.RedoLog{ - RedoRow: &model.RedoRowChangedEvent{ - Row: &model.RowChangedEvent{ - CommitTs: 6, - RowID: 2, - }, - }, - }, - wantErr: context.Canceled, - }, - { - name: "happy1", - args: arg{ - ctx: context.Background(), - maxNum: 3, - }, - readerRet: &model.RedoLog{ - RedoRow: &model.RedoRowChangedEvent{ - Row: &model.RowChangedEvent{ - CommitTs: 1, - RowID: 1, - }, - }, - }, - readerRet1: &model.RedoLog{ - RedoRow: &model.RedoRowChangedEvent{ - Row: &model.RowChangedEvent{ - CommitTs: 6, - RowID: 2, - }, - }, - }, - }, - { - name: "io.EOF err", - args: arg{ - ctx: context.Background(), - maxNum: 3, - }, - readerRet: &model.RedoLog{ - RedoRow: &model.RedoRowChangedEvent{ - Row: &model.RowChangedEvent{ - CommitTs: 5, - RowID: 1, - }, - }, - }, - readerRet1: &model.RedoLog{ - RedoRow: &model.RedoRowChangedEvent{ - Row: &model.RowChangedEvent{ - CommitTs: 6, - RowID: 2, - }, - }, - }, - readerErr: io.EOF, - }, - { - name: "err", - args: arg{ - ctx: context.Background(), - maxNum: 3, - }, - readerRet: &model.RedoLog{ - RedoRow: &model.RedoRowChangedEvent{ - Row: &model.RowChangedEvent{ - CommitTs: 5, - RowID: 1, - }, - }, - }, - readerRet1: &model.RedoLog{ - RedoRow: &model.RedoRowChangedEvent{ - Row: &model.RowChangedEvent{ - CommitTs: 6, - RowID: 2, - }, - }, - }, - readerErr: errors.New("xx"), - readerErr1: errors.New("xx"), - wantErr: errors.New("xx"), - }, - } - - for _, tt := range tests { - mockReader := &mockFileReader{} - mockReader.On("Read", mock.Anything).Return(tt.readerErr).Run(func(args mock.Arguments) { - arg := args.Get(0).(*model.RedoLog) - arg.RedoRow = tt.readerRet.RedoRow - arg.Type = model.RedoLogTypeRow - }).Times(int(tt.args.maxNum)) - mockReader.On("Read", mock.Anything).Return(io.EOF).Once() - - mockReader1 := &mockFileReader{} - mockReader1.On("Read", mock.Anything).Return(tt.readerErr1).Run(func(args mock.Arguments) { - arg := args.Get(0).(*model.RedoLog) - arg.RedoRow = tt.readerRet1.RedoRow - arg.Type = model.RedoLogTypeRow - }) - - l := &LogReader{ - rowReader: []fileReader{mockReader1, mockReader}, - rowHeap: logHeap{}, - cfg: &LogReaderConfig{ - startTs: 1, - endTs: 10, - }, - } - if tt.name == "context cancel" { - ctx1, cancel := context.WithCancel(context.Background()) - cancel() - tt.args.ctx = ctx1 - } - ret, err := l.ReadNextLog(tt.args.ctx, tt.args.maxNum) - if tt.wantErr != nil { - require.True(t, errors.ErrorEqual(tt.wantErr, err), tt.name) - require.Equal(t, 0, len(ret), tt.name) - } else { - require.Nil(t, err, tt.name) - require.EqualValues(t, tt.args.maxNum, len(ret), tt.name) - for i := 0; i < int(tt.args.maxNum); i++ { - if tt.name == "io.EOF err" { - require.Equal(t, ret[i].Row.CommitTs, tt.readerRet1.RedoRow.Row.CommitTs, tt.name) - continue - } - if tt.name == "happy1" { - require.Equal(t, ret[i].Row.CommitTs, tt.readerRet1.RedoRow.Row.CommitTs, tt.name) - continue - } - require.Equal(t, ret[i].Row.CommitTs, tt.readerRet1.RedoRow.Row.CommitTs, tt.name) - } - } - } -} - -func TestLogReaderReadNexDDL(t *testing.T) { - type arg struct { - ctx context.Context - maxNum uint64 - } - tests := []struct { - name string - args arg - wantErr error - readerErr error - readerErr1 error - readerRet *model.RedoLog - readerRet1 *model.RedoLog - }{ - { - name: "happy", - args: arg{ - ctx: context.Background(), - maxNum: 3, - }, - readerRet: &model.RedoLog{ - RedoDDL: &model.RedoDDLEvent{ - DDL: &model.DDLEvent{ - CommitTs: 15, - }, - }, - }, - readerRet1: &model.RedoLog{ - RedoDDL: &model.RedoDDLEvent{ - DDL: &model.DDLEvent{ - CommitTs: 6, - }, - }, - }, - }, - { - name: "context cancel", - args: arg{ - ctx: context.Background(), - maxNum: 3, - }, - readerRet: &model.RedoLog{ - RedoDDL: &model.RedoDDLEvent{ - DDL: &model.DDLEvent{ - CommitTs: 5, - }, - }, - }, - readerRet1: &model.RedoLog{ - RedoDDL: &model.RedoDDLEvent{ - DDL: &model.DDLEvent{ - CommitTs: 6, - }, - }, - }, - wantErr: context.Canceled, - }, - { - name: "happy1", - args: arg{ - ctx: context.Background(), - maxNum: 3, - }, - readerRet: &model.RedoLog{ - RedoDDL: &model.RedoDDLEvent{ - DDL: &model.DDLEvent{ - CommitTs: 1, - }, - }, - }, - readerRet1: &model.RedoLog{ - RedoDDL: &model.RedoDDLEvent{ - DDL: &model.DDLEvent{ - CommitTs: 6, - }, - }, - }, - }, - { - name: "io.EOF err", - args: arg{ - ctx: context.Background(), - maxNum: 3, - }, - readerRet: &model.RedoLog{ - RedoDDL: &model.RedoDDLEvent{ - DDL: &model.DDLEvent{ - CommitTs: 5, - }, - }, - }, - readerRet1: &model.RedoLog{ - RedoDDL: &model.RedoDDLEvent{ - DDL: &model.DDLEvent{ - CommitTs: 6, - }, - }, - }, - readerErr: io.EOF, - }, - { - name: "err", - args: arg{ - ctx: context.Background(), - maxNum: 3, - }, - readerRet: &model.RedoLog{ - RedoDDL: &model.RedoDDLEvent{ - DDL: &model.DDLEvent{ - CommitTs: 5, - }, - }, - }, - readerRet1: &model.RedoLog{ - RedoDDL: &model.RedoDDLEvent{ - DDL: &model.DDLEvent{ - CommitTs: 6, - }, - }, - }, - readerErr: errors.New("xx"), - readerErr1: errors.New("xx"), - wantErr: errors.New("xx"), - }, - } - - for _, tt := range tests { - mockReader := &mockFileReader{} - mockReader.On("Read", mock.Anything).Return(tt.readerErr).Run(func(args mock.Arguments) { - arg := args.Get(0).(*model.RedoLog) - arg.RedoDDL = tt.readerRet.RedoDDL - arg.Type = model.RedoLogTypeDDL - }).Times(int(tt.args.maxNum)) - mockReader.On("Read", mock.Anything).Return(io.EOF).Once() - mockReader1 := &mockFileReader{} - mockReader1.On("Read", mock.Anything).Return(tt.readerErr1).Run(func(args mock.Arguments) { - arg := args.Get(0).(*model.RedoLog) - arg.RedoDDL = tt.readerRet1.RedoDDL - arg.Type = model.RedoLogTypeDDL - }) - - l := &LogReader{ - ddlReader: []fileReader{mockReader1, mockReader}, - ddlHeap: logHeap{}, - cfg: &LogReaderConfig{ - startTs: 1, - endTs: 10, - }, - } - if tt.name == "context cancel" { - ctx1, cancel := context.WithCancel(context.Background()) - cancel() - tt.args.ctx = ctx1 - } - ret, err := l.ReadNextDDL(tt.args.ctx, tt.args.maxNum) - if tt.wantErr != nil { - require.True(t, errors.ErrorEqual(tt.wantErr, err), tt.name) - require.Equal(t, 0, len(ret), tt.name) - } else { - require.Nil(t, err, tt.name) - require.EqualValues(t, tt.args.maxNum, len(ret), tt.name) - for i := 0; i < int(tt.args.maxNum); i++ { - if tt.name == "io.EOF err" { - require.Equal(t, ret[i].DDL.CommitTs, tt.readerRet1.RedoDDL.DDL.CommitTs, tt.name) - continue - } - if tt.name == "happy1" { - require.Equal(t, ret[i].DDL.CommitTs, tt.readerRet1.RedoDDL.DDL.CommitTs, tt.name) - continue - } - require.Equal(t, ret[i].DDL.CommitTs, tt.readerRet1.RedoDDL.DDL.CommitTs, tt.name) - } - } - } -} - -func TestLogReaderClose(t *testing.T) { - tests := []struct { - name string - wantErr error - err error - }{ - { - name: "happy", - }, - { - name: "err", - err: errors.New("xx"), - wantErr: multierr.Append(errors.New("xx"), errors.New("xx")), - }, - } - - for _, tt := range tests { - mockReader := &mockFileReader{} - mockReader.On("Close").Return(tt.err) - l := &LogReader{ - rowReader: []fileReader{mockReader}, - ddlReader: []fileReader{mockReader}, - } - err := l.Close() - mockReader.AssertNumberOfCalls(t, "Close", 2) - if tt.wantErr != nil { - require.True(t, errors.ErrorEqual(tt.wantErr, err), tt.name) - } else { - require.Nil(t, err, tt.name) - } - } -} diff --git a/cdc/cdc/redo/writer/blackhole_writer.go b/cdc/cdc/redo/writer/blackhole_writer.go deleted file mode 100644 index cda15c97..00000000 --- a/cdc/cdc/redo/writer/blackhole_writer.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package writer - -import ( - "context" - "sync" - - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" - "go.uber.org/zap" -) - -// blackHoleSink defines a blackHole storage, it receives events and persists -// without any latency -type blackHoleWriter struct { - tableRtsMap map[model.TableID]uint64 - tableRtsMu sync.RWMutex - resolvedTs uint64 - checkpointTs uint64 -} - -func (bs *blackHoleWriter) DeleteAllLogs(ctx context.Context) error { - return nil -} - -// NewBlackHoleWriter creates a blackHole writer -func NewBlackHoleWriter() *blackHoleWriter { - return &blackHoleWriter{ - tableRtsMap: make(map[model.TableID]uint64), - } -} - -func (bs *blackHoleWriter) WriteLog(_ context.Context, tableID model.TableID, logs []*model.RedoRowChangedEvent) (resolvedTs uint64, err error) { - bs.tableRtsMu.Lock() - defer bs.tableRtsMu.Unlock() - if len(logs) == 0 { - return bs.tableRtsMap[tableID], nil - } - resolvedTs = bs.tableRtsMap[tableID] - current := logs[len(logs)-1].Row.CommitTs - bs.tableRtsMap[tableID] = current - log.Debug("write row redo logs", zap.Int("count", len(logs)), - zap.Uint64("resolvedTs", resolvedTs), zap.Uint64("current", current)) - return -} - -func (bs *blackHoleWriter) FlushLog(_ context.Context, tableID model.TableID, resolvedTs uint64) error { - bs.tableRtsMu.Lock() - defer bs.tableRtsMu.Unlock() - bs.tableRtsMap[tableID] = resolvedTs - return nil -} - -func (bs *blackHoleWriter) SendDDL(_ context.Context, ddl *model.RedoDDLEvent) error { - log.Debug("send ddl event", zap.Any("ddl", ddl)) - return nil -} - -func (bs *blackHoleWriter) EmitResolvedTs(_ context.Context, ts uint64) error { - bs.resolvedTs = ts - return nil -} - -func (bs *blackHoleWriter) EmitCheckpointTs(_ context.Context, ts uint64) error { - bs.checkpointTs = ts - return nil -} - -func (bs *blackHoleWriter) GetCurrentResolvedTs(_ context.Context, tableIDs []int64) (map[int64]uint64, error) { - bs.tableRtsMu.RLock() - defer bs.tableRtsMu.RUnlock() - rtsMap := make(map[int64]uint64, len(bs.tableRtsMap)) - for _, tableID := range tableIDs { - rtsMap[tableID] = bs.tableRtsMap[tableID] - } - return rtsMap, nil -} - -func (bs *blackHoleWriter) Close() error { - return nil -} diff --git a/cdc/cdc/redo/writer/file.go b/cdc/cdc/redo/writer/file.go deleted file mode 100644 index c76f0719..00000000 --- a/cdc/cdc/redo/writer/file.go +++ /dev/null @@ -1,556 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// Copyright 2015 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package writer - -import ( - "context" - "encoding/binary" - "fmt" - "io" - "io/ioutil" - "net/url" - "os" - "path/filepath" - "sync" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/log" - "github.com/pingcap/tidb/br/pkg/storage" - "github.com/prometheus/client_golang/prometheus" - "github.com/tikv/migration/cdc/cdc/redo/common" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/uber-go/atomic" - pioutil "go.etcd.io/etcd/pkg/ioutil" - "go.uber.org/multierr" - "go.uber.org/zap" -) - -const ( - // pageBytes is the alignment for flushing records to the backing Writer. - // It should be a multiple of the minimum sector size so that log can safely - // distinguish between torn writes and ordinary data corruption. - pageBytes = 8 * common.MinSectorSize -) - -const ( - defaultFlushIntervalInMs = 1000 - defaultS3Timeout = 3 * time.Second -) - -var ( - // for easy testing, not set to const - megabyte int64 = 1024 * 1024 - defaultMaxLogSize = 64 * megabyte -) - -//go:generate mockery --name=fileWriter --inpackage -type fileWriter interface { - io.WriteCloser - flusher - - // AdvanceTs receive the commitTs in the event from caller - AdvanceTs(commitTs uint64) - // GC run gc to remove useless files base on the checkPointTs - GC(checkPointTs uint64) error - // IsRunning check the fileWriter status - IsRunning() bool -} - -type flusher interface { - Flush() error -} - -// FileWriterConfig is the configuration used by a Writer. -type FileWriterConfig struct { - Dir string - ChangeFeedID string - CaptureID string - FileType string - CreateTime time.Time - // MaxLogSize is the maximum size of log in megabyte, defaults to defaultMaxLogSize. - MaxLogSize int64 - FlushIntervalInMs int64 - S3Storage bool - S3URI url.URL -} - -// Option define the writerOptions -type Option func(writer *writerOptions) - -type writerOptions struct { - getLogFileName func() string -} - -// WithLogFileName provide the Option for fileName -func WithLogFileName(f func() string) Option { - return func(o *writerOptions) { - if f != nil { - o.getLogFileName = f - } - } -} - -// Writer is a redo log event Writer which writes redo log events to a file. -type Writer struct { - cfg *FileWriterConfig - op *writerOptions - // maxCommitTS is the max commitTS among the events in one log file - maxCommitTS atomic.Uint64 - // the ts used in file name - commitTS atomic.Uint64 - // the ts send with the event - eventCommitTS atomic.Uint64 - running atomic.Bool - gcRunning atomic.Bool - size int64 - file *os.File - bw *pioutil.PageWriter - uint64buf []byte - storage storage.ExternalStorage - sync.RWMutex - - metricFsyncDuration prometheus.Observer - metricFlushAllDuration prometheus.Observer - metricWriteBytes prometheus.Gauge -} - -// NewWriter return a file rotated writer, TODO: extract to a common rotate Writer -func NewWriter(ctx context.Context, cfg *FileWriterConfig, opts ...Option) (*Writer, error) { - if cfg == nil { - return nil, cerror.WrapError(cerror.ErrRedoConfigInvalid, errors.New("FileWriterConfig can not be nil")) - } - - if cfg.FlushIntervalInMs == 0 { - cfg.FlushIntervalInMs = defaultFlushIntervalInMs - } - cfg.MaxLogSize *= megabyte - if cfg.MaxLogSize == 0 { - cfg.MaxLogSize = defaultMaxLogSize - } - var s3storage storage.ExternalStorage - if cfg.S3Storage { - var err error - s3storage, err = common.InitS3storage(ctx, cfg.S3URI) - if err != nil { - return nil, err - } - } - - op := &writerOptions{} - for _, opt := range opts { - opt(op) - } - w := &Writer{ - cfg: cfg, - op: op, - uint64buf: make([]byte, 8), - storage: s3storage, - - metricFsyncDuration: redoFsyncDurationHistogram.WithLabelValues(cfg.CaptureID, cfg.ChangeFeedID), - metricFlushAllDuration: redoFlushAllDurationHistogram.WithLabelValues(cfg.CaptureID, cfg.ChangeFeedID), - metricWriteBytes: redoWriteBytesGauge.WithLabelValues(cfg.CaptureID, cfg.ChangeFeedID), - } - - w.running.Store(true) - go w.runFlushToDisk(ctx, cfg.FlushIntervalInMs) - - return w, nil -} - -func (w *Writer) runFlushToDisk(ctx context.Context, flushIntervalInMs int64) { - ticker := time.NewTicker(time.Duration(flushIntervalInMs) * time.Millisecond) - defer ticker.Stop() - - for { - if !w.IsRunning() { - return - } - - select { - case <-ctx.Done(): - err := w.Close() - if err != nil { - log.Error("runFlushToDisk close fail", zap.String("changefeedID", w.cfg.ChangeFeedID), zap.Error(err)) - } - case <-ticker.C: - err := w.Flush() - if err != nil { - log.Error("redo log flush fail", zap.String("changefeedID", w.cfg.ChangeFeedID), zap.Error(err)) - } - } - } -} - -// Write implement write interface -// TODO: more general api with fileName generated by caller -func (w *Writer) Write(rawData []byte) (int, error) { - w.Lock() - defer w.Unlock() - - writeLen := int64(len(rawData)) - if writeLen > w.cfg.MaxLogSize { - return 0, cerror.ErrFileSizeExceed.GenWithStackByArgs(writeLen, w.cfg.MaxLogSize) - } - - if w.file == nil { - if err := w.openOrNew(len(rawData)); err != nil { - return 0, err - } - } - - if w.size+writeLen > w.cfg.MaxLogSize { - if err := w.rotate(); err != nil { - return 0, err - } - } - if w.maxCommitTS.Load() < w.eventCommitTS.Load() { - w.maxCommitTS.Store(w.eventCommitTS.Load()) - } - // ref: https://github.com/etcd-io/etcd/pull/5250 - lenField, padBytes := encodeFrameSize(len(rawData)) - if err := w.writeUint64(lenField, w.uint64buf); err != nil { - return 0, err - } - - if padBytes != 0 { - rawData = append(rawData, make([]byte, padBytes)...) - } - - n, err := w.bw.Write(rawData) - w.metricWriteBytes.Add(float64(n)) - w.size += int64(n) - return n, err -} - -// AdvanceTs implement Advance interface -func (w *Writer) AdvanceTs(commitTs uint64) { - w.eventCommitTS.Store(commitTs) -} - -func (w *Writer) writeUint64(n uint64, buf []byte) error { - binary.LittleEndian.PutUint64(buf, n) - v, err := w.bw.Write(buf) - w.metricWriteBytes.Add(float64(v)) - - return err -} - -// the func uses code from etcd wal/encoder.go -// ref: https://github.com/etcd-io/etcd/pull/5250 -func encodeFrameSize(dataBytes int) (lenField uint64, padBytes int) { - lenField = uint64(dataBytes) - // force 8 byte alignment so length never gets a torn write - padBytes = (8 - (dataBytes % 8)) % 8 - if padBytes != 0 { - lenField |= uint64(0x80|padBytes) << 56 - } - return lenField, padBytes -} - -// Close implements fileWriter.Close. -func (w *Writer) Close() error { - w.Lock() - defer w.Unlock() - // always set to false when closed, since if having err may not get fixed just by retry - defer w.running.Store(false) - - if !w.IsRunning() { - return nil - } - - redoFlushAllDurationHistogram.DeleteLabelValues(w.cfg.CaptureID, w.cfg.ChangeFeedID) - redoFsyncDurationHistogram.DeleteLabelValues(w.cfg.CaptureID, w.cfg.ChangeFeedID) - redoWriteBytesGauge.DeleteLabelValues(w.cfg.CaptureID, w.cfg.ChangeFeedID) - - return w.close() -} - -// IsRunning implement IsRunning interface -func (w *Writer) IsRunning() bool { - return w.running.Load() -} - -func (w *Writer) isGCRunning() bool { - return w.gcRunning.Load() -} - -func (w *Writer) close() error { - if w.file == nil { - return nil - } - err := w.flushAll() - if err != nil { - return err - } - - // rename the file name from commitTs.log.tmp to maxCommitTS.log if closed safely - // after rename, the file name could be used for search, since the ts is the max ts for all events in the file. - w.commitTS.Store(w.maxCommitTS.Load()) - err = os.Rename(w.file.Name(), w.filePath()) - if err != nil { - return cerror.WrapError(cerror.ErrRedoFileOp, err) - } - - if w.cfg.S3Storage { - ctx, cancel := context.WithTimeout(context.Background(), defaultS3Timeout) - defer cancel() - - err = w.renameInS3(ctx, w.file.Name(), w.filePath()) - if err != nil { - return cerror.WrapError(cerror.ErrS3StorageAPI, err) - } - } - - err = w.file.Close() - w.file = nil - return cerror.WrapError(cerror.ErrRedoFileOp, err) -} - -func (w *Writer) renameInS3(ctx context.Context, oldPath, newPath string) error { - err := w.writeToS3(ctx, newPath) - if err != nil { - return cerror.WrapError(cerror.ErrS3StorageAPI, err) - } - return cerror.WrapError(cerror.ErrS3StorageAPI, w.storage.DeleteFile(ctx, filepath.Base(oldPath))) -} - -func (w *Writer) getLogFileName() string { - if w.op != nil && w.op.getLogFileName != nil { - return w.op.getLogFileName() - } - return fmt.Sprintf("%s_%s_%d_%s_%d%s", w.cfg.CaptureID, w.cfg.ChangeFeedID, w.cfg.CreateTime.Unix(), w.cfg.FileType, w.commitTS.Load(), common.LogEXT) -} - -func (w *Writer) filePath() string { - return filepath.Join(w.cfg.Dir, w.getLogFileName()) -} - -func openTruncFile(name string) (*os.File, error) { - return os.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, common.DefaultFileMode) -} - -func (w *Writer) openNew() error { - err := os.MkdirAll(w.cfg.Dir, common.DefaultDirMode) - if err != nil { - return cerror.WrapError(cerror.ErrRedoFileOp, errors.Annotatef(err, "can't make dir: %s for new redo logfile", w.cfg.Dir)) - } - - // reset ts used in file name when new file - w.commitTS.Store(w.eventCommitTS.Load()) - w.maxCommitTS.Store(w.eventCommitTS.Load()) - path := w.filePath() + common.TmpEXT - f, err := openTruncFile(path) - if err != nil { - return cerror.WrapError(cerror.ErrRedoFileOp, errors.Annotate(err, "can't open new redo logfile")) - } - w.file = f - w.size = 0 - err = w.newPageWriter() - if err != nil { - return err - } - return nil -} - -func (w *Writer) openOrNew(writeLen int) error { - path := w.filePath() - info, err := os.Stat(path) - if os.IsNotExist(err) { - return w.openNew() - } - if err != nil { - return cerror.WrapError(cerror.ErrRedoFileOp, errors.Annotate(err, "error getting log file info")) - } - - if info.Size()+int64(writeLen) >= w.cfg.MaxLogSize { - return w.rotate() - } - - file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, common.DefaultFileMode) - if err != nil { - // return err let the caller decide next move - return cerror.WrapError(cerror.ErrRedoFileOp, err) - } - - w.file = file - w.size = info.Size() - err = w.newPageWriter() - if err != nil { - return err - } - return nil -} - -func (w *Writer) newPageWriter() error { - offset, err := w.file.Seek(0, io.SeekCurrent) - if err != nil { - return cerror.WrapError(cerror.ErrRedoFileOp, err) - } - w.bw = pioutil.NewPageWriter(w.file, pageBytes, int(offset)) - - return nil -} - -func (w *Writer) rotate() error { - if err := w.close(); err != nil { - return err - } - return w.openNew() -} - -// GC implement GC interface -func (w *Writer) GC(checkPointTs uint64) error { - if !w.IsRunning() || w.isGCRunning() { - return nil - } - - w.gcRunning.Store(true) - defer w.gcRunning.Store(false) - - remove, err := w.getShouldRemovedFiles(checkPointTs) - if err != nil { - return err - } - - var errs error - for _, f := range remove { - err := os.Remove(filepath.Join(w.cfg.Dir, f.Name())) - errs = multierr.Append(errs, err) - } - - if errs != nil { - return cerror.WrapError(cerror.ErrRedoFileOp, errs) - } - - if w.cfg.S3Storage { - // since if fail delete in s3, do not block any path, so just log the error if any - go func() { - var errs error - for _, f := range remove { - err := w.storage.DeleteFile(context.Background(), f.Name()) - errs = multierr.Append(errs, err) - } - if errs != nil { - errs = cerror.WrapError(cerror.ErrS3StorageAPI, errs) - log.Warn("delete redo log in s3 fail", zap.Error(errs)) - } - }() - } - - return nil -} - -// shouldRemoved remove the file which commitTs in file name (max commitTs of all event ts in the file) < checkPointTs, -// since all event ts < checkPointTs already sent to sink, the log is not needed any more for recovery -func (w *Writer) shouldRemoved(checkPointTs uint64, f os.FileInfo) (bool, error) { - if filepath.Ext(f.Name()) != common.LogEXT { - return false, nil - } - - commitTs, fileType, err := common.ParseLogFileName(f.Name()) - if err != nil { - return false, err - } - - return commitTs < checkPointTs && fileType == w.cfg.FileType, nil -} - -func (w *Writer) getShouldRemovedFiles(checkPointTs uint64) ([]os.FileInfo, error) { - files, err := ioutil.ReadDir(w.cfg.Dir) - if err != nil { - if os.IsNotExist(err) { - log.Warn("check removed log dir fail", zap.Error(err)) - return []os.FileInfo{}, nil - } - return nil, cerror.WrapError(cerror.ErrRedoFileOp, errors.Annotatef(err, "can't read log file directory: %s", w.cfg.Dir)) - } - - logFiles := []os.FileInfo{} - for _, f := range files { - ret, err := w.shouldRemoved(checkPointTs, f) - if err != nil { - log.Warn("check removed log file fail", - zap.String("log file", f.Name()), - zap.Error(err)) - continue - } - - if ret { - logFiles = append(logFiles, f) - } - } - - return logFiles, nil -} - -func (w *Writer) flushAll() error { - if w.file == nil { - return nil - } - - start := time.Now() - err := w.flush() - if err != nil { - return err - } - if !w.cfg.S3Storage { - return nil - } - - ctx, cancel := context.WithTimeout(context.Background(), defaultS3Timeout) - defer cancel() - - err = w.writeToS3(ctx, w.file.Name()) - w.metricFlushAllDuration.Observe(time.Since(start).Seconds()) - - return err -} - -// Flush implement Flush interface -func (w *Writer) Flush() error { - w.Lock() - defer w.Unlock() - - return w.flushAll() -} - -func (w *Writer) flush() error { - if w.file == nil { - return nil - } - - n, err := w.bw.FlushN() - w.metricWriteBytes.Add(float64(n)) - if err != nil { - return cerror.WrapError(cerror.ErrRedoFileOp, err) - } - - start := time.Now() - err = w.file.Sync() - w.metricFsyncDuration.Observe(time.Since(start).Seconds()) - - return cerror.WrapError(cerror.ErrRedoFileOp, err) -} - -func (w *Writer) writeToS3(ctx context.Context, name string) error { - fileData, err := os.ReadFile(name) - if err != nil { - return cerror.WrapError(cerror.ErrRedoFileOp, err) - } - - // Key in s3: aws.String(rs.options.Prefix + name), prefix should be changefeed name - return cerror.WrapError(cerror.ErrS3StorageAPI, w.storage.WriteFile(ctx, filepath.Base(name), fileData)) -} diff --git a/cdc/cdc/redo/writer/file_test.go b/cdc/cdc/redo/writer/file_test.go deleted file mode 100644 index c5c833aa..00000000 --- a/cdc/cdc/redo/writer/file_test.go +++ /dev/null @@ -1,283 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package writer - -import ( - "context" - "fmt" - "io/ioutil" - "net/url" - "os" - "path/filepath" - "testing" - "time" - - "github.com/golang/mock/gomock" - "github.com/pingcap/errors" - mockstorage "github.com/pingcap/tidb/br/pkg/mock/storage" - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/redo/common" - "github.com/tikv/migration/cdc/pkg/leakutil" - "github.com/uber-go/atomic" -) - -func TestMain(m *testing.M) { - originValue := defaultGCIntervalInMs - defaultGCIntervalInMs = 1 - defer func() { - defaultGCIntervalInMs = originValue - }() - - leakutil.SetUpLeakTest(m) -} - -func TestWriterWrite(t *testing.T) { - dir, err := ioutil.TempDir("", "redo-writer") - require.Nil(t, err) - defer os.RemoveAll(dir) - - w := &Writer{ - cfg: &FileWriterConfig{ - MaxLogSize: 10, - Dir: dir, - ChangeFeedID: "test-cf", - CaptureID: "cp", - FileType: common.DefaultRowLogFileType, - CreateTime: time.Date(2000, 1, 1, 1, 1, 1, 1, &time.Location{}), - }, - uint64buf: make([]byte, 8), - running: *atomic.NewBool(true), - metricWriteBytes: redoWriteBytesGauge.WithLabelValues("cp", "test-cf"), - metricFsyncDuration: redoFsyncDurationHistogram.WithLabelValues("cp", "test-cf"), - metricFlushAllDuration: redoFlushAllDurationHistogram.WithLabelValues("cp", "test-cf"), - } - - w.eventCommitTS.Store(1) - _, err = w.Write([]byte("tes1t11111")) - require.Nil(t, err) - // create a .tmp file - fileName := fmt.Sprintf("%s_%s_%d_%s_%d%s", w.cfg.CaptureID, w.cfg.ChangeFeedID, w.cfg.CreateTime.Unix(), w.cfg.FileType, 1, common.LogEXT) + common.TmpEXT - path := filepath.Join(w.cfg.Dir, fileName) - info, err := os.Stat(path) - require.Nil(t, err) - require.Equal(t, fileName, info.Name()) - - w.eventCommitTS.Store(12) - _, err = w.Write([]byte("tt")) - require.Nil(t, err) - w.eventCommitTS.Store(22) - _, err = w.Write([]byte("t")) - require.Nil(t, err) - - // after rotate, rename to .log - fileName = fmt.Sprintf("%s_%s_%d_%s_%d%s", w.cfg.CaptureID, w.cfg.ChangeFeedID, w.cfg.CreateTime.Unix(), w.cfg.FileType, 1, common.LogEXT) - path = filepath.Join(w.cfg.Dir, fileName) - info, err = os.Stat(path) - require.Nil(t, err) - require.Equal(t, fileName, info.Name()) - // create a .tmp file with first eventCommitTS as name - fileName = fmt.Sprintf("%s_%s_%d_%s_%d%s", w.cfg.CaptureID, w.cfg.ChangeFeedID, w.cfg.CreateTime.Unix(), w.cfg.FileType, 12, common.LogEXT) + common.TmpEXT - path = filepath.Join(w.cfg.Dir, fileName) - info, err = os.Stat(path) - require.Nil(t, err) - require.Equal(t, fileName, info.Name()) - err = w.Close() - require.Nil(t, err) - require.False(t, w.IsRunning()) - // safe close, rename to .log with max eventCommitTS as name - fileName = fmt.Sprintf("%s_%s_%d_%s_%d%s", w.cfg.CaptureID, w.cfg.ChangeFeedID, w.cfg.CreateTime.Unix(), w.cfg.FileType, 22, common.LogEXT) - path = filepath.Join(w.cfg.Dir, fileName) - info, err = os.Stat(path) - require.Nil(t, err) - require.Equal(t, fileName, info.Name()) - - w1 := &Writer{ - cfg: &FileWriterConfig{ - MaxLogSize: 10, - Dir: dir, - ChangeFeedID: "test-cf11", - CaptureID: "cp", - FileType: common.DefaultRowLogFileType, - CreateTime: time.Date(2000, 1, 1, 1, 1, 1, 1, &time.Location{}), - }, - uint64buf: make([]byte, 8), - running: *atomic.NewBool(true), - metricWriteBytes: redoWriteBytesGauge.WithLabelValues("cp", "test-cf11"), - metricFsyncDuration: redoFsyncDurationHistogram.WithLabelValues("cp", "test-cf11"), - metricFlushAllDuration: redoFlushAllDurationHistogram.WithLabelValues("cp", "test-cf11"), - } - - w1.eventCommitTS.Store(1) - _, err = w1.Write([]byte("tes1t11111")) - require.Nil(t, err) - // create a .tmp file - fileName = fmt.Sprintf("%s_%s_%d_%s_%d%s", w1.cfg.CaptureID, w1.cfg.ChangeFeedID, w1.cfg.CreateTime.Unix(), w1.cfg.FileType, 1, common.LogEXT) + common.TmpEXT - path = filepath.Join(w1.cfg.Dir, fileName) - info, err = os.Stat(path) - require.Nil(t, err) - require.Equal(t, fileName, info.Name()) - // change the file name, should cause CLose err - err = os.Rename(path, path+"new") - require.Nil(t, err) - err = w1.Close() - require.NotNil(t, err) - // closed anyway - require.False(t, w1.IsRunning()) -} - -func TestWriterGC(t *testing.T) { - dir, err := ioutil.TempDir("", "redo-GC") - require.Nil(t, err) - defer os.RemoveAll(dir) - - controller := gomock.NewController(t) - mockStorage := mockstorage.NewMockExternalStorage(controller) - mockStorage.EXPECT().WriteFile(gomock.Any(), "cp_test_946688461_row_1.log.tmp", gomock.Any()).Return(nil).Times(1) - mockStorage.EXPECT().WriteFile(gomock.Any(), "cp_test_946688461_row_1.log", gomock.Any()).Return(nil).Times(1) - mockStorage.EXPECT().DeleteFile(gomock.Any(), "cp_test_946688461_row_1.log.tmp").Return(nil).Times(1) - - mockStorage.EXPECT().WriteFile(gomock.Any(), "cp_test_946688461_row_2.log.tmp", gomock.Any()).Return(nil).Times(1) - mockStorage.EXPECT().WriteFile(gomock.Any(), "cp_test_946688461_row_2.log", gomock.Any()).Return(nil).Times(1) - mockStorage.EXPECT().DeleteFile(gomock.Any(), "cp_test_946688461_row_2.log.tmp").Return(nil).Times(1) - - mockStorage.EXPECT().WriteFile(gomock.Any(), "cp_test_946688461_row_3.log.tmp", gomock.Any()).Return(nil).Times(1) - mockStorage.EXPECT().WriteFile(gomock.Any(), "cp_test_946688461_row_3.log", gomock.Any()).Return(nil).Times(1) - mockStorage.EXPECT().DeleteFile(gomock.Any(), "cp_test_946688461_row_3.log.tmp").Return(nil).Times(1) - - mockStorage.EXPECT().DeleteFile(gomock.Any(), "cp_test_946688461_row_1.log").Return(errors.New("ignore err")).Times(1) - mockStorage.EXPECT().DeleteFile(gomock.Any(), "cp_test_946688461_row_2.log").Return(errors.New("ignore err")).Times(1) - - megabyte = 1 - cfg := &FileWriterConfig{ - Dir: dir, - ChangeFeedID: "test", - CaptureID: "cp", - MaxLogSize: 10, - FileType: common.DefaultRowLogFileType, - CreateTime: time.Date(2000, 1, 1, 1, 1, 1, 1, &time.Location{}), - FlushIntervalInMs: 5, - S3Storage: true, - } - w := &Writer{ - cfg: cfg, - uint64buf: make([]byte, 8), - storage: mockStorage, - metricWriteBytes: redoWriteBytesGauge.WithLabelValues(cfg.CaptureID, cfg.ChangeFeedID), - metricFsyncDuration: redoFsyncDurationHistogram.WithLabelValues(cfg.CaptureID, cfg.ChangeFeedID), - metricFlushAllDuration: redoFlushAllDurationHistogram.WithLabelValues(cfg.CaptureID, cfg.ChangeFeedID), - } - w.running.Store(true) - w.eventCommitTS.Store(1) - _, err = w.Write([]byte("t1111")) - require.Nil(t, err) - w.eventCommitTS.Store(2) - _, err = w.Write([]byte("t2222")) - require.Nil(t, err) - w.eventCommitTS.Store(3) - _, err = w.Write([]byte("t3333")) - require.Nil(t, err) - - files, err := ioutil.ReadDir(w.cfg.Dir) - require.Nil(t, err) - require.Equal(t, 3, len(files), "should have 3 log file") - - err = w.GC(3) - require.Nil(t, err) - - err = w.Close() - require.Nil(t, err) - require.False(t, w.IsRunning()) - files, err = ioutil.ReadDir(w.cfg.Dir) - require.Nil(t, err) - require.Equal(t, 1, len(files), "should have 1 log left after GC") - - ts, fileType, err := common.ParseLogFileName(files[0].Name()) - require.Nil(t, err, files[0].Name()) - require.EqualValues(t, 3, ts) - require.Equal(t, common.DefaultRowLogFileType, fileType) - time.Sleep(time.Duration(100) * time.Millisecond) - - w1 := &Writer{ - cfg: cfg, - uint64buf: make([]byte, 8), - storage: mockStorage, - } - w1.cfg.Dir += "not-exist" - w1.running.Store(true) - err = w1.GC(111) - require.Nil(t, err) -} - -func TestAdvanceTs(t *testing.T) { - w := &Writer{} - w.AdvanceTs(111) - require.EqualValues(t, 111, w.eventCommitTS.Load()) -} - -func TestNewWriter(t *testing.T) { - _, err := NewWriter(context.Background(), nil) - require.NotNil(t, err) - - s3URI, err := url.Parse("s3://logbucket/test-changefeed?endpoint=http://111/") - require.Nil(t, err) - - dir, err := ioutil.TempDir("", "redo-NewWriter") - require.Nil(t, err) - defer os.RemoveAll(dir) - - w, err := NewWriter(context.Background(), &FileWriterConfig{ - Dir: "sdfsf", - S3Storage: true, - S3URI: *s3URI, - }) - require.Nil(t, err) - time.Sleep(time.Duration(defaultFlushIntervalInMs+1) * time.Millisecond) - err = w.Close() - require.Nil(t, err) - require.False(t, w.IsRunning()) - - controller := gomock.NewController(t) - mockStorage := mockstorage.NewMockExternalStorage(controller) - mockStorage.EXPECT().WriteFile(gomock.Any(), "cp_test_946688461_ddl_0.log.tmp", gomock.Any()).Return(nil).Times(2) - mockStorage.EXPECT().WriteFile(gomock.Any(), "cp_test_946688461_ddl_0.log", gomock.Any()).Return(nil).Times(1) - mockStorage.EXPECT().DeleteFile(gomock.Any(), "cp_test_946688461_ddl_0.log.tmp").Return(nil).Times(1) - - w = &Writer{ - cfg: &FileWriterConfig{ - Dir: dir, - CaptureID: "cp", - ChangeFeedID: "test", - FileType: common.DefaultDDLLogFileType, - CreateTime: time.Date(2000, 1, 1, 1, 1, 1, 1, &time.Location{}), - S3Storage: true, - MaxLogSize: defaultMaxLogSize, - }, - uint64buf: make([]byte, 8), - storage: mockStorage, - metricWriteBytes: redoWriteBytesGauge.WithLabelValues("cp", "test"), - metricFsyncDuration: redoFsyncDurationHistogram.WithLabelValues("cp", "test"), - metricFlushAllDuration: redoFlushAllDurationHistogram.WithLabelValues("cp", "test"), - } - w.running.Store(true) - _, err = w.Write([]byte("test")) - require.Nil(t, err) - // - err = w.Flush() - require.Nil(t, err) - - err = w.Close() - require.Nil(t, err) - require.Equal(t, w.running.Load(), false) - time.Sleep(time.Duration(defaultFlushIntervalInMs+1) * time.Millisecond) -} diff --git a/cdc/cdc/redo/writer/metric.go b/cdc/cdc/redo/writer/metric.go deleted file mode 100644 index b35d8e20..00000000 --- a/cdc/cdc/redo/writer/metric.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package writer - -import ( - "github.com/prometheus/client_golang/prometheus" -) - -const ( - namespace = "ticdc" - subsystem = "redo" -) - -var ( - redoWriteBytesGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "write_bytes_total", - Help: "Total number of bytes redo log written", - }, []string{"capture", "changefeed"}) - - redoFsyncDurationHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "fsync_duration_seconds", - Help: "The latency distributions of fsync called by redo writer", - Buckets: prometheus.ExponentialBuckets(0.001, 2.0, 13), - }, []string{"capture", "changefeed"}) - - redoFlushAllDurationHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "flushall_duration_seconds", - Help: "The latency distributions of flushall called by redo writer", - Buckets: prometheus.ExponentialBuckets(0.001, 2.0, 13), - }, []string{"capture", "changefeed"}) - - redoTotalRowsCountGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "total_rows_count", - Help: "The total count of rows that are processed by redo writer", - }, []string{"capture", "changefeed"}) -) - -// InitMetrics registers all metrics in this file -func InitMetrics(registry *prometheus.Registry) { - registry.MustRegister(redoFsyncDurationHistogram) - registry.MustRegister(redoTotalRowsCountGauge) - registry.MustRegister(redoWriteBytesGauge) - registry.MustRegister(redoFlushAllDurationHistogram) -} diff --git a/cdc/cdc/redo/writer/mock_RedoLogWriter.go b/cdc/cdc/redo/writer/mock_RedoLogWriter.go deleted file mode 100644 index 3a493315..00000000 --- a/cdc/cdc/redo/writer/mock_RedoLogWriter.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -// Code generated by mockery v0.0.0-dev. DO NOT EDIT. - -package writer - -import ( - context "context" - - mock "github.com/stretchr/testify/mock" - model "github.com/tikv/migration/cdc/cdc/model" -) - -// MockRedoLogWriter is an autogenerated mock type for the RedoLogWriter type -type MockRedoLogWriter struct { - mock.Mock -} - -// Close provides a mock function with given fields: -func (_m *MockRedoLogWriter) Close() error { - ret := _m.Called() - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// EmitCheckpointTs provides a mock function with given fields: ctx, ts -func (_m *MockRedoLogWriter) EmitCheckpointTs(ctx context.Context, ts uint64) error { - ret := _m.Called(ctx, ts) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, uint64) error); ok { - r0 = rf(ctx, ts) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// EmitResolvedTs provides a mock function with given fields: ctx, ts -func (_m *MockRedoLogWriter) EmitResolvedTs(ctx context.Context, ts uint64) error { - ret := _m.Called(ctx, ts) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, uint64) error); ok { - r0 = rf(ctx, ts) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// FlushLog provides a mock function with given fields: ctx, tableID, ts -func (_m *MockRedoLogWriter) FlushLog(ctx context.Context, tableID int64, ts uint64) error { - ret := _m.Called(ctx, tableID, ts) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, int64, uint64) error); ok { - r0 = rf(ctx, tableID, ts) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// GetCurrentResolvedTs provides a mock function with given fields: ctx, tableIDs -func (_m *MockRedoLogWriter) GetCurrentResolvedTs(ctx context.Context, tableIDs []int64) (map[int64]uint64, error) { - ret := _m.Called(ctx, tableIDs) - - var r0 map[int64]uint64 - if rf, ok := ret.Get(0).(func(context.Context, []int64) map[int64]uint64); ok { - r0 = rf(ctx, tableIDs) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[int64]uint64) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, []int64) error); ok { - r1 = rf(ctx, tableIDs) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// SendDDL provides a mock function with given fields: ctx, ddl -func (_m *MockRedoLogWriter) SendDDL(ctx context.Context, ddl *model.RedoDDLEvent) error { - ret := _m.Called(ctx, ddl) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *model.RedoDDLEvent) error); ok { - r0 = rf(ctx, ddl) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// WriteLog provides a mock function with given fields: ctx, tableID, rows -func (_m *MockRedoLogWriter) WriteLog(ctx context.Context, tableID int64, rows []*model.RedoRowChangedEvent) (uint64, error) { - ret := _m.Called(ctx, tableID, rows) - - var r0 uint64 - if rf, ok := ret.Get(0).(func(context.Context, int64, []*model.RedoRowChangedEvent) uint64); ok { - r0 = rf(ctx, tableID, rows) - } else { - r0 = ret.Get(0).(uint64) - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, int64, []*model.RedoRowChangedEvent) error); ok { - r1 = rf(ctx, tableID, rows) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} diff --git a/cdc/cdc/redo/writer/mock_fileWriter.go b/cdc/cdc/redo/writer/mock_fileWriter.go deleted file mode 100644 index 82a34efd..00000000 --- a/cdc/cdc/redo/writer/mock_fileWriter.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -// Code generated by mockery v0.0.0-dev. DO NOT EDIT. - -package writer - -import mock "github.com/stretchr/testify/mock" - -// mockFileWriter is an autogenerated mock type for the fileWriter type -type mockFileWriter struct { - mock.Mock -} - -// AdvanceTs provides a mock function with given fields: commitTs -func (_m *mockFileWriter) AdvanceTs(commitTs uint64) { - _m.Called(commitTs) -} - -// Close provides a mock function with given fields: -func (_m *mockFileWriter) Close() error { - ret := _m.Called() - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Flush provides a mock function with given fields: -func (_m *mockFileWriter) Flush() error { - ret := _m.Called() - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// GC provides a mock function with given fields: checkPointTs -func (_m *mockFileWriter) GC(checkPointTs uint64) error { - ret := _m.Called(checkPointTs) - - var r0 error - if rf, ok := ret.Get(0).(func(uint64) error); ok { - r0 = rf(checkPointTs) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// IsRunning provides a mock function with given fields: -func (_m *mockFileWriter) IsRunning() bool { - ret := _m.Called() - - var r0 bool - if rf, ok := ret.Get(0).(func() bool); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// Write provides a mock function with given fields: p -func (_m *mockFileWriter) Write(p []byte) (int, error) { - ret := _m.Called(p) - - var r0 int - if rf, ok := ret.Get(0).(func([]byte) int); ok { - r0 = rf(p) - } else { - r0 = ret.Get(0).(int) - } - - var r1 error - if rf, ok := ret.Get(1).(func([]byte) error); ok { - r1 = rf(p) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} diff --git a/cdc/cdc/redo/writer/writer.go b/cdc/cdc/redo/writer/writer.go deleted file mode 100644 index 95b532c2..00000000 --- a/cdc/cdc/redo/writer/writer.go +++ /dev/null @@ -1,660 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package writer - -import ( - "context" - "fmt" - "io" - "io/ioutil" - "net/url" - "os" - "path/filepath" - "sync" - "time" - - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/pingcap/errors" - "github.com/pingcap/log" - "github.com/pingcap/tidb/br/pkg/storage" - "github.com/prometheus/client_golang/prometheus" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/redo/common" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "go.uber.org/multierr" - "go.uber.org/zap" - "golang.org/x/sync/errgroup" -) - -//go:generate mockery --name=RedoLogWriter --inpackage -// RedoLogWriter defines the interfaces used to write redo log, all operations are thread-safe -type RedoLogWriter interface { - io.Closer - - // WriteLog writer RedoRowChangedEvent to row log file - WriteLog(ctx context.Context, tableID int64, rows []*model.RedoRowChangedEvent) (resolvedTs uint64, err error) - - // SendDDL EmitCheckpointTs and EmitResolvedTs are called from owner only - // SendDDL writer RedoDDLEvent to ddl log file - SendDDL(ctx context.Context, ddl *model.RedoDDLEvent) error - - // FlushLog sends resolved-ts from table pipeline to log writer, it is - // essential to flush when a table doesn't have any row change event for - // some time, and the resolved ts of this table should be moved forward. - FlushLog(ctx context.Context, tableID int64, ts uint64) error - - // EmitCheckpointTs write CheckpointTs to meta file - EmitCheckpointTs(ctx context.Context, ts uint64) error - - // EmitResolvedTs write ResolvedTs to meta file - EmitResolvedTs(ctx context.Context, ts uint64) error - - // GetCurrentResolvedTs return all the ResolvedTs list for given tableIDs - GetCurrentResolvedTs(ctx context.Context, tableIDs []int64) (resolvedTsList map[int64]uint64, err error) - - // DeleteAllLogs delete all log files related to the changefeed, called from owner only when delete changefeed - DeleteAllLogs(ctx context.Context) error -} - -var defaultGCIntervalInMs = 5000 - -var ( - logWriters = map[string]*LogWriter{} - initLock sync.Mutex -) - -var redoLogPool = sync.Pool{ - New: func() interface{} { - return &model.RedoLog{} - }, -} - -// LogWriterConfig is the configuration used by a Writer. -type LogWriterConfig struct { - Dir string - ChangeFeedID string - CaptureID string - CreateTime time.Time - // MaxLogSize is the maximum size of log in megabyte, defaults to defaultMaxLogSize. - MaxLogSize int64 - FlushIntervalInMs int64 - S3Storage bool - // S3URI should be like S3URI="s3://logbucket/test-changefeed?endpoint=http://$S3_ENDPOINT/" - S3URI url.URL -} - -// LogWriter implement the RedoLogWriter interface -type LogWriter struct { - cfg *LogWriterConfig - rowWriter fileWriter - ddlWriter fileWriter - storage storage.ExternalStorage - meta *common.LogMeta - metaLock sync.RWMutex - - metricTotalRowsCount prometheus.Gauge -} - -// NewLogWriter creates a LogWriter instance. It is guaranteed only one LogWriter per changefeed -func NewLogWriter(ctx context.Context, cfg *LogWriterConfig) (*LogWriter, error) { - if cfg == nil { - return nil, cerror.WrapError(cerror.ErrRedoConfigInvalid, errors.New("LogWriterConfig can not be nil")) - } - - initLock.Lock() - defer initLock.Unlock() - - if v, ok := logWriters[cfg.ChangeFeedID]; ok { - // if cfg changed or already closed need create a new LogWriter - if cfg.String() == v.cfg.String() && !v.isStopped() { - return v, nil - } - } - - var err error - var logWriter *LogWriter - rowCfg := &FileWriterConfig{ - Dir: cfg.Dir, - ChangeFeedID: cfg.ChangeFeedID, - CaptureID: cfg.CaptureID, - FileType: common.DefaultRowLogFileType, - CreateTime: cfg.CreateTime, - MaxLogSize: cfg.MaxLogSize, - FlushIntervalInMs: cfg.FlushIntervalInMs, - S3Storage: cfg.S3Storage, - S3URI: cfg.S3URI, - } - ddlCfg := &FileWriterConfig{ - Dir: cfg.Dir, - ChangeFeedID: cfg.ChangeFeedID, - CaptureID: cfg.CaptureID, - FileType: common.DefaultDDLLogFileType, - CreateTime: cfg.CreateTime, - MaxLogSize: cfg.MaxLogSize, - FlushIntervalInMs: cfg.FlushIntervalInMs, - S3Storage: cfg.S3Storage, - S3URI: cfg.S3URI, - } - logWriter = &LogWriter{ - cfg: cfg, - } - logWriter.rowWriter, err = NewWriter(ctx, rowCfg) - if err != nil { - return nil, err - } - logWriter.ddlWriter, err = NewWriter(ctx, ddlCfg) - if err != nil { - return nil, err - } - - // since the error will not block write log, so keep go to the next init process - err = logWriter.initMeta(ctx) - if err != nil { - log.Warn("init redo meta fail", - zap.String("change feed", cfg.ChangeFeedID), - zap.Error(err)) - } - if cfg.S3Storage { - logWriter.storage, err = common.InitS3storage(ctx, cfg.S3URI) - if err != nil { - return nil, err - } - } - // close previous writer - if v, ok := logWriters[cfg.ChangeFeedID]; ok { - err = v.Close() - if err != nil { - return nil, err - } - } else { - if cfg.S3Storage { - // since other process get the remove changefeed job async, may still write some logs after owner delete the log - err = logWriter.preCleanUpS3(ctx) - if err != nil { - return nil, err - } - } - } - - logWriter.metricTotalRowsCount = redoTotalRowsCountGauge.WithLabelValues(cfg.CaptureID, cfg.ChangeFeedID) - logWriters[cfg.ChangeFeedID] = logWriter - go logWriter.runGC(ctx) - return logWriter, nil -} - -func (l *LogWriter) preCleanUpS3(ctx context.Context) error { - ret, err := l.storage.FileExists(ctx, l.getDeletedChangefeedMarker()) - if err != nil { - return cerror.WrapError(cerror.ErrS3StorageAPI, err) - } - if !ret { - return nil - } - - files, err := getAllFilesInS3(ctx, l) - if err != nil { - return err - } - - ff := []string{} - for _, file := range files { - if file != l.getDeletedChangefeedMarker() { - ff = append(ff, file) - } - } - err = l.deleteFilesInS3(ctx, ff) - if err != nil { - return err - } - err = l.storage.DeleteFile(ctx, l.getDeletedChangefeedMarker()) - if !isNotExistInS3(err) { - return cerror.WrapError(cerror.ErrS3StorageAPI, err) - } - - return nil -} - -func (l *LogWriter) initMeta(ctx context.Context) error { - select { - case <-ctx.Done(): - return errors.Trace(ctx.Err()) - default: - } - - l.meta = &common.LogMeta{ResolvedTsList: map[int64]uint64{}} - files, err := ioutil.ReadDir(l.cfg.Dir) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return cerror.WrapError(cerror.ErrRedoMetaInitialize, errors.Annotate(err, "can't read log file directory")) - } - - for _, file := range files { - if filepath.Ext(file.Name()) == common.MetaEXT { - path := filepath.Join(l.cfg.Dir, file.Name()) - fileData, err := os.ReadFile(path) - if err != nil { - return cerror.WrapError(cerror.ErrRedoMetaInitialize, err) - } - - _, err = l.meta.UnmarshalMsg(fileData) - if err != nil { - l.meta = &common.LogMeta{ResolvedTsList: map[int64]uint64{}} - return cerror.WrapError(cerror.ErrRedoMetaInitialize, err) - } - break - } - } - return nil -} - -func (l *LogWriter) runGC(ctx context.Context) { - ticker := time.NewTicker(time.Duration(defaultGCIntervalInMs) * time.Millisecond) - defer ticker.Stop() - - for { - if l.isStopped() { - return - } - - select { - case <-ctx.Done(): - err := l.Close() - if err != nil { - log.Error("runGC close fail", zap.String("changefeedID", l.cfg.ChangeFeedID), zap.Error(err)) - } - case <-ticker.C: - err := l.gc() - if err != nil { - log.Error("redo log GC fail", zap.String("changefeedID", l.cfg.ChangeFeedID), zap.Error(err)) - } - } - } -} - -func (l *LogWriter) gc() error { - l.metaLock.RLock() - ts := l.meta.CheckPointTs - l.metaLock.RUnlock() - - var err error - err = multierr.Append(err, l.rowWriter.GC(ts)) - err = multierr.Append(err, l.ddlWriter.GC(ts)) - return err -} - -// WriteLog implement WriteLog api -func (l *LogWriter) WriteLog(ctx context.Context, tableID int64, rows []*model.RedoRowChangedEvent) (uint64, error) { - select { - case <-ctx.Done(): - return 0, errors.Trace(ctx.Err()) - default: - } - - if l.isStopped() { - return 0, cerror.ErrRedoWriterStopped.GenWithStackByArgs() - } - if len(rows) == 0 { - return 0, nil - } - - maxCommitTs := l.setMaxCommitTs(tableID, 0) - for i, r := range rows { - if r == nil || r.Row == nil { - continue - } - - rl := redoLogPool.Get().(*model.RedoLog) - rl.RedoRow = r - rl.RedoDDL = nil - rl.Type = model.RedoLogTypeRow - // TODO: crc check - data, err := rl.MarshalMsg(nil) - if err != nil { - // TODO: just return 0 if err ? - return maxCommitTs, cerror.WrapError(cerror.ErrMarshalFailed, err) - } - - l.rowWriter.AdvanceTs(r.Row.CommitTs) - _, err = l.rowWriter.Write(data) - if err != nil { - l.metricTotalRowsCount.Add(float64(i)) - return maxCommitTs, err - } - - maxCommitTs = l.setMaxCommitTs(tableID, r.Row.CommitTs) - redoLogPool.Put(rl) - } - l.metricTotalRowsCount.Add(float64(len(rows))) - return maxCommitTs, nil -} - -// SendDDL implement SendDDL api -func (l *LogWriter) SendDDL(ctx context.Context, ddl *model.RedoDDLEvent) error { - select { - case <-ctx.Done(): - return errors.Trace(ctx.Err()) - default: - } - - if l.isStopped() { - return cerror.ErrRedoWriterStopped.GenWithStackByArgs() - } - if ddl == nil || ddl.DDL == nil { - return nil - } - - rl := redoLogPool.Get().(*model.RedoLog) - defer redoLogPool.Put(rl) - - rl.RedoDDL = ddl - rl.RedoRow = nil - rl.Type = model.RedoLogTypeDDL - data, err := rl.MarshalMsg(nil) - if err != nil { - return cerror.WrapError(cerror.ErrMarshalFailed, err) - } - - l.ddlWriter.AdvanceTs(ddl.DDL.CommitTs) - _, err = l.ddlWriter.Write(data) - return err -} - -// FlushLog implement FlushLog api -func (l *LogWriter) FlushLog(ctx context.Context, tableID int64, ts uint64) error { - select { - case <-ctx.Done(): - return errors.Trace(ctx.Err()) - default: - } - - if l.isStopped() { - return cerror.ErrRedoWriterStopped.GenWithStackByArgs() - } - - if err := l.flush(); err != nil { - return err - } - l.setMaxCommitTs(tableID, ts) - return nil -} - -// EmitCheckpointTs implement EmitCheckpointTs api -func (l *LogWriter) EmitCheckpointTs(ctx context.Context, ts uint64) error { - select { - case <-ctx.Done(): - return errors.Trace(ctx.Err()) - default: - } - - if l.isStopped() { - return cerror.ErrRedoWriterStopped.GenWithStackByArgs() - } - return l.flushLogMeta(ts, 0) -} - -// EmitResolvedTs implement EmitResolvedTs api -func (l *LogWriter) EmitResolvedTs(ctx context.Context, ts uint64) error { - select { - case <-ctx.Done(): - return errors.Trace(ctx.Err()) - default: - } - - if l.isStopped() { - return cerror.ErrRedoWriterStopped.GenWithStackByArgs() - } - - return l.flushLogMeta(0, ts) -} - -// GetCurrentResolvedTs implement GetCurrentResolvedTs api -func (l *LogWriter) GetCurrentResolvedTs(ctx context.Context, tableIDs []int64) (map[int64]uint64, error) { - select { - case <-ctx.Done(): - return nil, errors.Trace(ctx.Err()) - default: - } - - if len(tableIDs) == 0 { - return nil, nil - } - - l.metaLock.RLock() - defer l.metaLock.RUnlock() - - // need to make sure all data received got saved already - err := l.rowWriter.Flush() - if err != nil { - return nil, err - } - - ret := map[int64]uint64{} - for i := 0; i < len(tableIDs); i++ { - id := tableIDs[i] - if v, ok := l.meta.ResolvedTsList[id]; ok { - ret[id] = v - } - } - - return ret, nil -} - -// DeleteAllLogs implement DeleteAllLogs api -func (l *LogWriter) DeleteAllLogs(ctx context.Context) error { - err := l.Close() - if err != nil { - return err - } - - if !l.cfg.S3Storage { - err = os.RemoveAll(l.cfg.Dir) - if err != nil { - return cerror.WrapError(cerror.ErrRedoFileOp, err) - } - // after delete logs, rm the LogWriter since it is already closed - l.cleanUpLogWriter() - return nil - } - - files, err := getAllFilesInS3(ctx, l) - if err != nil { - return err - } - - err = l.deleteFilesInS3(ctx, files) - if err != nil { - return err - } - // after delete logs, rm the LogWriter since it is already closed - l.cleanUpLogWriter() - - // write a marker to s3, since other process get the remove changefeed job async, - // may still write some logs after owner delete the log - return l.writeDeletedMarkerToS3(ctx) -} - -func (l *LogWriter) getDeletedChangefeedMarker() string { - return fmt.Sprintf("delete_%s", l.cfg.ChangeFeedID) -} - -func (l *LogWriter) writeDeletedMarkerToS3(ctx context.Context) error { - return cerror.WrapError(cerror.ErrS3StorageAPI, l.storage.WriteFile(ctx, l.getDeletedChangefeedMarker(), []byte("D"))) -} - -func (l *LogWriter) cleanUpLogWriter() { - initLock.Lock() - defer initLock.Unlock() - delete(logWriters, l.cfg.ChangeFeedID) -} - -func (l *LogWriter) deleteFilesInS3(ctx context.Context, files []string) error { - eg, eCtx := errgroup.WithContext(ctx) - for _, f := range files { - name := f - eg.Go(func() error { - err := l.storage.DeleteFile(eCtx, name) - if err != nil { - // if fail then retry, may end up with notExit err, ignore the error - if !isNotExistInS3(err) { - return cerror.WrapError(cerror.ErrS3StorageAPI, err) - } - } - return nil - }) - } - return eg.Wait() -} - -func isNotExistInS3(err error) bool { - if err != nil { - if aerr, ok := errors.Cause(err).(awserr.Error); ok { // nolint:errorlint - switch aerr.Code() { - case s3.ErrCodeNoSuchKey: - return true - } - } - } - return false -} - -var getAllFilesInS3 = func(ctx context.Context, l *LogWriter) ([]string, error) { - files := []string{} - err := l.storage.WalkDir(ctx, &storage.WalkOption{}, func(path string, _ int64) error { - files = append(files, path) - return nil - }) - if err != nil { - return nil, cerror.WrapError(cerror.ErrS3StorageAPI, err) - } - - return files, nil -} - -// Close implements RedoLogWriter.Close. -func (l *LogWriter) Close() error { - redoTotalRowsCountGauge.DeleteLabelValues(l.cfg.CaptureID, l.cfg.ChangeFeedID) - - var err error - err = multierr.Append(err, l.rowWriter.Close()) - err = multierr.Append(err, l.ddlWriter.Close()) - return err -} - -func (l *LogWriter) setMaxCommitTs(tableID int64, commitTs uint64) uint64 { - l.metaLock.Lock() - defer l.metaLock.Unlock() - - if v, ok := l.meta.ResolvedTsList[tableID]; ok { - if v < commitTs { - l.meta.ResolvedTsList[tableID] = commitTs - } - } else { - l.meta.ResolvedTsList[tableID] = commitTs - } - - return l.meta.ResolvedTsList[tableID] -} - -// flush flushes all the buffered data to the disk. -func (l *LogWriter) flush() error { - err1 := l.flushLogMeta(0, 0) - err2 := l.ddlWriter.Flush() - err3 := l.rowWriter.Flush() - - err := multierr.Append(err1, err2) - err = multierr.Append(err, err3) - return err -} - -func (l *LogWriter) isStopped() bool { - return !l.ddlWriter.IsRunning() || !l.rowWriter.IsRunning() -} - -func (l *LogWriter) getMetafileName() string { - return fmt.Sprintf("%s_%s_%s%s", l.cfg.CaptureID, l.cfg.ChangeFeedID, common.DefaultMetaFileType, common.MetaEXT) -} - -func (l *LogWriter) flushLogMeta(checkPointTs, resolvedTs uint64) error { - l.metaLock.Lock() - defer l.metaLock.Unlock() - - if checkPointTs != 0 { - l.meta.CheckPointTs = checkPointTs - } - if resolvedTs != 0 { - l.meta.ResolvedTs = resolvedTs - } - data, err := l.meta.MarshalMsg(nil) - if err != nil { - return cerror.WrapError(cerror.ErrMarshalFailed, err) - } - - err = os.MkdirAll(l.cfg.Dir, common.DefaultDirMode) - if err != nil { - return cerror.WrapError(cerror.ErrRedoFileOp, errors.Annotate(err, "can't make dir for new redo logfile")) - } - - tmpFileName := l.filePath() + common.MetaTmpEXT - tmpFile, err := openTruncFile(tmpFileName) - if err != nil { - return cerror.WrapError(cerror.ErrRedoFileOp, err) - } - - _, err = tmpFile.Write(data) - if err != nil { - return cerror.WrapError(cerror.ErrRedoFileOp, err) - } - err = tmpFile.Sync() - if err != nil { - return cerror.WrapError(cerror.ErrRedoFileOp, err) - } - err = tmpFile.Close() - if err != nil { - return cerror.WrapError(cerror.ErrRedoFileOp, err) - } - - err = os.Rename(tmpFileName, l.filePath()) - if err != nil { - return cerror.WrapError(cerror.ErrRedoFileOp, err) - } - - if !l.cfg.S3Storage { - return nil - } - - ctx, cancel := context.WithTimeout(context.Background(), defaultS3Timeout) - defer cancel() - return l.writeMetaToS3(ctx) -} - -func (l *LogWriter) writeMetaToS3(ctx context.Context) error { - name := l.filePath() - fileData, err := os.ReadFile(name) - if err != nil { - return cerror.WrapError(cerror.ErrRedoFileOp, err) - } - - return cerror.WrapError(cerror.ErrS3StorageAPI, l.storage.WriteFile(ctx, l.getMetafileName(), fileData)) -} - -func (l *LogWriter) filePath() string { - return filepath.Join(l.cfg.Dir, l.getMetafileName()) -} - -func (cfg LogWriterConfig) String() string { - return fmt.Sprintf("%s:%s:%s:%d:%d:%s:%t", cfg.ChangeFeedID, cfg.CaptureID, cfg.Dir, cfg.MaxLogSize, cfg.FlushIntervalInMs, cfg.S3URI.String(), cfg.S3Storage) -} diff --git a/cdc/cdc/redo/writer/writer_test.go b/cdc/cdc/redo/writer/writer_test.go deleted file mode 100644 index be751128..00000000 --- a/cdc/cdc/redo/writer/writer_test.go +++ /dev/null @@ -1,949 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package writer - -import ( - "context" - "fmt" - "io/ioutil" - "math" - "net/url" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/golang/mock/gomock" - "github.com/pingcap/errors" - mockstorage "github.com/pingcap/tidb/br/pkg/mock/storage" - "github.com/pingcap/tidb/br/pkg/storage" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/redo/common" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "go.uber.org/multierr" -) - -func TestLogWriterWriteLog(t *testing.T) { - type arg struct { - ctx context.Context - tableID int64 - rows []*model.RedoRowChangedEvent - } - tests := []struct { - name string - args arg - wantTs uint64 - isRunning bool - writerErr error - wantErr error - }{ - { - name: "happy", - args: arg{ - ctx: context.Background(), - tableID: 1, - rows: []*model.RedoRowChangedEvent{ - { - Row: &model.RowChangedEvent{ - Table: &model.TableName{TableID: 111}, CommitTs: 1, - }, - }, - }, - }, - isRunning: true, - writerErr: nil, - }, - { - name: "writer err", - args: arg{ - ctx: context.Background(), - tableID: 1, - rows: []*model.RedoRowChangedEvent{ - {Row: nil}, - { - Row: &model.RowChangedEvent{ - Table: &model.TableName{TableID: 11}, CommitTs: 11, - }, - }, - }, - }, - writerErr: errors.New("err"), - wantErr: errors.New("err"), - isRunning: true, - }, - { - name: "len(rows)==0", - args: arg{ - ctx: context.Background(), - tableID: 1, - rows: []*model.RedoRowChangedEvent{}, - }, - writerErr: errors.New("err"), - isRunning: true, - }, - { - name: "isStopped", - args: arg{ - ctx: context.Background(), - tableID: 1, - rows: []*model.RedoRowChangedEvent{}, - }, - writerErr: cerror.ErrRedoWriterStopped, - isRunning: false, - wantErr: cerror.ErrRedoWriterStopped, - }, - { - name: "context cancel", - args: arg{ - ctx: context.Background(), - tableID: 1, - rows: []*model.RedoRowChangedEvent{}, - }, - writerErr: nil, - isRunning: true, - wantErr: context.Canceled, - }, - } - - for _, tt := range tests { - mockWriter := &mockFileWriter{} - mockWriter.On("Write", mock.Anything).Return(1, tt.writerErr) - mockWriter.On("IsRunning").Return(tt.isRunning) - mockWriter.On("AdvanceTs", mock.Anything) - writer := LogWriter{ - rowWriter: mockWriter, - ddlWriter: mockWriter, - meta: &common.LogMeta{ResolvedTsList: map[int64]uint64{}}, - metricTotalRowsCount: redoTotalRowsCountGauge.WithLabelValues("", ""), - } - if tt.name == "context cancel" { - ctx, cancel := context.WithCancel(context.Background()) - cancel() - tt.args.ctx = ctx - } - - _, err := writer.WriteLog(tt.args.ctx, tt.args.tableID, tt.args.rows) - if tt.wantErr != nil { - require.Truef(t, errors.ErrorEqual(tt.wantErr, err), tt.name) - } else { - require.Nil(t, err, tt.name) - } - } -} - -func TestLogWriterSendDDL(t *testing.T) { - type arg struct { - ctx context.Context - tableID int64 - ddl *model.RedoDDLEvent - } - tests := []struct { - name string - args arg - wantTs uint64 - isRunning bool - writerErr error - wantErr error - }{ - { - name: "happy", - args: arg{ - ctx: context.Background(), - tableID: 1, - ddl: &model.RedoDDLEvent{DDL: &model.DDLEvent{CommitTs: 1}}, - }, - isRunning: true, - writerErr: nil, - }, - { - name: "writer err", - args: arg{ - ctx: context.Background(), - tableID: 1, - ddl: &model.RedoDDLEvent{DDL: &model.DDLEvent{CommitTs: 1}}, - }, - writerErr: errors.New("err"), - wantErr: errors.New("err"), - isRunning: true, - }, - { - name: "ddl nil", - args: arg{ - ctx: context.Background(), - tableID: 1, - ddl: nil, - }, - writerErr: errors.New("err"), - isRunning: true, - }, - { - name: "isStopped", - args: arg{ - ctx: context.Background(), - tableID: 1, - ddl: &model.RedoDDLEvent{DDL: &model.DDLEvent{CommitTs: 1}}, - }, - writerErr: cerror.ErrRedoWriterStopped, - isRunning: false, - wantErr: cerror.ErrRedoWriterStopped, - }, - { - name: "context cancel", - args: arg{ - ctx: context.Background(), - tableID: 1, - ddl: &model.RedoDDLEvent{DDL: &model.DDLEvent{CommitTs: 1}}, - }, - writerErr: nil, - isRunning: true, - wantErr: context.Canceled, - }, - } - - for _, tt := range tests { - mockWriter := &mockFileWriter{} - mockWriter.On("Write", mock.Anything).Return(1, tt.writerErr) - mockWriter.On("IsRunning").Return(tt.isRunning) - mockWriter.On("AdvanceTs", mock.Anything) - writer := LogWriter{ - rowWriter: mockWriter, - ddlWriter: mockWriter, - meta: &common.LogMeta{ResolvedTsList: map[int64]uint64{}}, - } - - if tt.name == "context cancel" { - ctx, cancel := context.WithCancel(context.Background()) - cancel() - tt.args.ctx = ctx - } - - err := writer.SendDDL(tt.args.ctx, tt.args.ddl) - if tt.wantErr != nil { - require.True(t, errors.ErrorEqual(tt.wantErr, err), tt.name) - } else { - require.Nil(t, err, tt.name) - } - } -} - -func TestLogWriterFlushLog(t *testing.T) { - type arg struct { - ctx context.Context - tableID int64 - ts uint64 - } - tests := []struct { - name string - args arg - wantTs uint64 - isRunning bool - flushErr error - wantErr error - }{ - { - name: "happy", - args: arg{ - ctx: context.Background(), - tableID: 1, - ts: 1, - }, - isRunning: true, - flushErr: nil, - }, - { - name: "flush err", - args: arg{ - ctx: context.Background(), - tableID: 1, - ts: 1, - }, - flushErr: errors.New("err"), - wantErr: multierr.Append(errors.New("err"), errors.New("err")), - isRunning: true, - }, - { - name: "isStopped", - args: arg{ - ctx: context.Background(), - tableID: 1, - ts: 1, - }, - flushErr: cerror.ErrRedoWriterStopped, - isRunning: false, - wantErr: cerror.ErrRedoWriterStopped, - }, - { - name: "context cancel", - args: arg{ - ctx: context.Background(), - tableID: 1, - ts: 1, - }, - flushErr: nil, - isRunning: true, - wantErr: context.Canceled, - }, - } - - dir, err := ioutil.TempDir("", "redo-FlushLog") - require.Nil(t, err) - defer os.RemoveAll(dir) - - for _, tt := range tests { - controller := gomock.NewController(t) - mockStorage := mockstorage.NewMockExternalStorage(controller) - if tt.isRunning && tt.name != "context cancel" { - mockStorage.EXPECT().WriteFile(gomock.Any(), "cp_test-cf_meta.meta", gomock.Any()).Return(nil).Times(1) - } - mockWriter := &mockFileWriter{} - mockWriter.On("Flush", mock.Anything).Return(tt.flushErr) - mockWriter.On("IsRunning").Return(tt.isRunning) - cfg := &LogWriterConfig{ - Dir: dir, - ChangeFeedID: "test-cf", - CaptureID: "cp", - MaxLogSize: 10, - CreateTime: time.Date(2000, 1, 1, 1, 1, 1, 1, &time.Location{}), - FlushIntervalInMs: 5, - S3Storage: true, - } - writer := LogWriter{ - rowWriter: mockWriter, - ddlWriter: mockWriter, - meta: &common.LogMeta{ResolvedTsList: map[int64]uint64{}}, - cfg: cfg, - storage: mockStorage, - } - - if tt.name == "context cancel" { - ctx, cancel := context.WithCancel(context.Background()) - cancel() - tt.args.ctx = ctx - } - err := writer.FlushLog(tt.args.ctx, tt.args.tableID, tt.args.ts) - if tt.wantErr != nil { - require.True(t, errors.ErrorEqual(tt.wantErr, err), err.Error()+tt.wantErr.Error()) - } else { - require.Nil(t, err, tt.name) - require.Equal(t, tt.args.ts, writer.meta.ResolvedTsList[tt.args.tableID], tt.name) - } - } -} - -func TestLogWriterEmitCheckpointTs(t *testing.T) { - type arg struct { - ctx context.Context - ts uint64 - } - tests := []struct { - name string - args arg - wantTs uint64 - isRunning bool - flushErr error - wantErr error - }{ - { - name: "happy", - args: arg{ - ctx: context.Background(), - ts: 1, - }, - isRunning: true, - flushErr: nil, - }, - { - name: "isStopped", - args: arg{ - ctx: context.Background(), - ts: 1, - }, - flushErr: cerror.ErrRedoWriterStopped, - isRunning: false, - wantErr: cerror.ErrRedoWriterStopped, - }, - { - name: "context cancel", - args: arg{ - ctx: context.Background(), - ts: 1, - }, - flushErr: nil, - isRunning: true, - wantErr: context.Canceled, - }, - } - - dir, err := ioutil.TempDir("", "redo-EmitCheckpointTs") - require.Nil(t, err) - defer os.RemoveAll(dir) - - for _, tt := range tests { - controller := gomock.NewController(t) - mockStorage := mockstorage.NewMockExternalStorage(controller) - if tt.isRunning && tt.name != "context cancel" { - mockStorage.EXPECT().WriteFile(gomock.Any(), "cp_test-cf_meta.meta", gomock.Any()).Return(nil).Times(1) - } - - mockWriter := &mockFileWriter{} - mockWriter.On("IsRunning").Return(tt.isRunning) - cfg := &LogWriterConfig{ - Dir: dir, - ChangeFeedID: "test-cf", - CaptureID: "cp", - MaxLogSize: 10, - CreateTime: time.Date(2000, 1, 1, 1, 1, 1, 1, &time.Location{}), - FlushIntervalInMs: 5, - S3Storage: true, - } - writer := LogWriter{ - rowWriter: mockWriter, - ddlWriter: mockWriter, - meta: &common.LogMeta{ResolvedTsList: map[int64]uint64{}}, - cfg: cfg, - storage: mockStorage, - } - - if tt.name == "context cancel" { - ctx, cancel := context.WithCancel(context.Background()) - cancel() - tt.args.ctx = ctx - } - err := writer.EmitCheckpointTs(tt.args.ctx, tt.args.ts) - if tt.wantErr != nil { - require.True(t, errors.ErrorEqual(tt.wantErr, err), tt.name) - } else { - require.Nil(t, err, tt.name) - require.Equal(t, tt.args.ts, writer.meta.CheckPointTs, tt.name) - } - } -} - -func TestLogWriterEmitResolvedTs(t *testing.T) { - type arg struct { - ctx context.Context - - ts uint64 - } - tests := []struct { - name string - args arg - wantTs uint64 - isRunning bool - flushErr error - wantErr error - }{ - { - name: "happy", - args: arg{ - ctx: context.Background(), - ts: 1, - }, - isRunning: true, - flushErr: nil, - }, - { - name: "isStopped", - args: arg{ - ctx: context.Background(), - ts: 1, - }, - flushErr: cerror.ErrRedoWriterStopped, - isRunning: false, - wantErr: cerror.ErrRedoWriterStopped, - }, - { - name: "context cancel", - args: arg{ - ctx: context.Background(), - ts: 1, - }, - flushErr: nil, - isRunning: true, - wantErr: context.Canceled, - }, - } - - dir, err := ioutil.TempDir("", "redo-ResolvedTs") - require.Nil(t, err) - defer os.RemoveAll(dir) - - for _, tt := range tests { - controller := gomock.NewController(t) - mockStorage := mockstorage.NewMockExternalStorage(controller) - if tt.isRunning && tt.name != "context cancel" { - mockStorage.EXPECT().WriteFile(gomock.Any(), "cp_test-cf_meta.meta", gomock.Any()).Return(nil).Times(1) - } - mockWriter := &mockFileWriter{} - mockWriter.On("IsRunning").Return(tt.isRunning) - cfg := &LogWriterConfig{ - Dir: dir, - ChangeFeedID: "test-cf", - CaptureID: "cp", - MaxLogSize: 10, - CreateTime: time.Date(2000, 1, 1, 1, 1, 1, 1, &time.Location{}), - FlushIntervalInMs: 5, - S3Storage: true, - } - writer := LogWriter{ - rowWriter: mockWriter, - ddlWriter: mockWriter, - meta: &common.LogMeta{ResolvedTsList: map[int64]uint64{}}, - cfg: cfg, - storage: mockStorage, - } - - if tt.name == "context cancel" { - ctx, cancel := context.WithCancel(context.Background()) - cancel() - tt.args.ctx = ctx - } - err := writer.EmitResolvedTs(tt.args.ctx, tt.args.ts) - if tt.wantErr != nil { - require.True(t, errors.ErrorEqual(tt.wantErr, err), tt.name) - } else { - require.Nil(t, err, tt.name) - require.Equal(t, tt.args.ts, writer.meta.ResolvedTs, tt.name) - } - } -} - -func TestLogWriterGetCurrentResolvedTs(t *testing.T) { - type arg struct { - ctx context.Context - ts map[int64]uint64 - tableIDs []int64 - } - tests := []struct { - name string - args arg - wantTs map[int64]uint64 - wantErr error - }{ - { - name: "happy", - args: arg{ - ctx: context.Background(), - ts: map[int64]uint64{1: 1, 2: 2}, - tableIDs: []int64{1, 2, 3}, - }, - wantTs: map[int64]uint64{1: 1, 2: 2}, - }, - { - name: "len(tableIDs)==0", - args: arg{ - ctx: context.Background(), - }, - }, - { - name: "context cancel", - args: arg{ - ctx: context.Background(), - }, - wantErr: context.Canceled, - }, - } - - dir, err := ioutil.TempDir("", "redo-GetCurrentResolvedTs") - require.Nil(t, err) - defer os.RemoveAll(dir) - - for _, tt := range tests { - mockWriter := &mockFileWriter{} - mockWriter.On("Flush", mock.Anything).Return(nil) - mockWriter.On("IsRunning").Return(true) - cfg := &LogWriterConfig{ - Dir: dir, - ChangeFeedID: "test-cf", - CaptureID: "cp", - MaxLogSize: 10, - CreateTime: time.Date(2000, 1, 1, 1, 1, 1, 1, &time.Location{}), - FlushIntervalInMs: 5, - } - writer := LogWriter{ - rowWriter: mockWriter, - ddlWriter: mockWriter, - meta: &common.LogMeta{ResolvedTsList: map[int64]uint64{}}, - cfg: cfg, - } - - if tt.name == "context cancel" { - ctx, cancel := context.WithCancel(context.Background()) - cancel() - tt.args.ctx = ctx - } - for k, v := range tt.args.ts { - _ = writer.FlushLog(tt.args.ctx, k, v) - } - ret, err := writer.GetCurrentResolvedTs(tt.args.ctx, tt.args.tableIDs) - if tt.wantErr != nil { - require.True(t, errors.ErrorEqual(tt.wantErr, err), tt.name, err.Error()) - } else { - require.Nil(t, err, tt.name) - require.Equal(t, len(ret), len(tt.wantTs)) - for k, v := range tt.wantTs { - require.Equal(t, v, ret[k]) - } - } - } -} - -func TestNewLogWriter(t *testing.T) { - _, err := NewLogWriter(context.Background(), nil) - require.NotNil(t, err) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - cfg := &LogWriterConfig{ - Dir: "dirt", - ChangeFeedID: "test-cf", - CaptureID: "cp", - MaxLogSize: 10, - CreateTime: time.Date(2000, 1, 1, 1, 1, 1, 1, &time.Location{}), - FlushIntervalInMs: 5, - } - ll, err := NewLogWriter(ctx, cfg) - require.Nil(t, err) - time.Sleep(time.Duration(defaultGCIntervalInMs+1) * time.Millisecond) - require.Equal(t, map[int64]uint64{}, ll.meta.ResolvedTsList) - - ll2, err := NewLogWriter(ctx, cfg) - require.Nil(t, err) - require.Same(t, ll, ll2) - - cfg1 := &LogWriterConfig{ - Dir: "dirt111", - ChangeFeedID: "test-cf", - CaptureID: "cp", - MaxLogSize: 10, - CreateTime: time.Date(2000, 1, 1, 1, 1, 1, 1, &time.Location{}), - FlushIntervalInMs: 5, - } - ll1, err := NewLogWriter(ctx, cfg1) - require.Nil(t, err) - require.NotSame(t, ll, ll1) - - ll2, err = NewLogWriter(ctx, cfg) - require.Nil(t, err) - require.NotSame(t, ll, ll2) - - dir, err := ioutil.TempDir("", "redo-NewLogWriter") - require.Nil(t, err) - defer os.RemoveAll(dir) - fileName := fmt.Sprintf("%s_%s_%d_%s%s", "cp", "test-changefeed", time.Now().Unix(), common.DefaultMetaFileType, common.MetaEXT) - path := filepath.Join(dir, fileName) - f, err := os.Create(path) - require.Nil(t, err) - - meta := &common.LogMeta{ - CheckPointTs: 11, - ResolvedTs: 22, - } - data, err := meta.MarshalMsg(nil) - require.Nil(t, err) - _, err = f.Write(data) - require.Nil(t, err) - - cfg = &LogWriterConfig{ - Dir: dir, - ChangeFeedID: "test-cf", - CaptureID: "cp", - MaxLogSize: 10, - CreateTime: time.Date(2000, 1, 1, 1, 1, 1, 1, &time.Location{}), - FlushIntervalInMs: 5, - } - l, err := NewLogWriter(ctx, cfg) - require.Nil(t, err) - err = l.Close() - require.Nil(t, err) - require.True(t, l.isStopped()) - require.Equal(t, cfg.Dir, l.cfg.Dir) - require.Equal(t, meta.CheckPointTs, l.meta.CheckPointTs) - require.Equal(t, meta.ResolvedTs, l.meta.ResolvedTs) - require.Equal(t, map[int64]uint64{}, l.meta.ResolvedTsList) - time.Sleep(time.Millisecond * time.Duration(math.Max(float64(defaultFlushIntervalInMs), float64(defaultGCIntervalInMs))+1)) - - origin := common.InitS3storage - defer func() { - common.InitS3storage = origin - }() - controller := gomock.NewController(t) - mockStorage := mockstorage.NewMockExternalStorage(controller) - // skip pre cleanup - mockStorage.EXPECT().FileExists(gomock.Any(), gomock.Any()).Return(false, nil) - common.InitS3storage = func(ctx context.Context, uri url.URL) (storage.ExternalStorage, error) { - return mockStorage, nil - } - cfg3 := &LogWriterConfig{ - Dir: dir, - ChangeFeedID: "test-cf112232", - CaptureID: "cp", - MaxLogSize: 10, - CreateTime: time.Date(2000, 1, 1, 1, 1, 1, 1, &time.Location{}), - FlushIntervalInMs: 5, - S3Storage: true, - } - l3, err := NewLogWriter(ctx, cfg3) - require.Nil(t, err) - err = l3.Close() - require.Nil(t, err) -} - -func TestWriterRedoGC(t *testing.T) { - cfg := &LogWriterConfig{ - Dir: "dir", - ChangeFeedID: "test-cf", - CaptureID: "cp", - MaxLogSize: 10, - CreateTime: time.Date(2000, 1, 1, 1, 1, 1, 1, &time.Location{}), - FlushIntervalInMs: 5, - } - - type args struct { - isRunning bool - } - tests := []struct { - name string - args args - }{ - { - name: "running", - args: args{ - isRunning: true, - }, - }, - { - name: "stopped", - args: args{ - isRunning: false, - }, - }, - } - for _, tt := range tests { - mockWriter := &mockFileWriter{} - mockWriter.On("IsRunning").Return(tt.args.isRunning).Twice() - mockWriter.On("Close").Return(nil) - mockWriter.On("IsRunning").Return(false) - - if tt.args.isRunning { - mockWriter.On("GC", mock.Anything).Return(nil) - } - writer := LogWriter{ - rowWriter: mockWriter, - ddlWriter: mockWriter, - meta: &common.LogMeta{ResolvedTsList: map[int64]uint64{}}, - cfg: cfg, - } - go writer.runGC(context.Background()) - time.Sleep(time.Duration(defaultGCIntervalInMs+1) * time.Millisecond) - - writer.Close() - mockWriter.AssertNumberOfCalls(t, "Close", 2) - - if tt.args.isRunning { - mockWriter.AssertCalled(t, "GC", mock.Anything) - } else { - mockWriter.AssertNotCalled(t, "GC", mock.Anything) - } - } -} - -func TestDeleteAllLogs(t *testing.T) { - fileName := "1" - fileName1 := "11" - - type args struct { - enableS3 bool - } - - tests := []struct { - name string - args args - closeErr error - getAllFilesInS3Err error - deleteFileErr error - writeFileErr error - wantErr string - }{ - { - name: "happy local", - args: args{enableS3: false}, - }, - { - name: "happy s3", - args: args{enableS3: true}, - }, - { - name: "close err", - args: args{enableS3: true}, - closeErr: errors.New("xx"), - wantErr: ".*xx*.", - }, - { - name: "getAllFilesInS3 err", - args: args{enableS3: true}, - getAllFilesInS3Err: errors.New("xx"), - wantErr: ".*xx*.", - }, - { - name: "deleteFile normal err", - args: args{enableS3: true}, - deleteFileErr: errors.New("xx"), - wantErr: ".*ErrS3StorageAPI*.", - }, - { - name: "deleteFile notExist err", - args: args{enableS3: true}, - deleteFileErr: awserr.New(s3.ErrCodeNoSuchKey, "no such key", nil), - }, - { - name: "writerFile err", - args: args{enableS3: true}, - writeFileErr: errors.New("xx"), - wantErr: ".*xx*.", - }, - } - - for _, tt := range tests { - dir, err := ioutil.TempDir("", "redo-DeleteAllLogs") - require.Nil(t, err) - path := filepath.Join(dir, fileName) - _, err = os.Create(path) - require.Nil(t, err) - path = filepath.Join(dir, fileName1) - _, err = os.Create(path) - require.Nil(t, err) - - origin := getAllFilesInS3 - getAllFilesInS3 = func(ctx context.Context, l *LogWriter) ([]string, error) { - return []string{fileName, fileName1}, tt.getAllFilesInS3Err - } - controller := gomock.NewController(t) - mockStorage := mockstorage.NewMockExternalStorage(controller) - - mockStorage.EXPECT().DeleteFile(gomock.Any(), gomock.Any()).Return(tt.deleteFileErr).MaxTimes(2) - mockStorage.EXPECT().WriteFile(gomock.Any(), gomock.Any(), gomock.Any()).Return(tt.writeFileErr).MaxTimes(1) - - mockWriter := &mockFileWriter{} - mockWriter.On("Close").Return(tt.closeErr) - cfg := &LogWriterConfig{ - Dir: dir, - ChangeFeedID: "test-cf", - CaptureID: "cp", - MaxLogSize: 10, - CreateTime: time.Date(2000, 1, 1, 1, 1, 1, 1, &time.Location{}), - FlushIntervalInMs: 5, - S3Storage: tt.args.enableS3, - } - writer := LogWriter{ - rowWriter: mockWriter, - ddlWriter: mockWriter, - meta: &common.LogMeta{ResolvedTsList: map[int64]uint64{}}, - cfg: cfg, - storage: mockStorage, - } - if strings.Contains(tt.name, "happy") { - logWriters[writer.cfg.ChangeFeedID] = &writer - } - ret := writer.DeleteAllLogs(context.Background()) - if tt.wantErr != "" { - require.Regexp(t, tt.wantErr, ret.Error(), tt.name) - } else { - require.Nil(t, ret, tt.name) - _, ok := logWriters[writer.cfg.ChangeFeedID] - require.False(t, ok, tt.name) - if !tt.args.enableS3 { - _, err := os.Stat(dir) - require.True(t, os.IsNotExist(err), tt.name) - } - } - os.RemoveAll(dir) - getAllFilesInS3 = origin - } -} - -func TestPreCleanUpS3(t *testing.T) { - testCases := []struct { - name string - fileExistsErr error - fileExists bool - getAllFilesInS3Err error - deleteFileErr error - wantErr string - }{ - { - name: "happy no marker", - fileExists: false, - }, - { - name: "fileExists err", - fileExistsErr: errors.New("xx"), - wantErr: ".*xx*.", - }, - { - name: "getAllFilesInS3 err", - fileExists: true, - getAllFilesInS3Err: errors.New("xx"), - wantErr: ".*xx*.", - }, - { - name: "deleteFile normal err", - fileExists: true, - deleteFileErr: errors.New("xx"), - wantErr: ".*ErrS3StorageAPI*.", - }, - { - name: "deleteFile notExist err", - fileExists: true, - deleteFileErr: awserr.New(s3.ErrCodeNoSuchKey, "no such key", nil), - }, - } - - for _, tc := range testCases { - origin := getAllFilesInS3 - getAllFilesInS3 = func(ctx context.Context, l *LogWriter) ([]string, error) { - return []string{"1", "11", "delete_test-cf"}, tc.getAllFilesInS3Err - } - controller := gomock.NewController(t) - mockStorage := mockstorage.NewMockExternalStorage(controller) - - mockStorage.EXPECT().FileExists(gomock.Any(), gomock.Any()).Return(tc.fileExists, tc.fileExistsErr) - mockStorage.EXPECT().DeleteFile(gomock.Any(), gomock.Any()).Return(tc.deleteFileErr).MaxTimes(3) - - cfg := &LogWriterConfig{ - Dir: "dir", - ChangeFeedID: "test-cf", - CaptureID: "cp", - MaxLogSize: 10, - CreateTime: time.Date(2000, 1, 1, 1, 1, 1, 1, &time.Location{}), - FlushIntervalInMs: 5, - } - writer := LogWriter{ - cfg: cfg, - storage: mockStorage, - } - ret := writer.preCleanUpS3(context.Background()) - if tc.wantErr != "" { - require.Regexp(t, tc.wantErr, ret.Error(), tc.name) - } else { - require.Nil(t, ret, tc.name) - } - getAllFilesInS3 = origin - } -} diff --git a/cdc/cdc/scheduler/agent.go b/cdc/cdc/scheduler/agent.go index 855ec747..b4c3c09e 100644 --- a/cdc/cdc/scheduler/agent.go +++ b/cdc/cdc/scheduler/agent.go @@ -39,28 +39,29 @@ type Agent interface { GetLastSentCheckpointTs() (checkpointTs model.Ts) } -// TableExecutor is an abstraction for "Processor". +// KeySpanExecutor is an abstraction for "Processor". // // This interface is so designed that it would be the least problematic // to adapt the current Processor implementation to it. // TODO find a way to make the semantics easier to understand. -type TableExecutor interface { - AddTable(ctx context.Context, tableID model.TableID) (done bool, err error) - RemoveTable(ctx context.Context, tableID model.TableID) (done bool, err error) - IsAddTableFinished(ctx context.Context, tableID model.TableID) (done bool) - IsRemoveTableFinished(ctx context.Context, tableID model.TableID) (done bool) - - // GetAllCurrentTables should return all tables that are being run, +// TODO: modify +type KeySpanExecutor interface { + AddKeySpan(ctx context.Context, keyspanID model.KeySpanID, start []byte, end []byte) (done bool, err error) + RemoveKeySpan(ctx context.Context, keyspanID model.KeySpanID) (done bool, err error) + IsAddKeySpanFinished(ctx context.Context, keyspanID model.KeySpanID) (done bool) + IsRemoveKeySpanFinished(ctx context.Context, keyspanID model.KeySpanID) (done bool) + + // GetAllCurrentKeySpans should return all keyspans that are being run, // being added and being removed. // // NOTE: two subsequent calls to the method should return the same - // result, unless there is a call to AddTable, RemoveTable, IsAddTableFinished - // or IsRemoveTableFinished in between two calls to this method. - GetAllCurrentTables() []model.TableID + // result, unless there is a call to AddKeySpan, RemoveKeySpan, IsAddKeySpanFinished + // or IsRemoveKeySpanFinished in between two calls to this method. + GetAllCurrentKeySpans() []model.KeySpanID // GetCheckpoint returns the local checkpoint-ts and resolved-ts of // the processor. Its calculation should take into consideration all - // tables that would have been returned if GetAllCurrentTables had been + // keyspans that would have been returned if GetAllCurrentKeySpans had been // called immediately before. GetCheckpoint() (checkpointTs, resolvedTs model.Ts) } @@ -69,10 +70,10 @@ type TableExecutor interface { // and should be able to know whether there are any messages not yet acknowledged // by the owner. type ProcessorMessenger interface { - // FinishTableOperation notifies the owner that a table operation has finished. - FinishTableOperation(ctx context.Context, tableID model.TableID) (done bool, err error) + // FinishKeySpanOperation notifies the owner that a keyspan operation has finished. + FinishKeySpanOperation(ctx context.Context, keyspanID model.KeySpanID) (done bool, err error) // SyncTaskStatuses informs the owner of the processor's current internal state. - SyncTaskStatuses(ctx context.Context, running, adding, removing []model.TableID) (done bool, err error) + SyncTaskStatuses(ctx context.Context, running, adding, removing []model.KeySpanID) (done bool, err error) // SendCheckpoint sends the owner the processor's local watermarks, i.e., checkpoint-ts and resolved-ts. SendCheckpoint(ctx context.Context, checkpointTs model.Ts, resolvedTs model.Ts) (done bool, err error) @@ -92,9 +93,9 @@ type BaseAgentConfig struct { // BaseAgent is an implementation of Agent. // It implements the basic logic and is useful only if the Processor -// implements its own TableExecutor and ProcessorMessenger. +// implements its own KeySpanExecutor and ProcessorMessenger. type BaseAgent struct { - executor TableExecutor + executor KeySpanExecutor communicator ProcessorMessenger // pendingOpsMu protects pendingOps. @@ -105,9 +106,9 @@ type BaseAgent struct { // the Deque stores *agentOperation. pendingOps deque.Deque - // tableOperations is a map from tableID to the operation + // keyspanOperations is a map from keyspanID to the operation // that is currently being processed. - tableOperations map[model.TableID]*agentOperation + keyspanOperations map[model.KeySpanID]*agentOperation // needSyncNow indicates that the agent needs to send the // current owner a sync message as soon as possible. @@ -131,22 +132,22 @@ type BaseAgent struct { // NewBaseAgent creates a new BaseAgent. func NewBaseAgent( changeFeedID model.ChangeFeedID, - executor TableExecutor, + executor KeySpanExecutor, messenger ProcessorMessenger, config *BaseAgentConfig, ) *BaseAgent { logger := log.L().With(zap.String("changefeed-id", changeFeedID)) return &BaseAgent{ - pendingOps: deque.NewDeque(), - tableOperations: map[model.TableID]*agentOperation{}, - logger: logger, - executor: executor, - ownerInfo: &ownerInfo{}, - communicator: messenger, - needSyncNow: atomic.NewBool(true), - checkpointSender: newCheckpointSender(messenger, logger, config.SendCheckpointTsInterval), - ownerHasChanged: atomic.NewBool(false), - config: config, + pendingOps: deque.NewDeque(), + keyspanOperations: map[model.KeySpanID]*agentOperation{}, + logger: logger, + executor: executor, + ownerInfo: &ownerInfo{}, + communicator: messenger, + needSyncNow: atomic.NewBool(true), + checkpointSender: newCheckpointSender(messenger, logger, config.SendCheckpointTsInterval), + ownerHasChanged: atomic.NewBool(false), + config: config, } } @@ -159,8 +160,10 @@ const ( ) type agentOperation struct { - TableID model.TableID - IsDelete bool + KeySpanID model.KeySpanID + IsDelete bool + Start []byte + End []byte status agentOperationStatus } @@ -204,11 +207,11 @@ func (a *BaseAgent) Tick(ctx context.Context) error { opsToApply := a.popPendingOps() for _, op := range opsToApply { - if _, ok := a.tableOperations[op.TableID]; ok { + if _, ok := a.keyspanOperations[op.KeySpanID]; ok { a.logger.DPanic("duplicate operation", zap.Any("op", op)) - return cerrors.ErrProcessorDuplicateOperations.GenWithStackByArgs(op.TableID) + return cerrors.ErrProcessorDuplicateOperations.GenWithStackByArgs(op.KeySpanID) } - a.tableOperations[op.TableID] = op + a.keyspanOperations[op.KeySpanID] = op } if err := a.processOperations(ctx); err != nil { @@ -238,27 +241,27 @@ func (a *BaseAgent) popPendingOps() (opsToApply []*agentOperation) { // sendSync needs to be called with a.pendingOpsMu held. func (a *BaseAgent) sendSync(ctx context.Context) (bool, error) { - var adding, removing, running []model.TableID - for _, op := range a.tableOperations { + var adding, removing, running []model.KeySpanID + for _, op := range a.keyspanOperations { if !op.IsDelete { - adding = append(adding, op.TableID) + adding = append(adding, op.KeySpanID) } else { - removing = append(removing, op.TableID) + removing = append(removing, op.KeySpanID) } } - for _, tableID := range a.executor.GetAllCurrentTables() { - if _, ok := a.tableOperations[tableID]; ok { - // Tables with a pending operation is not in the Running state. + for _, keyspanID := range a.executor.GetAllCurrentKeySpans() { + if _, ok := a.keyspanOperations[keyspanID]; ok { + // KeySpans with a pending operation is not in the Running state. continue } - running = append(running, tableID) + running = append(running, keyspanID) } - // We are sorting these so that there content can be predictable in tests. + // We are sorting these so that there content can be predickeysapn in tests. // TODO try to find a better way. - util.SortTableIDs(running) - util.SortTableIDs(adding) - util.SortTableIDs(removing) + util.SortKeySpanIDs(running) + util.SortKeySpanIDs(adding) + util.SortKeySpanIDs(removing) done, err := a.communicator.SyncTaskStatuses(ctx, running, adding, removing) if err != nil { return false, errors.Trace(err) @@ -266,15 +269,15 @@ func (a *BaseAgent) sendSync(ctx context.Context) (bool, error) { return done, nil } -// processOperations tries to make progress on each pending table operations. -// It queries the executor for the current status of each table. +// processOperations tries to make progress on each pending keyspan operations. +// It queries the executor for the current status of each keyspan. func (a *BaseAgent) processOperations(ctx context.Context) error { - for tableID, op := range a.tableOperations { + for keyspanID, op := range a.keyspanOperations { switch op.status { case operationReceived: if !op.IsDelete { - // add table - done, err := a.executor.AddTable(ctx, op.TableID) + // add keyspan + done, err := a.executor.AddKeySpan(ctx, op.KeySpanID, op.Start, op.End) if err != nil { return errors.Trace(err) } @@ -282,8 +285,8 @@ func (a *BaseAgent) processOperations(ctx context.Context) error { break } } else { - // delete table - done, err := a.executor.RemoveTable(ctx, op.TableID) + // delete keyspan + done, err := a.executor.RemoveKeySpan(ctx, op.KeySpanID) if err != nil { return errors.Trace(err) } @@ -296,9 +299,9 @@ func (a *BaseAgent) processOperations(ctx context.Context) error { case operationProcessed: var done bool if !op.IsDelete { - done = a.executor.IsAddTableFinished(ctx, op.TableID) + done = a.executor.IsAddKeySpanFinished(ctx, op.KeySpanID) } else { - done = a.executor.IsRemoveTableFinished(ctx, op.TableID) + done = a.executor.IsRemoveKeySpanFinished(ctx, op.KeySpanID) } if !done { break @@ -306,12 +309,12 @@ func (a *BaseAgent) processOperations(ctx context.Context) error { op.status = operationFinished fallthrough case operationFinished: - done, err := a.communicator.FinishTableOperation(ctx, op.TableID) + done, err := a.communicator.FinishKeySpanOperation(ctx, op.KeySpanID) if err != nil { return errors.Trace(err) } if done { - delete(a.tableOperations, tableID) + delete(a.keyspanOperations, keyspanID) } } } @@ -320,9 +323,9 @@ func (a *BaseAgent) processOperations(ctx context.Context) error { func (a *BaseAgent) sendCheckpoint(ctx context.Context) error { checkpointProvider := func() (checkpointTs, resolvedTs model.Ts, ok bool) { - // We cannot have a meaningful checkpoint for a processor running NO table. - if len(a.executor.GetAllCurrentTables()) == 0 { - a.logger.Debug("no table is running, skip sending checkpoint") + // We cannot have a meaningful checkpoint for a processor running NO keyspan. + if len(a.executor.GetAllCurrentKeySpans()) == 0 { + a.logger.Debug("no keyspan is running, skip sending checkpoint") return 0, 0, false // false indicates no available checkpoint } checkpointTs, resolvedTs = a.executor.GetCheckpoint() @@ -341,12 +344,14 @@ func (a *BaseAgent) sendCheckpoint(ctx context.Context) error { func (a *BaseAgent) OnOwnerDispatchedTask( ownerCaptureID model.CaptureID, ownerRev int64, - tableID model.TableID, + keyspanID model.KeySpanID, + start []byte, + end []byte, isDelete bool, ) { if !a.updateOwnerInfo(ownerCaptureID, ownerRev) { a.logger.Info("task from stale owner ignored", - zap.Int64("table-id", tableID), + zap.Uint64("keyspan-id", keyspanID), zap.Bool("is-delete", isDelete)) return } @@ -355,9 +360,11 @@ func (a *BaseAgent) OnOwnerDispatchedTask( defer a.pendingOpsMu.Unlock() op := &agentOperation{ - TableID: tableID, - IsDelete: isDelete, - status: operationReceived, + KeySpanID: keyspanID, + IsDelete: isDelete, + Start: start, + End: end, + status: operationReceived, } a.pendingOps.PushBack(op) diff --git a/cdc/cdc/scheduler/agent_mock.go b/cdc/cdc/scheduler/agent_mock.go index 1603b17e..ccc95eb4 100644 --- a/cdc/cdc/scheduler/agent_mock.go +++ b/cdc/cdc/scheduler/agent_mock.go @@ -28,12 +28,12 @@ type MockProcessorMessenger struct { mock.Mock } -func (m *MockProcessorMessenger) FinishTableOperation(ctx cdcContext.Context, tableID model.TableID) (bool, error) { - args := m.Called(ctx, tableID) +func (m *MockProcessorMessenger) FinishKeySpanOperation(ctx cdcContext.Context, keyspanID model.KeySpanID) (bool, error) { + args := m.Called(ctx, keyspanID) return args.Bool(0), args.Error(1) } -func (m *MockProcessorMessenger) SyncTaskStatuses(ctx cdcContext.Context, running, adding, removing []model.TableID) (bool, error) { +func (m *MockProcessorMessenger) SyncTaskStatuses(ctx cdcContext.Context, running, adding, removing []model.KeySpanID) (bool, error) { args := m.Called(ctx, running, adding, removing) return args.Bool(0), args.Error(1) } @@ -76,72 +76,72 @@ func (s *MockCheckpointSender) LastSentCheckpointTs() model.Ts { return s.lastSentCheckpointTs } -type MockTableExecutor struct { +type MockKeySpanExecutor struct { mock.Mock t *testing.T - Adding, Running, Removing map[model.TableID]struct{} + Adding, Running, Removing map[model.KeySpanID]struct{} } -func NewMockTableExecutor(t *testing.T) *MockTableExecutor { - return &MockTableExecutor{ +func NewMockKeySpanExecutor(t *testing.T) *MockKeySpanExecutor { + return &MockKeySpanExecutor{ t: t, - Adding: map[model.TableID]struct{}{}, - Running: map[model.TableID]struct{}{}, - Removing: map[model.TableID]struct{}{}, + Adding: map[model.KeySpanID]struct{}{}, + Running: map[model.KeySpanID]struct{}{}, + Removing: map[model.KeySpanID]struct{}{}, } } -func (e *MockTableExecutor) AddTable(ctx cdcContext.Context, tableID model.TableID) (bool, error) { - log.Info("AddTable", zap.Int64("table-id", tableID)) - require.NotContains(e.t, e.Adding, tableID) - require.NotContains(e.t, e.Running, tableID) - require.NotContains(e.t, e.Removing, tableID) - args := e.Called(ctx, tableID) +func (e *MockKeySpanExecutor) AddKeySpan(ctx cdcContext.Context, keyspanID model.KeySpanID, Start []byte, End []byte) (bool, error) { + log.Info("AddKeySpan", zap.Uint64("keyspan-id", keyspanID)) + require.NotContains(e.t, e.Adding, keyspanID) + require.NotContains(e.t, e.Running, keyspanID) + require.NotContains(e.t, e.Removing, keyspanID) + args := e.Called(ctx, keyspanID) if args.Bool(0) { - // If the mock return value indicates a success, then we record the added table. - e.Adding[tableID] = struct{}{} + // If the mock return value indicates a success, then we record the added keyspan. + e.Adding[keyspanID] = struct{}{} } return args.Bool(0), args.Error(1) } -func (e *MockTableExecutor) RemoveTable(ctx cdcContext.Context, tableID model.TableID) (bool, error) { - log.Info("RemoveTable", zap.Int64("table-id", tableID)) - args := e.Called(ctx, tableID) - require.Contains(e.t, e.Running, tableID) - require.NotContains(e.t, e.Removing, tableID) - delete(e.Running, tableID) - e.Removing[tableID] = struct{}{} +func (e *MockKeySpanExecutor) RemoveKeySpan(ctx cdcContext.Context, keyspanID model.KeySpanID) (bool, error) { + log.Info("RemoveKeySpan", zap.Uint64("keyspan-id", keyspanID)) + args := e.Called(ctx, keyspanID) + require.Contains(e.t, e.Running, keyspanID) + require.NotContains(e.t, e.Removing, keyspanID) + delete(e.Running, keyspanID) + e.Removing[keyspanID] = struct{}{} return args.Bool(0), args.Error(1) } -func (e *MockTableExecutor) IsAddTableFinished(ctx cdcContext.Context, tableID model.TableID) bool { - _, ok := e.Running[tableID] +func (e *MockKeySpanExecutor) IsAddKeySpanFinished(ctx cdcContext.Context, keyspanID model.KeySpanID) bool { + _, ok := e.Running[keyspanID] return ok } -func (e *MockTableExecutor) IsRemoveTableFinished(ctx cdcContext.Context, tableID model.TableID) bool { - _, ok := e.Removing[tableID] +func (e *MockKeySpanExecutor) IsRemoveKeySpanFinished(ctx cdcContext.Context, keyspanID model.KeySpanID) bool { + _, ok := e.Removing[keyspanID] return !ok } -func (e *MockTableExecutor) GetAllCurrentTables() []model.TableID { - var ret []model.TableID - for tableID := range e.Adding { - ret = append(ret, tableID) +func (e *MockKeySpanExecutor) GetAllCurrentKeySpans() []model.KeySpanID { + var ret []model.KeySpanID + for keyspanID := range e.Adding { + ret = append(ret, keyspanID) } - for tableID := range e.Running { - ret = append(ret, tableID) + for keyspanID := range e.Running { + ret = append(ret, keyspanID) } - for tableID := range e.Removing { - ret = append(ret, tableID) + for keyspanID := range e.Removing { + ret = append(ret, keyspanID) } return ret } -func (e *MockTableExecutor) GetCheckpoint() (checkpointTs, resolvedTs model.Ts) { +func (e *MockKeySpanExecutor) GetCheckpoint() (checkpointTs, resolvedTs model.Ts) { args := e.Called() return args.Get(0).(model.Ts), args.Get(1).(model.Ts) } diff --git a/cdc/cdc/scheduler/agent_test.go b/cdc/cdc/scheduler/agent_test.go index dc1a8e37..462aee06 100644 --- a/cdc/cdc/scheduler/agent_test.go +++ b/cdc/cdc/scheduler/agent_test.go @@ -25,22 +25,24 @@ import ( // read only var agentConfigForTesting = &BaseAgentConfig{SendCheckpointTsInterval: 0} -func TestAgentAddTable(t *testing.T) { +func TestAgentAddKeySpan(t *testing.T) { ctx := cdcContext.NewBackendContext4Test(false) - executor := NewMockTableExecutor(t) + executor := NewMockKeySpanExecutor(t) messenger := &MockProcessorMessenger{} agent := NewBaseAgent("test-cf", executor, messenger, agentConfigForTesting) - messenger.On("SyncTaskStatuses", mock.Anything, []model.TableID(nil), []model.TableID(nil), []model.TableID(nil)). + messenger.On("SyncTaskStatuses", mock.Anything, []model.KeySpanID(nil), []model.KeySpanID(nil), []model.KeySpanID(nil)). Return(true, nil) err := agent.Tick(ctx) require.NoError(t, err) messenger.AssertExpectations(t) + start, end := []byte{1}, []byte{5} + executor.ExpectedCalls = nil messenger.ExpectedCalls = nil - agent.OnOwnerDispatchedTask("capture-1", 1, model.TableID(1), false) - executor.On("AddTable", mock.Anything, model.TableID(1)).Return(true, nil) + agent.OnOwnerDispatchedTask("capture-1", 1, model.KeySpanID(1), start, end, false) + executor.On("AddKeySpan", mock.Anything, model.KeySpanID(1)).Return(true, nil) messenger.On("OnOwnerChanged", mock.Anything, "capture-1") err = agent.Tick(ctx) @@ -49,11 +51,11 @@ func TestAgentAddTable(t *testing.T) { executor.ExpectedCalls = nil messenger.ExpectedCalls = nil - delete(executor.Adding, model.TableID(1)) - executor.Running[model.TableID(1)] = struct{}{} + delete(executor.Adding, model.KeySpanID(1)) + executor.Running[model.KeySpanID(1)] = struct{}{} executor.On("GetCheckpoint").Return(model.Ts(1002), model.Ts(1000)) messenger.On("SendCheckpoint", mock.Anything, model.Ts(1002), model.Ts(1000)).Return(true, nil) - messenger.On("FinishTableOperation", mock.Anything, model.TableID(1)).Return(true, nil) + messenger.On("FinishKeySpanOperation", mock.Anything, model.KeySpanID(1)).Return(true, nil) err = agent.Tick(ctx) require.NoError(t, err) @@ -71,17 +73,17 @@ func TestAgentAddTable(t *testing.T) { messenger.AssertExpectations(t) } -func TestAgentRemoveTable(t *testing.T) { +func TestAgentRemoveKeySpan(t *testing.T) { ctx := cdcContext.NewBackendContext4Test(false) - executor := NewMockTableExecutor(t) - executor.Running[model.TableID(1)] = struct{}{} - executor.Running[model.TableID(2)] = struct{}{} + executor := NewMockKeySpanExecutor(t) + executor.Running[model.KeySpanID(1)] = struct{}{} + executor.Running[model.KeySpanID(2)] = struct{}{} messenger := &MockProcessorMessenger{} agent := NewBaseAgent("test-cf", executor, messenger, agentConfigForTesting) agent.OnOwnerAnnounce("capture-2", 1) - messenger.On("SyncTaskStatuses", mock.Anything, []model.TableID{1, 2}, []model.TableID(nil), []model.TableID(nil)). + messenger.On("SyncTaskStatuses", mock.Anything, []model.KeySpanID{1, 2}, []model.KeySpanID(nil), []model.KeySpanID(nil)). Return(true, nil) messenger.On("OnOwnerChanged", mock.Anything, "capture-2") executor.On("GetCheckpoint").Return(model.Ts(1000), model.Ts(1000)) @@ -90,12 +92,14 @@ func TestAgentRemoveTable(t *testing.T) { require.NoError(t, err) messenger.AssertExpectations(t) + start, end := []byte{1}, []byte{5} + executor.ExpectedCalls = nil messenger.ExpectedCalls = nil - agent.OnOwnerDispatchedTask("capture-2", 1, model.TableID(1), true) + agent.OnOwnerDispatchedTask("capture-2", 1, model.KeySpanID(1), start, end, true) executor.On("GetCheckpoint").Return(model.Ts(1000), model.Ts(1000)) messenger.On("SendCheckpoint", mock.Anything, model.Ts(1000), model.Ts(1000)).Return(true, nil) - executor.On("RemoveTable", mock.Anything, model.TableID(1)).Return(true, nil) + executor.On("RemoveKeySpan", mock.Anything, model.KeySpanID(1)).Return(true, nil) messenger.On("Barrier", mock.Anything).Return(true) err = agent.Tick(ctx) require.NoError(t, err) @@ -105,7 +109,7 @@ func TestAgentRemoveTable(t *testing.T) { executor.ExpectedCalls = nil messenger.ExpectedCalls = nil executor.On("GetCheckpoint").Return(model.Ts(1000), model.Ts(1000)) - messenger.On("SyncTaskStatuses", mock.Anything, []model.TableID{2}, []model.TableID(nil), []model.TableID{1}). + messenger.On("SyncTaskStatuses", mock.Anything, []model.KeySpanID{2}, []model.KeySpanID(nil), []model.KeySpanID{1}). Return(true, nil) messenger.On("OnOwnerChanged", mock.Anything, "capture-3") messenger.On("SendCheckpoint", mock.Anything, model.Ts(1000), model.Ts(1000)).Return(true, nil) @@ -117,10 +121,10 @@ func TestAgentRemoveTable(t *testing.T) { executor.ExpectedCalls = nil messenger.ExpectedCalls = nil - delete(executor.Removing, model.TableID(1)) + delete(executor.Removing, model.KeySpanID(1)) executor.On("GetCheckpoint").Return(model.Ts(1002), model.Ts(1000)) messenger.On("Barrier", mock.Anything).Return(true) - messenger.On("FinishTableOperation", mock.Anything, model.TableID(1)).Return(true, nil) + messenger.On("FinishKeySpanOperation", mock.Anything, model.KeySpanID(1)).Return(true, nil) messenger.On("SendCheckpoint", mock.Anything, model.Ts(1002), model.Ts(1000)).Return(true, nil) err = agent.Tick(ctx) @@ -128,20 +132,21 @@ func TestAgentRemoveTable(t *testing.T) { messenger.AssertExpectations(t) } -func TestAgentOwnerChangedWhileAddingTable(t *testing.T) { +func TestAgentOwnerChangedWhileAddingKeySpan(t *testing.T) { ctx := cdcContext.NewBackendContext4Test(false) - executor := NewMockTableExecutor(t) + executor := NewMockKeySpanExecutor(t) messenger := &MockProcessorMessenger{} agent := NewBaseAgent("test-cf", executor, messenger, agentConfigForTesting) - messenger.On("SyncTaskStatuses", mock.Anything, []model.TableID(nil), []model.TableID(nil), []model.TableID(nil)). + messenger.On("SyncTaskStatuses", mock.Anything, []model.KeySpanID(nil), []model.KeySpanID(nil), []model.KeySpanID(nil)). Return(true, nil) err := agent.Tick(ctx) require.NoError(t, err) messenger.AssertExpectations(t) - agent.OnOwnerDispatchedTask("capture-1", 1, model.TableID(1), false) - executor.On("AddTable", mock.Anything, model.TableID(1)).Return(true, nil) + start, end := []byte{1}, []byte{5} + agent.OnOwnerDispatchedTask("capture-1", 1, model.KeySpanID(1), start, end, false) + executor.On("AddKeySpan", mock.Anything, model.KeySpanID(1)).Return(true, nil) messenger.On("OnOwnerChanged", mock.Anything, "capture-1") err = agent.Tick(ctx) @@ -161,7 +166,7 @@ func TestAgentOwnerChangedWhileAddingTable(t *testing.T) { messenger.ExpectedCalls = nil agent.OnOwnerAnnounce("capture-2", 2) messenger.On("OnOwnerChanged", mock.Anything, "capture-2") - messenger.On("SyncTaskStatuses", mock.Anything, []model.TableID(nil), []model.TableID{1}, []model.TableID(nil)). + messenger.On("SyncTaskStatuses", mock.Anything, []model.KeySpanID(nil), []model.KeySpanID{1}, []model.KeySpanID(nil)). Return(true, nil) messenger.On("Barrier", mock.Anything).Return(true) executor.On("GetCheckpoint").Return(model.Ts(1002), model.Ts(1000)) @@ -175,18 +180,19 @@ func TestAgentOwnerChangedWhileAddingTable(t *testing.T) { func TestAgentReceiveFromStaleOwner(t *testing.T) { ctx := cdcContext.NewBackendContext4Test(false) - executor := NewMockTableExecutor(t) + executor := NewMockKeySpanExecutor(t) messenger := &MockProcessorMessenger{} agent := NewBaseAgent("test-cf", executor, messenger, agentConfigForTesting) agent.checkpointSender = &MockCheckpointSender{} - messenger.On("SyncTaskStatuses", mock.Anything, []model.TableID(nil), []model.TableID(nil), []model.TableID(nil)). + messenger.On("SyncTaskStatuses", mock.Anything, []model.KeySpanID(nil), []model.KeySpanID(nil), []model.KeySpanID(nil)). Return(true, nil) err := agent.Tick(ctx) require.NoError(t, err) messenger.AssertExpectations(t) - agent.OnOwnerDispatchedTask("capture-1", 1, model.TableID(1), false) - executor.On("AddTable", mock.Anything, model.TableID(1)).Return(true, nil) + start, end := []byte{1}, []byte{5} + agent.OnOwnerDispatchedTask("capture-1", 1, model.KeySpanID(1), start, end, false) + executor.On("AddKeySpan", mock.Anything, model.KeySpanID(1)).Return(true, nil) messenger.On("OnOwnerChanged", mock.Anything, "capture-1") err = agent.Tick(ctx) @@ -197,7 +203,8 @@ func TestAgentReceiveFromStaleOwner(t *testing.T) { messenger.ExpectedCalls = nil executor.On("GetCheckpoint").Return(model.Ts(1002), model.Ts(1000)) // Stale owner - agent.OnOwnerDispatchedTask("capture-2", 0, model.TableID(2), false) + start, end = []byte{5}, []byte{6} + agent.OnOwnerDispatchedTask("capture-2", 0, model.KeySpanID(2), start, end, false) err = agent.Tick(ctx) require.NoError(t, err) @@ -216,11 +223,11 @@ func TestAgentReceiveFromStaleOwner(t *testing.T) { func TestOwnerMismatchShouldPanic(t *testing.T) { ctx := cdcContext.NewBackendContext4Test(false) - executor := NewMockTableExecutor(t) + executor := NewMockKeySpanExecutor(t) messenger := &MockProcessorMessenger{} agent := NewBaseAgent("test-cf", executor, messenger, agentConfigForTesting) agent.checkpointSender = &MockCheckpointSender{} - messenger.On("SyncTaskStatuses", mock.Anything, []model.TableID(nil), []model.TableID(nil), []model.TableID(nil)). + messenger.On("SyncTaskStatuses", mock.Anything, []model.KeySpanID(nil), []model.KeySpanID(nil), []model.KeySpanID(nil)). Return(true, nil) err := agent.Tick(ctx) require.NoError(t, err) diff --git a/cdc/cdc/scheduler/balancer.go b/cdc/cdc/scheduler/balancer.go index 94bbe740..92e57785 100644 --- a/cdc/cdc/scheduler/balancer.go +++ b/cdc/cdc/scheduler/balancer.go @@ -26,40 +26,41 @@ import ( // will be automatically rescheduled, during which the target captures // will be chosen so that the workload is the most balanced. // -// The FindTarget method is also used when we need to schedule any table, +// The FindTarget method is also used when we need to schedule any keyspan, // not only when we need to rebalance. +// TODO: Modify type balancer interface { - // FindVictims returns a set of possible victim tables. - // Removing these tables will make the workload more balanced. + // FindVictims returns a set of possible victim keyspans. + // Removing these keyspans will make the workload more balanced. FindVictims( - tables *util.TableSet, + keyspans *util.KeySpanSet, captures map[model.CaptureID]*model.CaptureInfo, - ) (tablesToRemove []*util.TableRecord) + ) (keyspansToRemove []*util.KeySpanRecord) - // FindTarget returns a target capture to add a table to. + // FindTarget returns a target capture to add a keyspan to. FindTarget( - tables *util.TableSet, + keyspans *util.KeySpanSet, captures map[model.CaptureID]*model.CaptureInfo, ) (minLoadCapture model.CaptureID, ok bool) } -// tableNumberBalancer implements a balance strategy based on the -// current number of tables replicated by each capture. +// keyspanNumberBalancer implements a balance strategy based on the +// current number of keyspans replicated by each capture. // TODO: Implement finer-grained balance strategy based on the actual -// workload of each table. -type tableNumberBalancer struct { +// workload of each keyspan. +type keyspanNumberBalancer struct { logger *zap.Logger } -func newTableNumberRebalancer(logger *zap.Logger) balancer { - return &tableNumberBalancer{ +func newKeySpanNumberRebalancer(logger *zap.Logger) balancer { + return &keyspanNumberBalancer{ logger: logger, } } -// FindTarget returns the capture with the smallest workload (in table count). -func (r *tableNumberBalancer) FindTarget( - tables *util.TableSet, +// FindTarget returns the capture with the smallest workload (in keyspan count). +func (r *keyspanNumberBalancer) FindTarget( + keyspans *util.KeySpanSet, captures map[model.CaptureID]*model.CaptureInfo, ) (minLoadCapture model.CaptureID, ok bool) { if len(captures) == 0 { @@ -71,9 +72,9 @@ func (r *tableNumberBalancer) FindTarget( captureWorkload[captureID] = 0 } - for captureID, tables := range tables.GetAllTablesGroupedByCaptures() { - // We use the number of tables as workload - captureWorkload[captureID] = len(tables) + for captureID, keyspans := range keyspans.GetAllKeySpansGroupedByCaptures() { + // We use the number of keyspans as workload + captureWorkload[captureID] = len(keyspans) } candidate := "" @@ -95,62 +96,62 @@ func (r *tableNumberBalancer) FindTarget( // FindVictims returns some victims to remove. // Read the comment in the function body on the details of the victim selection. -func (r *tableNumberBalancer) FindVictims( - tables *util.TableSet, +func (r *keyspanNumberBalancer) FindVictims( + keyspans *util.KeySpanSet, captures map[model.CaptureID]*model.CaptureInfo, -) []*util.TableRecord { - // Algorithm overview: We try to remove some tables as the victims so that - // no captures are assigned more tables than the average workload measured in table number, +) []*util.KeySpanRecord { + // Algorithm overview: We try to remove some keyspans as the victims so that + // no captures are assigned more keyspans than the average workload measured in keyspan number, // modulo the necessary margin due to the fraction part of the average. // // In formula, we try to maintain the invariant: // - // num(tables assigned to any capture) < num(tables) / num(captures) + 1 + // num(keyspans assigned to any capture) < num(keyspans) / num(captures) + 1 - totalTableNum := len(tables.GetAllTables()) + totalKeySpanNum := len(keyspans.GetAllKeySpans()) captureNum := len(captures) if captureNum == 0 { return nil } - upperLimitPerCapture := int(math.Ceil(float64(totalTableNum) / float64(captureNum))) + upperLimitPerCapture := int(math.Ceil(float64(totalKeySpanNum) / float64(captureNum))) r.logger.Info("Start rebalancing", - zap.Int("table-num", totalTableNum), + zap.Int("keyspan-num", totalKeySpanNum), zap.Int("capture-num", captureNum), zap.Int("target-limit", upperLimitPerCapture)) - var victims []*util.TableRecord - for _, tables := range tables.GetAllTablesGroupedByCaptures() { - var tableList []model.TableID - for tableID := range tables { - tableList = append(tableList, tableID) + var victims []*util.KeySpanRecord + for _, keyspans := range keyspans.GetAllKeySpansGroupedByCaptures() { + var keyspanList []model.KeySpanID + for keyspanID := range keyspans { + keyspanList = append(keyspanList, keyspanID) } - // We sort the tableIDs here so that the result is deterministic, + // We sort the keyspanIDs here so that the result is deterministic, // which would aid testing and debugging. - util.SortTableIDs(tableList) + util.SortKeySpanIDs(keyspanList) - tableNum2Remove := len(tables) - upperLimitPerCapture - if tableNum2Remove <= 0 { + keyspanNum2Remove := len(keyspans) - upperLimitPerCapture + if keyspanNum2Remove <= 0 { continue } - // here we pick `tableNum2Remove` tables to delete, - for _, tableID := range tableList { - if tableNum2Remove <= 0 { + // here we pick `keyspanNum2Remove` keyspans to delete, + for _, keyspanID := range keyspanList { + if keyspanNum2Remove <= 0 { break } - record := tables[tableID] + record := keyspans[keyspanID] if record == nil { panic("unreachable") } - r.logger.Info("Rebalance: find victim table", - zap.Any("table-record", record)) + r.logger.Info("Rebalance: find victim keyspan", + zap.Any("keyspan-record", record)) victims = append(victims, record) - tableNum2Remove-- + keyspanNum2Remove-- } } return victims diff --git a/cdc/cdc/scheduler/balancer_test.go b/cdc/cdc/scheduler/balancer_test.go index 97c78b1e..9d01f218 100644 --- a/cdc/cdc/scheduler/balancer_test.go +++ b/cdc/cdc/scheduler/balancer_test.go @@ -23,31 +23,31 @@ import ( ) func TestBalancerFindVictims(t *testing.T) { - balancer := newTableNumberRebalancer(zap.L()) - tables := util.NewTableSet() + balancer := newKeySpanNumberRebalancer(zap.L()) + keyspans := util.NewKeySpanSet() - tables.AddTableRecord(&util.TableRecord{ - TableID: 1, + keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 1, CaptureID: "capture-1", }) - tables.AddTableRecord(&util.TableRecord{ - TableID: 2, + keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 2, CaptureID: "capture-1", }) - tables.AddTableRecord(&util.TableRecord{ - TableID: 3, + keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 3, CaptureID: "capture-1", }) - tables.AddTableRecord(&util.TableRecord{ - TableID: 4, + keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 4, CaptureID: "capture-1", }) - tables.AddTableRecord(&util.TableRecord{ - TableID: 5, + keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 5, CaptureID: "capture-2", }) - tables.AddTableRecord(&util.TableRecord{ - TableID: 6, + keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 6, CaptureID: "capture-2", }) @@ -63,44 +63,44 @@ func TestBalancerFindVictims(t *testing.T) { }, } - victims := balancer.FindVictims(tables, mockCaptureInfos) + victims := balancer.FindVictims(keyspans, mockCaptureInfos) require.Len(t, victims, 2) - require.Contains(t, victims, &util.TableRecord{ - TableID: 1, + require.Contains(t, victims, &util.KeySpanRecord{ + KeySpanID: 1, CaptureID: "capture-1", }) - require.Contains(t, victims, &util.TableRecord{ - TableID: 2, + require.Contains(t, victims, &util.KeySpanRecord{ + KeySpanID: 2, CaptureID: "capture-1", }) } func TestBalancerFindTarget(t *testing.T) { - balancer := newTableNumberRebalancer(zap.L()) - tables := util.NewTableSet() + balancer := newKeySpanNumberRebalancer(zap.L()) + keyspans := util.NewKeySpanSet() - tables.AddTableRecord(&util.TableRecord{ - TableID: 1, + keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 1, CaptureID: "capture-1", }) - tables.AddTableRecord(&util.TableRecord{ - TableID: 2, + keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 2, CaptureID: "capture-1", }) - tables.AddTableRecord(&util.TableRecord{ - TableID: 3, + keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 3, CaptureID: "capture-1", }) - tables.AddTableRecord(&util.TableRecord{ - TableID: 4, + keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 4, CaptureID: "capture-2", }) - tables.AddTableRecord(&util.TableRecord{ - TableID: 5, + keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 5, CaptureID: "capture-2", }) - tables.AddTableRecord(&util.TableRecord{ - TableID: 6, + keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 6, CaptureID: "capture-3", }) @@ -116,15 +116,15 @@ func TestBalancerFindTarget(t *testing.T) { }, } - target, ok := balancer.FindTarget(tables, mockCaptureInfos) + target, ok := balancer.FindTarget(keyspans, mockCaptureInfos) require.True(t, ok) require.Equal(t, "capture-3", target) } func TestBalancerNoCaptureAvailable(t *testing.T) { - balancer := newTableNumberRebalancer(zap.L()) - tables := util.NewTableSet() + balancer := newKeySpanNumberRebalancer(zap.L()) + keyspans := util.NewKeySpanSet() - _, ok := balancer.FindTarget(tables, map[model.CaptureID]*model.CaptureInfo{}) + _, ok := balancer.FindTarget(keyspans, map[model.CaptureID]*model.CaptureInfo{}) require.False(t, ok) } diff --git a/cdc/cdc/scheduler/info_provider.go b/cdc/cdc/scheduler/info_provider.go index 3b64b75c..31b7014a 100644 --- a/cdc/cdc/scheduler/info_provider.go +++ b/cdc/cdc/scheduler/info_provider.go @@ -33,28 +33,28 @@ func (s *BaseScheduleDispatcher) GetTaskStatuses() (map[model.CaptureID]*model.T s.mu.Lock() defer s.mu.Unlock() - tablesPerCapture := s.tables.GetAllTablesGroupedByCaptures() - ret := make(map[model.CaptureID]*model.TaskStatus, len(tablesPerCapture)) - for captureID, tables := range tablesPerCapture { + keyspansPerCapture := s.keyspans.GetAllKeySpansGroupedByCaptures() + ret := make(map[model.CaptureID]*model.TaskStatus, len(keyspansPerCapture)) + for captureID, keyspans := range keyspansPerCapture { ret[captureID] = &model.TaskStatus{ - Tables: make(map[model.TableID]*model.TableReplicaInfo), - Operation: make(map[model.TableID]*model.TableOperation), + KeySpans: make(map[model.KeySpanID]*model.KeySpanReplicaInfo), + Operation: make(map[model.KeySpanID]*model.KeySpanOperation), } - for tableID, record := range tables { - ret[captureID].Tables[tableID] = &model.TableReplicaInfo{ + for keyspanID, record := range keyspans { + ret[captureID].KeySpans[keyspanID] = &model.KeySpanReplicaInfo{ StartTs: 0, // We no longer maintain this information } switch record.Status { - case util.RunningTable: + case util.RunningKeySpan: continue - case util.AddingTable: - ret[captureID].Operation[tableID] = &model.TableOperation{ + case util.AddingKeySpan: + ret[captureID].Operation[keyspanID] = &model.KeySpanOperation{ Delete: false, Status: model.OperDispatched, BoundaryTs: 0, // We no longer maintain this information } - case util.RemovingTable: - ret[captureID].Operation[tableID] = &model.TableOperation{ + case util.RemovingKeySpan: + ret[captureID].Operation[keyspanID] = &model.KeySpanOperation{ Delete: true, Status: model.OperDispatched, BoundaryTs: 0, // We no longer maintain this information diff --git a/cdc/cdc/scheduler/info_provider_test.go b/cdc/cdc/scheduler/info_provider_test.go index a6057cff..6aa7cd53 100644 --- a/cdc/cdc/scheduler/info_provider_test.go +++ b/cdc/cdc/scheduler/info_provider_test.go @@ -37,30 +37,30 @@ func injectSchedulerStateForInfoProviderTest(dispatcher *BaseScheduleDispatcher) ResolvedTs: 1600, }, } - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: 1, + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 1, CaptureID: "capture-1", - Status: util.RunningTable, + Status: util.RunningKeySpan, }) - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: 2, + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 2, CaptureID: "capture-2", - Status: util.RunningTable, + Status: util.RunningKeySpan, }) - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: 3, + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 3, CaptureID: "capture-1", - Status: util.RunningTable, + Status: util.RunningKeySpan, }) - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: 4, + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 4, CaptureID: "capture-2", - Status: util.AddingTable, + Status: util.AddingKeySpan, }) - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: 5, + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 5, CaptureID: "capture-1", - Status: util.RemovingTable, + Status: util.RemovingKeySpan, }) } @@ -72,12 +72,12 @@ func TestInfoProviderTaskStatus(t *testing.T) { require.NoError(t, err) require.Equal(t, map[model.CaptureID]*model.TaskStatus{ "capture-1": { - Tables: map[model.TableID]*model.TableReplicaInfo{ + KeySpans: map[model.KeySpanID]*model.KeySpanReplicaInfo{ 1: {}, 3: {}, 5: {}, }, - Operation: map[model.TableID]*model.TableOperation{ + Operation: map[model.KeySpanID]*model.KeySpanOperation{ 5: { Delete: true, Status: model.OperDispatched, @@ -85,11 +85,11 @@ func TestInfoProviderTaskStatus(t *testing.T) { }, }, "capture-2": { - Tables: map[model.TableID]*model.TableReplicaInfo{ + KeySpans: map[model.KeySpanID]*model.KeySpanReplicaInfo{ 2: {}, 4: {}, }, - Operation: map[model.TableID]*model.TableOperation{ + Operation: map[model.KeySpanID]*model.KeySpanOperation{ 4: { Delete: false, Status: model.OperDispatched, diff --git a/cdc/cdc/scheduler/move_keyspan_manager.go b/cdc/cdc/scheduler/move_keyspan_manager.go new file mode 100644 index 00000000..719d4740 --- /dev/null +++ b/cdc/cdc/scheduler/move_keyspan_manager.go @@ -0,0 +1,201 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package scheduler + +import ( + "sync" + + "github.com/pingcap/errors" + "github.com/tikv/migration/cdc/cdc/model" + "github.com/tikv/migration/cdc/pkg/context" +) + +// Design Notes: +// +// This file contains the definition and implementation of the move kyespan manager, +// which is responsible for implementing the logic for manual keysapn moves. The logic +// here will ultimately be triggered by the user's call to the move keyspan HTTP API. +// +// POST /api/v1/changefeeds/{changefeed_id}/keyspans/move_keyspan +// +// Abstracting out moveKeySpanManager makes it easier to both test the implementation +// and modify the behavior of this API. +// +// The moveKeySpanManager will help the ScheduleDispatcher to track which keyspans are being +// moved to which capture. + +// removeKeySpanFunc is a function used to de-schedule a keyspan from its current processor. +type removeKeySpanFunc = func( + ctx context.Context, + keyspanID model.KeySpanID, + target model.CaptureID) (result removeKeySpanResult, err error) + +type removeKeySpanResult int + +const ( + // removeKeySpanResultOK indicates that the keyspan has been de-scheduled + removeKeySpanResultOK removeKeySpanResult = iota + 1 + + // removeKeySpanResultUnavailable indicates that the keyspan + // is temporarily not available for removal. The operation + // can be tried again later. + removeKeySpanResultUnavailable + + // removeKeySpanResultGiveUp indicates that the operation is + // not successful but there is no point in trying again. Such as when + // 1) the keyspan to be removed is not found, + // 2) the capture to move the keyspan to is not found. + removeKeySpanResultGiveUp +) + +type moveKeySpanManager interface { + // Add adds a keyspan to the move keyspan manager. + // It returns false **if the keyspan is already being moved manually**. + Add(keyspanID model.KeySpanID, target model.CaptureID) (ok bool) + + // DoRemove tries to de-schedule as many keyspans as possible by using the + // given function fn. If the function fn returns false, it means the keyspan + // can not be removed (de-scheduled) for now. + DoRemove(ctx context.Context, fn removeKeySpanFunc) (ok bool, err error) + + // GetTargetByKeySpanID returns the target capture ID of the given keyspan. + // It will only return a target if the keyspan is in the process of being manually + // moved, and the request to de-schedule the given keyspan has already been sent. + GetTargetByKeySpanID(keyspanID model.KeySpanID) (target model.CaptureID, ok bool) + + // MarkDone informs the moveKeySpanManager that the given keyspan has successfully + // been moved. + MarkDone(keyspanID model.KeySpanID) + + // OnCaptureRemoved informs the moveKeySpanManager that a capture has gone offline. + // Then the moveKeySpanManager will clear all pending jobs to that capture. + OnCaptureRemoved(captureID model.CaptureID) +} + +type moveKeySpanJobStatus int + +const ( + moveKeySpanJobStatusReceived = moveKeySpanJobStatus(iota + 1) + moveKeySpanJobStatusRemoved +) + +type moveKeySpanJob struct { + target model.CaptureID + status moveKeySpanJobStatus +} + +type moveKeySpanManagerImpl struct { + mu sync.Mutex + moveKeySpanJobs map[model.KeySpanID]*moveKeySpanJob +} + +func newMoveKeySpanManager() moveKeySpanManager { + return &moveKeySpanManagerImpl{ + moveKeySpanJobs: make(map[model.KeySpanID]*moveKeySpanJob), + } +} + +func (m *moveKeySpanManagerImpl) Add(keyspanID model.KeySpanID, target model.CaptureID) bool { + m.mu.Lock() + defer m.mu.Unlock() + + if _, ok := m.moveKeySpanJobs[keyspanID]; ok { + // Returns false if the keyspan is already in a move keyspan job. + return false + } + + m.moveKeySpanJobs[keyspanID] = &moveKeySpanJob{ + target: target, + status: moveKeySpanJobStatusReceived, + } + return true +} + +func (m *moveKeySpanManagerImpl) DoRemove(ctx context.Context, fn removeKeySpanFunc) (ok bool, err error) { + m.mu.Lock() + defer m.mu.Unlock() + + // This function tries to remove as many keyspans as possible. + // But when we cannot proceed (i.e., fn returns false), we return false, + // so that the caller can retry later. + + for keyspanID, job := range m.moveKeySpanJobs { + if job.status == moveKeySpanJobStatusRemoved { + continue + } + + result, err := fn(ctx, keyspanID, job.target) + if err != nil { + return false, errors.Trace(err) + } + + switch result { + case removeKeySpanResultOK: + job.status = moveKeySpanJobStatusRemoved + continue + case removeKeySpanResultGiveUp: + delete(m.moveKeySpanJobs, keyspanID) + // Giving up means that we can move forward, + // so there is no need to return false here. + continue + case removeKeySpanResultUnavailable: + } + + // Returning false means that there is a keyspan that cannot be removed for now. + // This is usually caused by temporary unavailability of underlying resources, such + // as a congestion in the messaging client. + // + // So when we have returned false, the caller should try again later and refrain from + // other scheduling operations. + return false, nil + } + return true, nil +} + +func (m *moveKeySpanManagerImpl) GetTargetByKeySpanID(keyspanID model.KeySpanID) (model.CaptureID, bool) { + m.mu.Lock() + defer m.mu.Unlock() + + job, ok := m.moveKeySpanJobs[keyspanID] + if !ok { + return "", false + } + + // Only after the keyspan has been removed by the moveKeySpanManager, + // can we provide the target. Otherwise, we risk interfering with + // other operations. + if job.status != moveKeySpanJobStatusRemoved { + return "", false + } + + return job.target, true +} + +func (m *moveKeySpanManagerImpl) MarkDone(keyspanID model.KeySpanID) { + m.mu.Lock() + defer m.mu.Unlock() + + delete(m.moveKeySpanJobs, keyspanID) +} + +func (m *moveKeySpanManagerImpl) OnCaptureRemoved(captureID model.CaptureID) { + m.mu.Lock() + defer m.mu.Unlock() + + for keyspanID, job := range m.moveKeySpanJobs { + if job.target == captureID { + delete(m.moveKeySpanJobs, keyspanID) + } + } +} diff --git a/cdc/cdc/scheduler/move_keyspan_manager_test.go b/cdc/cdc/scheduler/move_keyspan_manager_test.go new file mode 100644 index 00000000..21df707f --- /dev/null +++ b/cdc/cdc/scheduler/move_keyspan_manager_test.go @@ -0,0 +1,137 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package scheduler + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tikv/migration/cdc/cdc/model" + "github.com/tikv/migration/cdc/pkg/context" +) + +func TestMoveKeySpanManagerBasics(t *testing.T) { + m := newMoveKeySpanManager() + + // Test 1: Add a keyspan. + m.Add(1, "capture-1") + _, ok := m.GetTargetByKeySpanID(1) + require.False(t, ok) + + // Test 2: Add a keyspan again. + m.Add(2, "capture-2") + _, ok = m.GetTargetByKeySpanID(2) + require.False(t, ok) + + // Test 3: Add a keyspan with the same ID. + ok = m.Add(2, "capture-2-1") + require.False(t, ok) + + ctx := context.NewBackendContext4Test(false) + // Test 4: Remove one keyspan + var removedKeySpan model.KeySpanID + ok, err := m.DoRemove(ctx, func(ctx context.Context, keyspanID model.KeySpanID, _ model.CaptureID) (result removeKeySpanResult, err error) { + if removedKeySpan != 0 { + return removeKeySpanResultUnavailable, nil + } + removedKeySpan = keyspanID + return removeKeySpanResultOK, nil + }) + require.NoError(t, err) + require.False(t, ok) + require.Containsf(t, []model.KeySpanID{1, 2}, removedKeySpan, "removedKeySpan: %d", removedKeySpan) + + // Test 5: Check removed keyspan's target + target, ok := m.GetTargetByKeySpanID(removedKeySpan) + require.True(t, ok) + require.Equal(t, fmt.Sprintf("capture-%d", removedKeySpan), target) + + // Test 6: Remove another keyspan + var removedKeySpan1 model.KeySpanID + _, err = m.DoRemove(ctx, func(ctx context.Context, keyspanID model.KeySpanID, _ model.CaptureID) (result removeKeySpanResult, err error) { + if removedKeySpan1 != 0 { + require.Fail(t, "Should not have been called twice") + } + removedKeySpan1 = keyspanID + return removeKeySpanResultOK, nil + }) + require.NoError(t, err) + + // Test 7: Mark keyspan done + m.MarkDone(1) + _, ok = m.GetTargetByKeySpanID(1) + require.False(t, ok) +} + +func TestMoveKeySpanManagerCaptureRemoved(t *testing.T) { + m := newMoveKeySpanManager() + + ok := m.Add(1, "capture-1") + require.True(t, ok) + + ok = m.Add(2, "capture-2") + require.True(t, ok) + + ok = m.Add(3, "capture-1") + require.True(t, ok) + + ok = m.Add(4, "capture-2") + require.True(t, ok) + + m.OnCaptureRemoved("capture-2") + + ctx := context.NewBackendContext4Test(false) + var count int + ok, err := m.DoRemove(ctx, + func(ctx context.Context, keyspanID model.KeySpanID, target model.CaptureID) (result removeKeySpanResult, err error) { + require.NotEqual(t, model.KeySpanID(2), keyspanID) + require.NotEqual(t, model.KeySpanID(4), keyspanID) + require.Equal(t, "capture-1", target) + count++ + return removeKeySpanResultOK, nil + }, + ) + require.NoError(t, err) + require.True(t, ok) +} + +func TestMoveKeySpanManagerGiveUp(t *testing.T) { + m := newMoveKeySpanManager() + + ok := m.Add(1, "capture-1") + require.True(t, ok) + + ok = m.Add(2, "capture-2") + require.True(t, ok) + + ctx := context.NewBackendContext4Test(false) + ok, err := m.DoRemove(ctx, + func(ctx context.Context, keyspanID model.KeySpanID, target model.CaptureID) (result removeKeySpanResult, err error) { + if keyspanID == 1 { + return removeKeySpanResultOK, nil + } + return removeKeySpanResultGiveUp, nil + }, + ) + require.NoError(t, err) + require.True(t, ok) + + target, ok := m.GetTargetByKeySpanID(1) + require.True(t, ok) + require.Equal(t, "capture-1", target) + + _, ok = m.GetTargetByKeySpanID(2) + require.False(t, ok) +} diff --git a/cdc/cdc/scheduler/move_table_manager.go b/cdc/cdc/scheduler/move_table_manager.go deleted file mode 100644 index a5f006c5..00000000 --- a/cdc/cdc/scheduler/move_table_manager.go +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package scheduler - -import ( - "sync" - - "github.com/pingcap/errors" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/context" -) - -// Design Notes: -// -// This file contains the definition and implementation of the move table manager, -// which is responsible for implementing the logic for manual table moves. The logic -// here will ultimately be triggered by the user's call to the move table HTTP API. -// -// POST /api/v1/changefeeds/{changefeed_id}/tables/move_table -// -// Abstracting out moveTableManager makes it easier to both test the implementation -// and modify the behavior of this API. -// -// The moveTableManager will help the ScheduleDispatcher to track which tables are being -// moved to which capture. - -// removeTableFunc is a function used to de-schedule a table from its current processor. -type removeTableFunc = func( - ctx context.Context, - tableID model.TableID, - target model.CaptureID) (result removeTableResult, err error) - -type removeTableResult int - -const ( - // removeTableResultOK indicates that the table has been de-scheduled - removeTableResultOK removeTableResult = iota + 1 - - // removeTableResultUnavailable indicates that the table - // is temporarily not available for removal. The operation - // can be tried again later. - removeTableResultUnavailable - - // removeTableResultGiveUp indicates that the operation is - // not successful but there is no point in trying again. Such as when - // 1) the table to be removed is not found, - // 2) the capture to move the table to is not found. - removeTableResultGiveUp -) - -type moveTableManager interface { - // Add adds a table to the move table manager. - // It returns false **if the table is already being moved manually**. - Add(tableID model.TableID, target model.CaptureID) (ok bool) - - // DoRemove tries to de-schedule as many tables as possible by using the - // given function fn. If the function fn returns false, it means the table - // can not be removed (de-scheduled) for now. - DoRemove(ctx context.Context, fn removeTableFunc) (ok bool, err error) - - // GetTargetByTableID returns the target capture ID of the given table. - // It will only return a target if the table is in the process of being manually - // moved, and the request to de-schedule the given table has already been sent. - GetTargetByTableID(tableID model.TableID) (target model.CaptureID, ok bool) - - // MarkDone informs the moveTableManager that the given table has successfully - // been moved. - MarkDone(tableID model.TableID) - - // OnCaptureRemoved informs the moveTableManager that a capture has gone offline. - // Then the moveTableManager will clear all pending jobs to that capture. - OnCaptureRemoved(captureID model.CaptureID) -} - -type moveTableJobStatus int - -const ( - moveTableJobStatusReceived = moveTableJobStatus(iota + 1) - moveTableJobStatusRemoved -) - -type moveTableJob struct { - target model.CaptureID - status moveTableJobStatus -} - -type moveTableManagerImpl struct { - mu sync.Mutex - moveTableJobs map[model.TableID]*moveTableJob -} - -func newMoveTableManager() moveTableManager { - return &moveTableManagerImpl{ - moveTableJobs: make(map[model.TableID]*moveTableJob), - } -} - -func (m *moveTableManagerImpl) Add(tableID model.TableID, target model.CaptureID) bool { - m.mu.Lock() - defer m.mu.Unlock() - - if _, ok := m.moveTableJobs[tableID]; ok { - // Returns false if the table is already in a move table job. - return false - } - - m.moveTableJobs[tableID] = &moveTableJob{ - target: target, - status: moveTableJobStatusReceived, - } - return true -} - -func (m *moveTableManagerImpl) DoRemove(ctx context.Context, fn removeTableFunc) (ok bool, err error) { - m.mu.Lock() - defer m.mu.Unlock() - - // This function tries to remove as many tables as possible. - // But when we cannot proceed (i.e., fn returns false), we return false, - // so that the caller can retry later. - - for tableID, job := range m.moveTableJobs { - if job.status == moveTableJobStatusRemoved { - continue - } - - result, err := fn(ctx, tableID, job.target) - if err != nil { - return false, errors.Trace(err) - } - - switch result { - case removeTableResultOK: - job.status = moveTableJobStatusRemoved - continue - case removeTableResultGiveUp: - delete(m.moveTableJobs, tableID) - // Giving up means that we can move forward, - // so there is no need to return false here. - continue - case removeTableResultUnavailable: - } - - // Returning false means that there is a table that cannot be removed for now. - // This is usually caused by temporary unavailability of underlying resources, such - // as a congestion in the messaging client. - // - // So when we have returned false, the caller should try again later and refrain from - // other scheduling operations. - return false, nil - } - return true, nil -} - -func (m *moveTableManagerImpl) GetTargetByTableID(tableID model.TableID) (model.CaptureID, bool) { - m.mu.Lock() - defer m.mu.Unlock() - - job, ok := m.moveTableJobs[tableID] - if !ok { - return "", false - } - - // Only after the table has been removed by the moveTableManager, - // can we provide the target. Otherwise, we risk interfering with - // other operations. - if job.status != moveTableJobStatusRemoved { - return "", false - } - - return job.target, true -} - -func (m *moveTableManagerImpl) MarkDone(tableID model.TableID) { - m.mu.Lock() - defer m.mu.Unlock() - - delete(m.moveTableJobs, tableID) -} - -func (m *moveTableManagerImpl) OnCaptureRemoved(captureID model.CaptureID) { - m.mu.Lock() - defer m.mu.Unlock() - - for tableID, job := range m.moveTableJobs { - if job.target == captureID { - delete(m.moveTableJobs, tableID) - } - } -} diff --git a/cdc/cdc/scheduler/move_table_manager_test.go b/cdc/cdc/scheduler/move_table_manager_test.go deleted file mode 100644 index f0d5f764..00000000 --- a/cdc/cdc/scheduler/move_table_manager_test.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package scheduler - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/context" -) - -func TestMoveTableManagerBasics(t *testing.T) { - m := newMoveTableManager() - - // Test 1: Add a table. - m.Add(1, "capture-1") - _, ok := m.GetTargetByTableID(1) - require.False(t, ok) - - // Test 2: Add a table again. - m.Add(2, "capture-2") - _, ok = m.GetTargetByTableID(2) - require.False(t, ok) - - // Test 3: Add a table with the same ID. - ok = m.Add(2, "capture-2-1") - require.False(t, ok) - - ctx := context.NewBackendContext4Test(false) - // Test 4: Remove one table - var removedTable model.TableID - ok, err := m.DoRemove(ctx, func(ctx context.Context, tableID model.TableID, _ model.CaptureID) (result removeTableResult, err error) { - if removedTable != 0 { - return removeTableResultUnavailable, nil - } - removedTable = tableID - return removeTableResultOK, nil - }) - require.NoError(t, err) - require.False(t, ok) - require.Containsf(t, []model.TableID{1, 2}, removedTable, "removedTable: %d", removedTable) - - // Test 5: Check removed table's target - target, ok := m.GetTargetByTableID(removedTable) - require.True(t, ok) - require.Equal(t, fmt.Sprintf("capture-%d", removedTable), target) - - // Test 6: Remove another table - var removedTable1 model.TableID - _, err = m.DoRemove(ctx, func(ctx context.Context, tableID model.TableID, _ model.CaptureID) (result removeTableResult, err error) { - if removedTable1 != 0 { - require.Fail(t, "Should not have been called twice") - } - removedTable1 = tableID - return removeTableResultOK, nil - }) - require.NoError(t, err) - - // Test 7: Mark table done - m.MarkDone(1) - _, ok = m.GetTargetByTableID(1) - require.False(t, ok) -} - -func TestMoveTableManagerCaptureRemoved(t *testing.T) { - m := newMoveTableManager() - - ok := m.Add(1, "capture-1") - require.True(t, ok) - - ok = m.Add(2, "capture-2") - require.True(t, ok) - - ok = m.Add(3, "capture-1") - require.True(t, ok) - - ok = m.Add(4, "capture-2") - require.True(t, ok) - - m.OnCaptureRemoved("capture-2") - - ctx := context.NewBackendContext4Test(false) - var count int - ok, err := m.DoRemove(ctx, - func(ctx context.Context, tableID model.TableID, target model.CaptureID) (result removeTableResult, err error) { - require.NotEqual(t, model.TableID(2), tableID) - require.NotEqual(t, model.TableID(4), tableID) - require.Equal(t, "capture-1", target) - count++ - return removeTableResultOK, nil - }, - ) - require.NoError(t, err) - require.True(t, ok) -} - -func TestMoveTableManagerGiveUp(t *testing.T) { - m := newMoveTableManager() - - ok := m.Add(1, "capture-1") - require.True(t, ok) - - ok = m.Add(2, "capture-2") - require.True(t, ok) - - ctx := context.NewBackendContext4Test(false) - ok, err := m.DoRemove(ctx, - func(ctx context.Context, tableID model.TableID, target model.CaptureID) (result removeTableResult, err error) { - if tableID == 1 { - return removeTableResultOK, nil - } - return removeTableResultGiveUp, nil - }, - ) - require.NoError(t, err) - require.True(t, ok) - - target, ok := m.GetTargetByTableID(1) - require.True(t, ok) - require.Equal(t, "capture-1", target) - - _, ok = m.GetTargetByTableID(2) - require.False(t, ok) -} diff --git a/cdc/cdc/scheduler/schedule_dispatcher.go b/cdc/cdc/scheduler/schedule_dispatcher.go index d36060c5..faf46e88 100644 --- a/cdc/cdc/scheduler/schedule_dispatcher.go +++ b/cdc/cdc/scheduler/schedule_dispatcher.go @@ -31,20 +31,20 @@ const ( CheckpointCannotProceed = model.Ts(0) ) -// ScheduleDispatcher is an interface for a table scheduler used in Owner. +// ScheduleDispatcher is an interface for a keyspan scheduler used in Owner. type ScheduleDispatcher interface { // Tick is called periodically to update the SchedulerDispatcher on the latest state of replication. // This function should NOT be assumed to be thread-safe. No concurrent calls allowed. Tick( ctx context.Context, checkpointTs model.Ts, // Latest global checkpoint of the changefeed - currentTables []model.TableID, // All tables that SHOULD be replicated (or started) at the current checkpoint. + currentKeySpans []model.KeySpanID, // All keyspans that SHOULD be replicated (or started) at the current checkpoint. captures map[model.CaptureID]*model.CaptureInfo, // All captures that are alive according to the latest Etcd states. ) (newCheckpointTs, newResolvedTs model.Ts, err error) - // MoveTable requests that a table be moved to target. + // MoveKeySpan requests that a keyspan be moved to target. // It should be thread-safe. - MoveTable(tableID model.TableID, target model.CaptureID) + MoveKeySpan(keyspanID model.KeySpanID, target model.CaptureID) // Rebalance triggers a rebalance operation. // It should be thread-safe @@ -56,12 +56,12 @@ type ScheduleDispatcher interface { // an implementation of ScheduleDispatcherCommunicator to supply BaseScheduleDispatcher // some methods to specify its behavior. type ScheduleDispatcherCommunicator interface { - // DispatchTable should send a dispatch command to the Processor. - DispatchTable(ctx context.Context, + // DispatchKeySpan should send a dispatch command to the Processor. + DispatchKeySpan(ctx context.Context, changeFeedID model.ChangeFeedID, - tableID model.TableID, + keyspanID model.KeySpanID, captureID model.CaptureID, - isDelete bool, // True when we want to remove a table from the capture. + isDelete bool, // True when we want to remove a keyspan from the capture. ) (done bool, err error) // Announce announces to the specified capture that the current node has become the Owner. @@ -80,13 +80,13 @@ const ( // ScheduleDispatcherCommunicator. type BaseScheduleDispatcher struct { mu sync.Mutex - tables *util.TableSet // information of all actually running tables + keyspans *util.KeySpanSet // information of all actually running kespans captures map[model.CaptureID]*model.CaptureInfo // basic information of all captures captureStatus map[model.CaptureID]*captureStatus // more information on the captures checkpointTs model.Ts // current checkpoint-ts - moveTableManager moveTableManager - balancer balancer + moveKeySpanManager moveKeySpanManager + balancer balancer lastTickCaptureCount int needRebalance bool @@ -107,10 +107,10 @@ func NewBaseScheduleDispatcher( logger := log.L().With(zap.String("changefeed-id", changeFeedID)) return &BaseScheduleDispatcher{ - tables: util.NewTableSet(), + keyspans: util.NewKeySpanSet(), captureStatus: map[model.CaptureID]*captureStatus{}, - moveTableManager: newMoveTableManager(), - balancer: newTableNumberRebalancer(logger), + moveKeySpanManager: newMoveKeySpanManager(), + balancer: newKeySpanNumberRebalancer(logger), changeFeedID: changeFeedID, logger: logger, communicator: communicator, @@ -122,7 +122,7 @@ func NewBaseScheduleDispatcher( type captureStatus struct { // SyncStatus indicates what we know about the capture's internal state. // We need to know this before we can make decision whether to - // dispatch a table. + // dispatch a keyspan. SyncStatus captureSyncStatus // Watermark fields @@ -141,7 +141,7 @@ const ( // no response yet. captureSyncSent // captureSyncFinished indicates that the capture has been fully initialized and is ready to - // accept `DispatchTable` messages. + // accept `DispatchKeySpan` messages. captureSyncFinished ) @@ -149,9 +149,9 @@ const ( func (s *BaseScheduleDispatcher) Tick( ctx context.Context, checkpointTs model.Ts, - // currentTables are tables that SHOULD be running given the current checkpoint-ts. + // currentKeySpans are keyspans that SHOULD be running given the current checkpoint-ts. // It is maintained by the caller of this function. - currentTables []model.TableID, + currentKeySpans []model.KeySpanID, captures map[model.CaptureID]*model.CaptureInfo, ) (newCheckpointTs, resolvedTs model.Ts, err error) { s.mu.Lock() @@ -191,26 +191,26 @@ func (s *BaseScheduleDispatcher) Tick( if !done { // Returns early if not all captures have synced their states with us. // We need to know all captures' status in order to proceed. - // This is crucial for ensuring that no table is double-scheduled. + // This is crucial for ensuring that no keyspan is double-scheduled. return CheckpointCannotProceed, CheckpointCannotProceed, nil } - s.descheduleTablesFromDownCaptures() + s.descheduleKeySpansFromDownCaptures() - shouldReplicateTableSet := make(map[model.TableID]struct{}) - for _, tableID := range currentTables { - shouldReplicateTableSet[tableID] = struct{}{} + shouldReplicateKeySpanSet := make(map[model.KeySpanID]struct{}) + for _, keyspanID := range currentKeySpans { + shouldReplicateKeySpanSet[keyspanID] = struct{}{} } - // findDiffTables compares the tables that should be running and - // the tables that are actually running. - // Note: Tables that are being added and removed are considered + // findDiffKeySpans compares the keyspans that should be running and + // the keyspans that are actually running. + // Note: keyspans that are being added and removed are considered // "running" for the purpose of comparison, and we do not interrupt // these operations. - toAdd, toRemove := s.findDiffTables(shouldReplicateTableSet) + toAdd, toRemove := s.findDiffKeySpans(shouldReplicateKeySpanSet) - for _, tableID := range toAdd { - ok, err := s.addTable(ctx, tableID) + for _, keyspanID := range toAdd { + ok, err := s.addKeySpan(ctx, keyspanID) if err != nil { return CheckpointCannotProceed, CheckpointCannotProceed, errors.Trace(err) } @@ -219,17 +219,17 @@ func (s *BaseScheduleDispatcher) Tick( } } - for _, tableID := range toRemove { - record, ok := s.tables.GetTableRecord(tableID) + for _, keyspanID := range toRemove { + record, ok := s.keyspans.GetKeySpanRecord(keyspanID) if !ok { - s.logger.Panic("table not found", zap.Int64("table-id", tableID)) + s.logger.Panic("keyspan not found", zap.Uint64("keyspan-id", keyspanID)) } - if record.Status != util.RunningTable { + if record.Status != util.RunningKeySpan { // another operation is in progress continue } - ok, err := s.removeTable(ctx, tableID) + ok, err := s.removeKeySpan(ctx, keyspanID) if err != nil { return CheckpointCannotProceed, CheckpointCannotProceed, errors.Trace(err) } @@ -239,16 +239,16 @@ func (s *BaseScheduleDispatcher) Tick( } checkAllTasksNormal := func() bool { - return s.tables.CountTableByStatus(util.RunningTable) == len(currentTables) && - s.tables.CountTableByStatus(util.AddingTable) == 0 && - s.tables.CountTableByStatus(util.RemovingTable) == 0 + return s.keyspans.CountKeySpanByStatus(util.RunningKeySpan) == len(currentKeySpans) && + s.keyspans.CountKeySpanByStatus(util.AddingKeySpan) == 0 && + s.keyspans.CountKeySpanByStatus(util.RemovingKeySpan) == 0 } if !checkAllTasksNormal() { return CheckpointCannotProceed, CheckpointCannotProceed, nil } - // handleMoveTableJobs tries to execute user-specified manual move table jobs. - ok, err := s.handleMoveTableJobs(ctx) + // handleMoveKeySpanJobs tries to execute user-specified manual move keyspan jobs. + ok, err := s.handleMoveKeySpanJobs(ctx) if err != nil { return CheckpointCannotProceed, CheckpointCannotProceed, errors.Trace(err) } @@ -282,9 +282,9 @@ func (s *BaseScheduleDispatcher) calculateTs() (checkpointTs, resolvedTs model.T resolvedTs = math.MaxUint64 for captureID, status := range s.captureStatus { - if s.tables.CountTableByCaptureID(captureID) == 0 { + if s.keyspans.CountKeySpanByCaptureID(captureID) == 0 { // the checkpoint (as well as resolved-ts) from a capture - // that is not replicating any table is meaningless. + // that is not replicating any keyspan is meaningless. continue } if status.ResolvedTs < resolvedTs { @@ -337,66 +337,69 @@ func (s *BaseScheduleDispatcher) syncCaptures(ctx context.Context) (capturesAllS panic("unreachable") } } + s.logger.Debug("syncCaptures: size of captures, size of sync finished captures", + zap.Int("size of captures", len(s.captureStatus)), + zap.Int("size of finished captures", finishedCount)) return finishedCount == len(s.captureStatus), nil } -// descheduleTablesFromDownCaptures removes tables from `s.tables` that are +// descheduleKeySpansFromDownCaptures removes keyspans from `s.keyspans` that are // associated with a capture that no longer exists. // `s.captures` MUST be updated before calling this method. -func (s *BaseScheduleDispatcher) descheduleTablesFromDownCaptures() { - for _, captureID := range s.tables.GetDistinctCaptures() { +func (s *BaseScheduleDispatcher) descheduleKeySpansFromDownCaptures() { + for _, captureID := range s.keyspans.GetDistinctCaptures() { // If the capture is not in the current list of captures, it means that // the capture has been removed from the system. if _, ok := s.captures[captureID]; !ok { - // Remove records for all table previously replicated by the + // Remove records for all keyspan previously replicated by the // gone capture. - removed := s.tables.RemoveTableRecordByCaptureID(captureID) - s.logger.Info("capture down, removing tables", + removed := s.keyspans.RemoveKeySpanRecordByCaptureID(captureID) + s.logger.Info("capture down, removing keyspans", zap.String("capture-id", captureID), - zap.Any("removed-tables", removed)) - s.moveTableManager.OnCaptureRemoved(captureID) + zap.Any("removed-keyspans", removed)) + s.moveKeySpanManager.OnCaptureRemoved(captureID) } } } -func (s *BaseScheduleDispatcher) findDiffTables( - shouldReplicateTables map[model.TableID]struct{}, -) (toAdd, toRemove []model.TableID) { - // Find tables that need to be added. - for tableID := range shouldReplicateTables { - if _, ok := s.tables.GetTableRecord(tableID); !ok { - // table is not found in `s.tables`. - toAdd = append(toAdd, tableID) +func (s *BaseScheduleDispatcher) findDiffKeySpans( + shouldReplicateKeySpans map[model.KeySpanID]struct{}, +) (toAdd, toRemove []model.KeySpanID) { + // Find keyspans that need to be added. + for keyspanID := range shouldReplicateKeySpans { + if _, ok := s.keyspans.GetKeySpanRecord(keyspanID); !ok { + // keyspan is not found in `s.keyspans`. + toAdd = append(toAdd, keyspanID) } } - // Find tables that need to be removed. - for tableID := range s.tables.GetAllTables() { - if _, ok := shouldReplicateTables[tableID]; !ok { - // table is not found in `shouldReplicateTables`. - toRemove = append(toRemove, tableID) + // Find keyspans that need to be removed. + for keyspanID := range s.keyspans.GetAllKeySpans() { + if _, ok := shouldReplicateKeySpans[keyspanID]; !ok { + // keyspan is not found in `shouldReplicateKeySpans`. + toRemove = append(toRemove, keyspanID) } } return } -func (s *BaseScheduleDispatcher) addTable( +func (s *BaseScheduleDispatcher) addKeySpan( ctx context.Context, - tableID model.TableID, + keyspanID model.KeySpanID, ) (done bool, err error) { - // A user triggered move-table will have had the target recorded. - target, ok := s.moveTableManager.GetTargetByTableID(tableID) + // A user triggered move-keyspan will have had the target recorded. + target, ok := s.moveKeySpanManager.GetTargetByKeySpanID(keyspanID) isManualMove := ok if !ok { - target, ok = s.balancer.FindTarget(s.tables, s.captures) + target, ok = s.balancer.FindTarget(s.keyspans, s.captures) if !ok { s.logger.Warn("no active capture") return true, nil } } - ok, err = s.communicator.DispatchTable(ctx, s.changeFeedID, tableID, target, false) + ok, err = s.communicator.DispatchKeySpan(ctx, s.changeFeedID, keyspanID, target, false) if err != nil { return false, errors.Trace(err) } @@ -405,30 +408,30 @@ func (s *BaseScheduleDispatcher) addTable( return false, nil } if isManualMove { - s.moveTableManager.MarkDone(tableID) + s.moveKeySpanManager.MarkDone(keyspanID) } - if ok := s.tables.AddTableRecord(&util.TableRecord{ - TableID: tableID, + if ok := s.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: keyspanID, CaptureID: target, - Status: util.AddingTable, + Status: util.AddingKeySpan, }); !ok { - s.logger.Panic("duplicate table", zap.Int64("table-id", tableID)) + s.logger.Panic("duplicate keyspan", zap.Uint64("keyspan-id", keyspanID)) } return true, nil } -func (s *BaseScheduleDispatcher) removeTable( +func (s *BaseScheduleDispatcher) removeKeySpan( ctx context.Context, - tableID model.TableID, + keyspanID model.KeySpanID, ) (done bool, err error) { - record, ok := s.tables.GetTableRecord(tableID) + record, ok := s.keyspans.GetKeySpanRecord(keyspanID) if !ok { - s.logger.Panic("table not found", zap.Int64("table-id", tableID)) + s.logger.Panic("keyspan not found", zap.Uint64("keyspan-id", keyspanID)) } - // need to delete table + // need to delete keyspan captureID := record.CaptureID - ok, err = s.communicator.DispatchTable(ctx, s.changeFeedID, tableID, captureID, true) + ok, err = s.communicator.DispatchKeySpan(ctx, s.changeFeedID, keyspanID, captureID, true) if err != nil { return false, errors.Trace(err) } @@ -436,45 +439,45 @@ func (s *BaseScheduleDispatcher) removeTable( return false, nil } - record.Status = util.RemovingTable - s.tables.UpdateTableRecord(record) + record.Status = util.RemovingKeySpan + s.keyspans.UpdateKeySpanRecord(record) return true, nil } -// MoveTable implements the interface SchedulerDispatcher. -func (s *BaseScheduleDispatcher) MoveTable(tableID model.TableID, target model.CaptureID) { - if !s.moveTableManager.Add(tableID, target) { - log.Info("Move Table command has been ignored, because the last user triggered"+ +// MoveKeySpan implements the interface SchedulerDispatcher. +func (s *BaseScheduleDispatcher) MoveKeySpan(keyspanID model.KeySpanID, target model.CaptureID) { + if !s.moveKeySpanManager.Add(keyspanID, target) { + log.Info("Move KeySpan command has been ignored, because the last user triggered"+ "move has not finished", - zap.Int64("table-id", tableID), + zap.Uint64("keyspan-id", keyspanID), zap.String("target-capture", target)) } } -func (s *BaseScheduleDispatcher) handleMoveTableJobs(ctx context.Context) (bool, error) { - removeAllDone, err := s.moveTableManager.DoRemove(ctx, - func(ctx context.Context, tableID model.TableID, target model.CaptureID) (removeTableResult, error) { - _, ok := s.tables.GetTableRecord(tableID) +func (s *BaseScheduleDispatcher) handleMoveKeySpanJobs(ctx context.Context) (bool, error) { + removeAllDone, err := s.moveKeySpanManager.DoRemove(ctx, + func(ctx context.Context, keyspanID model.KeySpanID, target model.CaptureID) (removeKeySpanResult, error) { + _, ok := s.keyspans.GetKeySpanRecord(keyspanID) if !ok { - s.logger.Warn("table does not exist", zap.Int64("table-id", tableID)) - return removeTableResultGiveUp, nil + s.logger.Warn("keyspan does not exist", zap.Uint64("keyspan-id", keyspanID)) + return removeKeySpanResultGiveUp, nil } if _, ok := s.captures[target]; !ok { - s.logger.Warn("move table target does not exist", - zap.Int64("table-id", tableID), + s.logger.Warn("move keyspan target does not exist", + zap.Uint64("keyspan-id", keyspanID), zap.String("target-capture", target)) - return removeTableResultGiveUp, nil + return removeKeySpanResultGiveUp, nil } - ok, err := s.removeTable(ctx, tableID) + ok, err := s.removeKeySpan(ctx, keyspanID) if err != nil { - return removeTableResultUnavailable, errors.Trace(err) + return removeKeySpanResultUnavailable, errors.Trace(err) } if !ok { - return removeTableResultUnavailable, nil + return removeKeySpanResultUnavailable, nil } - return removeTableResultOK, nil + return removeKeySpanResultOK, nil }, ) if err != nil { @@ -489,15 +492,15 @@ func (s *BaseScheduleDispatcher) Rebalance() { } func (s *BaseScheduleDispatcher) rebalance(ctx context.Context) (done bool, err error) { - tablesToRemove := s.balancer.FindVictims(s.tables, s.captures) - for _, record := range tablesToRemove { - if record.Status != util.RunningTable { - s.logger.DPanic("unexpected table status", - zap.Any("table-record", record)) + keyspansToRemove := s.balancer.FindVictims(s.keyspans, s.captures) + for _, record := range keyspansToRemove { + if record.Status != util.RunningKeySpan { + s.logger.DPanic("unexpected keyspan status", + zap.Any("keyspan-record", record)) } - // Removes the table from the current capture - ok, err := s.communicator.DispatchTable(ctx, s.changeFeedID, record.TableID, record.CaptureID, true) + // Removes the keyspan from the current capture + ok, err := s.communicator.DispatchKeySpan(ctx, s.changeFeedID, record.KeySpanID, record.CaptureID, true) if err != nil { return false, errors.Trace(err) } @@ -505,21 +508,21 @@ func (s *BaseScheduleDispatcher) rebalance(ctx context.Context) (done bool, err return false, nil } - record.Status = util.RemovingTable - s.tables.UpdateTableRecord(record) + record.Status = util.RemovingKeySpan + s.keyspans.UpdateKeySpanRecord(record) } return true, nil } -// OnAgentFinishedTableOperation is called when a table operation has been finished by +// OnAgentFinishedKeySpanOperation is called when a keyspan operation has been finished by // the processor. -func (s *BaseScheduleDispatcher) OnAgentFinishedTableOperation(captureID model.CaptureID, tableID model.TableID) { +func (s *BaseScheduleDispatcher) OnAgentFinishedKeySpanOperation(captureID model.CaptureID, keyspanID model.KeySpanID) { s.mu.Lock() defer s.mu.Unlock() logger := s.logger.With( zap.String("capture-id", captureID), - zap.Int64("table-id", tableID), + zap.Uint64("keyspan-id", keyspanID), ) if _, ok := s.captures[captureID]; !ok { @@ -527,9 +530,9 @@ func (s *BaseScheduleDispatcher) OnAgentFinishedTableOperation(captureID model.C return } - record, ok := s.tables.GetTableRecord(tableID) + record, ok := s.keyspans.GetKeySpanRecord(keyspanID) if !ok { - logger.Warn("response about a stale table, ignore") + logger.Warn("response about a stale keyspan, ignore") return } @@ -540,20 +543,20 @@ func (s *BaseScheduleDispatcher) OnAgentFinishedTableOperation(captureID model.C logger.Info("owner received dispatch finished") switch record.Status { - case util.AddingTable: - record.Status = util.RunningTable - s.tables.UpdateTableRecord(record) - case util.RemovingTable: - if !s.tables.RemoveTableRecord(tableID) { - logger.Panic("failed to remove table") + case util.AddingKeySpan: + record.Status = util.RunningKeySpan + s.keyspans.UpdateKeySpanRecord(record) + case util.RemovingKeySpan: + if !s.keyspans.RemoveKeySpanRecord(keyspanID) { + logger.Panic("failed to remove keyspan") } - case util.RunningTable: + case util.RunningKeySpan: logger.Panic("response to invalid dispatch message") } } // OnAgentSyncTaskStatuses is called when the processor sends its complete current state. -func (s *BaseScheduleDispatcher) OnAgentSyncTaskStatuses(captureID model.CaptureID, running, adding, removing []model.TableID) { +func (s *BaseScheduleDispatcher) OnAgentSyncTaskStatuses(captureID model.CaptureID, running, adding, removing []model.KeySpanID) { s.mu.Lock() defer s.mu.Unlock() @@ -568,10 +571,10 @@ func (s *BaseScheduleDispatcher) OnAgentSyncTaskStatuses(captureID model.Capture zap.Any("removing", removing)) } - // Clear all tables previously run by the sender capture, + // Clear all keyspans previously run by the sender capture, // because `Sync` tells the Owner to reset its state regarding // the sender capture. - s.tables.RemoveTableRecordByCaptureID(captureID) + s.keyspans.RemoveKeySpanRecordByCaptureID(captureID) if _, ok := s.captureStatus[captureID]; !ok { logger.Warn("received sync from a capture not previously tracked, ignore", @@ -579,29 +582,29 @@ func (s *BaseScheduleDispatcher) OnAgentSyncTaskStatuses(captureID model.Capture return } - for _, tableID := range adding { - if record, ok := s.tables.GetTableRecord(tableID); ok { - logger.Panic("duplicate table tasks", - zap.Int64("table-id", tableID), + for _, keyspanID := range adding { + if record, ok := s.keyspans.GetKeySpanRecord(keyspanID); ok { + logger.Panic("duplicate keyspan tasks", + zap.Uint64("keyspan-id", keyspanID), zap.String("actual-capture-id", record.CaptureID)) } - s.tables.AddTableRecord(&util.TableRecord{TableID: tableID, CaptureID: captureID, Status: util.AddingTable}) + s.keyspans.AddKeySpanRecord(&util.KeySpanRecord{KeySpanID: keyspanID, CaptureID: captureID, Status: util.AddingKeySpan}) } - for _, tableID := range running { - if record, ok := s.tables.GetTableRecord(tableID); ok { - logger.Panic("duplicate table tasks", - zap.Int64("table-id", tableID), + for _, keyspanID := range running { + if record, ok := s.keyspans.GetKeySpanRecord(keyspanID); ok { + logger.Panic("duplicate keyspan tasks", + zap.Uint64("keyspan-id", keyspanID), zap.String("actual-capture-id", record.CaptureID)) } - s.tables.AddTableRecord(&util.TableRecord{TableID: tableID, CaptureID: captureID, Status: util.RunningTable}) + s.keyspans.AddKeySpanRecord(&util.KeySpanRecord{KeySpanID: keyspanID, CaptureID: captureID, Status: util.RunningKeySpan}) } - for _, tableID := range removing { - if record, ok := s.tables.GetTableRecord(tableID); ok { - logger.Panic("duplicate table tasks", - zap.Int64("table-id", tableID), + for _, keyspanID := range removing { + if record, ok := s.keyspans.GetKeySpanRecord(keyspanID); ok { + logger.Panic("duplicate keyspan tasks", + zap.Uint64("keyspan-id", keyspanID), zap.String("actual-capture-id", record.CaptureID)) } - s.tables.AddTableRecord(&util.TableRecord{TableID: tableID, CaptureID: captureID, Status: util.RemovingTable}) + s.keyspans.AddKeySpanRecord(&util.KeySpanRecord{KeySpanID: keyspanID, CaptureID: captureID, Status: util.RemovingKeySpan}) } s.captureStatus[captureID].SyncStatus = captureSyncFinished diff --git a/cdc/cdc/scheduler/schedule_dispatcher_test.go b/cdc/cdc/scheduler/schedule_dispatcher_test.go index 8c1eff09..dd8f5ad0 100644 --- a/cdc/cdc/scheduler/schedule_dispatcher_test.go +++ b/cdc/cdc/scheduler/schedule_dispatcher_test.go @@ -30,42 +30,42 @@ var _ ScheduleDispatcherCommunicator = (*mockScheduleDispatcherCommunicator)(nil type mockScheduleDispatcherCommunicator struct { mock.Mock - addTableRecords map[model.CaptureID][]model.TableID - removeTableRecords map[model.CaptureID][]model.TableID + addKeySpanRecords map[model.CaptureID][]model.KeySpanID + removeKeySpanRecords map[model.CaptureID][]model.KeySpanID } func NewMockScheduleDispatcherCommunicator() *mockScheduleDispatcherCommunicator { return &mockScheduleDispatcherCommunicator{ - addTableRecords: map[model.CaptureID][]model.TableID{}, - removeTableRecords: map[model.CaptureID][]model.TableID{}, + addKeySpanRecords: map[model.CaptureID][]model.KeySpanID{}, + removeKeySpanRecords: map[model.CaptureID][]model.KeySpanID{}, } } func (m *mockScheduleDispatcherCommunicator) Reset() { - m.addTableRecords = map[model.CaptureID][]model.TableID{} - m.removeTableRecords = map[model.CaptureID][]model.TableID{} + m.addKeySpanRecords = map[model.CaptureID][]model.KeySpanID{} + m.removeKeySpanRecords = map[model.CaptureID][]model.KeySpanID{} m.Mock.ExpectedCalls = nil m.Mock.Calls = nil } -func (m *mockScheduleDispatcherCommunicator) DispatchTable( +func (m *mockScheduleDispatcherCommunicator) DispatchKeySpan( ctx cdcContext.Context, changeFeedID model.ChangeFeedID, - tableID model.TableID, + keyspanID model.KeySpanID, captureID model.CaptureID, isDelete bool, ) (done bool, err error) { - log.Info("dispatch table called", + log.Info("dispatch keyspan called", zap.String("changefeed-id", changeFeedID), - zap.Int64("table-id", tableID), + zap.Uint64("keyspan-id", keyspanID), zap.String("capture-id", captureID), zap.Bool("is-delete", isDelete)) if !isDelete { - m.addTableRecords[captureID] = append(m.addTableRecords[captureID], tableID) + m.addKeySpanRecords[captureID] = append(m.addKeySpanRecords[captureID], keyspanID) } else { - m.removeTableRecords[captureID] = append(m.removeTableRecords[captureID], tableID) + m.removeKeySpanRecords[captureID] = append(m.removeKeySpanRecords[captureID], keyspanID) } - args := m.Called(ctx, changeFeedID, tableID, captureID, isDelete) + args := m.Called(ctx, changeFeedID, keyspanID, captureID, isDelete) return args.Bool(0), args.Error(1) } @@ -90,7 +90,7 @@ var defaultMockCaptureInfos = map[model.CaptureID]*model.CaptureInfo{ }, } -func TestDispatchTable(t *testing.T) { +func TestDispatchKeySpan(t *testing.T) { t.Parallel() ctx := cdcContext.NewBackendContext4Test(false) @@ -99,61 +99,61 @@ func TestDispatchTable(t *testing.T) { communicator.On("Announce", mock.Anything, "cf-1", "capture-1").Return(true, nil) communicator.On("Announce", mock.Anything, "cf-1", "capture-2").Return(true, nil) - checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1000, []model.TableID{1, 2, 3}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1000, []model.KeySpanID{1, 2, 3}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.AssertExpectations(t) - dispatcher.OnAgentSyncTaskStatuses("capture-1", []model.TableID{}, []model.TableID{}, []model.TableID{}) - dispatcher.OnAgentSyncTaskStatuses("capture-2", []model.TableID{}, []model.TableID{}, []model.TableID{}) + dispatcher.OnAgentSyncTaskStatuses("capture-1", []model.KeySpanID{}, []model.KeySpanID{}, []model.KeySpanID{}) + dispatcher.OnAgentSyncTaskStatuses("capture-2", []model.KeySpanID{}, []model.KeySpanID{}, []model.KeySpanID{}) communicator.Reset() - // Injects a dispatch table failure - communicator.On("DispatchTable", mock.Anything, "cf-1", mock.Anything, mock.Anything, false). + // Injects a dispatch keyspan failure + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", mock.Anything, mock.Anything, false). Return(false, nil) - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.TableID{1, 2, 3}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.KeySpanID{1, 2, 3}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.AssertExpectations(t) communicator.Reset() - communicator.On("DispatchTable", mock.Anything, "cf-1", model.TableID(1), mock.Anything, false). + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", model.KeySpanID(1), mock.Anything, false). Return(true, nil) - communicator.On("DispatchTable", mock.Anything, "cf-1", model.TableID(2), mock.Anything, false). + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", model.KeySpanID(2), mock.Anything, false). Return(true, nil) - communicator.On("DispatchTable", mock.Anything, "cf-1", model.TableID(3), mock.Anything, false). + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", model.KeySpanID(3), mock.Anything, false). Return(true, nil) - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.TableID{1, 2, 3}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.KeySpanID{1, 2, 3}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.AssertExpectations(t) - require.NotEqual(t, 0, len(communicator.addTableRecords["capture-1"])) - require.NotEqual(t, 0, len(communicator.addTableRecords["capture-2"])) - require.Equal(t, 0, len(communicator.removeTableRecords["capture-1"])) - require.Equal(t, 0, len(communicator.removeTableRecords["capture-2"])) + require.NotEqual(t, 0, len(communicator.addKeySpanRecords["capture-1"])) + require.NotEqual(t, 0, len(communicator.addKeySpanRecords["capture-2"])) + require.Equal(t, 0, len(communicator.removeKeySpanRecords["capture-1"])) + require.Equal(t, 0, len(communicator.removeKeySpanRecords["capture-2"])) dispatcher.OnAgentCheckpoint("capture-1", 2000, 2000) dispatcher.OnAgentCheckpoint("capture-1", 2001, 2001) communicator.ExpectedCalls = nil - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.TableID{1, 2, 3}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.KeySpanID{1, 2, 3}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.AssertExpectations(t) - for captureID, tables := range communicator.addTableRecords { - for _, tableID := range tables { - dispatcher.OnAgentFinishedTableOperation(captureID, tableID) + for captureID, keyspans := range communicator.addKeySpanRecords { + for _, keyspanID := range keyspans { + dispatcher.OnAgentFinishedKeySpanOperation(captureID, keyspanID) } } communicator.Reset() - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.TableID{1, 2, 3}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.KeySpanID{1, 2, 3}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, model.Ts(1000), checkpointTs) require.Equal(t, model.Ts(1000), resolvedTs) @@ -161,7 +161,7 @@ func TestDispatchTable(t *testing.T) { dispatcher.OnAgentCheckpoint("capture-1", 1100, 1400) dispatcher.OnAgentCheckpoint("capture-2", 1200, 1300) communicator.Reset() - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.TableID{1, 2, 3}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.KeySpanID{1, 2, 3}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, model.Ts(1100), checkpointTs) require.Equal(t, model.Ts(1300), resolvedTs) @@ -177,7 +177,7 @@ func TestSyncCaptures(t *testing.T) { communicator.On("Announce", mock.Anything, "cf-1", "capture-1").Return(false, nil) communicator.On("Announce", mock.Anything, "cf-1", "capture-2").Return(false, nil) - checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1500, []model.TableID{1, 2, 3, 4, 5}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1500, []model.KeySpanID{1, 2, 3, 4, 5}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) @@ -185,30 +185,30 @@ func TestSyncCaptures(t *testing.T) { communicator.Reset() communicator.On("Announce", mock.Anything, "cf-1", "capture-1").Return(true, nil) communicator.On("Announce", mock.Anything, "cf-1", "capture-2").Return(true, nil) - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1500, []model.TableID{1, 2, 3, 4, 5}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1500, []model.KeySpanID{1, 2, 3, 4, 5}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) - dispatcher.OnAgentSyncTaskStatuses("capture-1", []model.TableID{1, 2, 3}, []model.TableID{4, 5}, []model.TableID{6, 7}) - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1500, []model.TableID{1, 2, 3, 4, 5}, defaultMockCaptureInfos) + dispatcher.OnAgentSyncTaskStatuses("capture-1", []model.KeySpanID{1, 2, 3}, []model.KeySpanID{4, 5}, []model.KeySpanID{6, 7}) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1500, []model.KeySpanID{1, 2, 3, 4, 5}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.Reset() - dispatcher.OnAgentFinishedTableOperation("capture-1", 4) - dispatcher.OnAgentFinishedTableOperation("capture-1", 5) - dispatcher.OnAgentSyncTaskStatuses("capture-2", []model.TableID(nil), []model.TableID(nil), []model.TableID(nil)) - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1500, []model.TableID{1, 2, 3, 4, 5}, defaultMockCaptureInfos) + dispatcher.OnAgentFinishedKeySpanOperation("capture-1", 4) + dispatcher.OnAgentFinishedKeySpanOperation("capture-1", 5) + dispatcher.OnAgentSyncTaskStatuses("capture-2", []model.KeySpanID(nil), []model.KeySpanID(nil), []model.KeySpanID(nil)) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1500, []model.KeySpanID{1, 2, 3, 4, 5}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.Reset() - dispatcher.OnAgentFinishedTableOperation("capture-1", 6) - dispatcher.OnAgentFinishedTableOperation("capture-1", 7) - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1500, []model.TableID{1, 2, 3, 4, 5}, defaultMockCaptureInfos) + dispatcher.OnAgentFinishedKeySpanOperation("capture-1", 6) + dispatcher.OnAgentFinishedKeySpanOperation("capture-1", 7) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1500, []model.KeySpanID{1, 2, 3, 4, 5}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, model.Ts(1500), checkpointTs) require.Equal(t, model.Ts(1500), resolvedTs) @@ -225,16 +225,16 @@ func TestSyncUnknownCapture(t *testing.T) { dispatcher.captureStatus = map[model.CaptureID]*captureStatus{} // empty capture status // Sends a sync from an unknown capture - dispatcher.OnAgentSyncTaskStatuses("capture-1", []model.TableID{1, 2, 3}, []model.TableID{4, 5}, []model.TableID{6, 7}) + dispatcher.OnAgentSyncTaskStatuses("capture-1", []model.KeySpanID{1, 2, 3}, []model.KeySpanID{4, 5}, []model.KeySpanID{6, 7}) // We expect the `Sync` to be ignored. - checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1500, []model.TableID{1, 2, 3, 4, 5}, mockCaptureInfos) + checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1500, []model.KeySpanID{1, 2, 3, 4, 5}, mockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) } -func TestRemoveTable(t *testing.T) { +func TestRemoveKeySpan(t *testing.T) { t.Parallel() ctx := cdcContext.NewBackendContext4Test(false) @@ -252,48 +252,48 @@ func TestRemoveTable(t *testing.T) { ResolvedTs: 1500, }, } - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: 1, + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 1, CaptureID: "capture-1", - Status: util.RunningTable, + Status: util.RunningKeySpan, }) - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: 2, + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 2, CaptureID: "capture-2", - Status: util.RunningTable, + Status: util.RunningKeySpan, }) - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: 3, + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 3, CaptureID: "capture-1", - Status: util.RunningTable, + Status: util.RunningKeySpan, }) - checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1500, []model.TableID{1, 2, 3}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1500, []model.KeySpanID{1, 2, 3}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, model.Ts(1500), checkpointTs) require.Equal(t, model.Ts(1500), resolvedTs) - // Inject a dispatch table failure - communicator.On("DispatchTable", mock.Anything, "cf-1", model.TableID(3), "capture-1", true). + // Inject a dispatch keyspan failure + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", model.KeySpanID(3), "capture-1", true). Return(false, nil) - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1500, []model.TableID{1, 2}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1500, []model.KeySpanID{1, 2}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.AssertExpectations(t) communicator.Reset() - communicator.On("DispatchTable", mock.Anything, "cf-1", model.TableID(3), "capture-1", true). + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", model.KeySpanID(3), "capture-1", true). Return(true, nil) - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1500, []model.TableID{1, 2}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1500, []model.KeySpanID{1, 2}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.AssertExpectations(t) - dispatcher.OnAgentFinishedTableOperation("capture-1", 3) + dispatcher.OnAgentFinishedKeySpanOperation("capture-1", 3) communicator.Reset() - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1500, []model.TableID{1, 2}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1500, []model.KeySpanID{1, 2}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, model.Ts(1500), checkpointTs) require.Equal(t, model.Ts(1500), resolvedTs) @@ -325,25 +325,25 @@ func TestCaptureGone(t *testing.T) { ResolvedTs: 1500, }, } - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: 1, + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 1, CaptureID: "capture-1", - Status: util.RunningTable, + Status: util.RunningKeySpan, }) - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: 2, + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 2, CaptureID: "capture-2", - Status: util.RunningTable, + Status: util.RunningKeySpan, }) - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: 3, + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 3, CaptureID: "capture-1", - Status: util.RunningTable, + Status: util.RunningKeySpan, }) - communicator.On("DispatchTable", mock.Anything, "cf-1", model.TableID(2), "capture-1", false). + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", model.KeySpanID(2), "capture-1", false). Return(true, nil) - checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1500, []model.TableID{1, 2, 3}, mockCaptureInfos) + checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1500, []model.KeySpanID{1, 2, 3}, mockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) @@ -368,33 +368,33 @@ func TestCaptureRestarts(t *testing.T) { ResolvedTs: 1500, }, } - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: 1, + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 1, CaptureID: "capture-1", - Status: util.RunningTable, + Status: util.RunningKeySpan, }) - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: 2, + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 2, CaptureID: "capture-2", - Status: util.RunningTable, + Status: util.RunningKeySpan, }) - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: 3, + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 3, CaptureID: "capture-1", - Status: util.RunningTable, + Status: util.RunningKeySpan, }) - dispatcher.OnAgentSyncTaskStatuses("capture-2", []model.TableID{}, []model.TableID{}, []model.TableID{}) - communicator.On("DispatchTable", mock.Anything, "cf-1", model.TableID(2), "capture-2", false). + dispatcher.OnAgentSyncTaskStatuses("capture-2", []model.KeySpanID{}, []model.KeySpanID{}, []model.KeySpanID{}) + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", model.KeySpanID(2), "capture-2", false). Return(true, nil) - checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1500, []model.TableID{1, 2, 3}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1500, []model.KeySpanID{1, 2, 3}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.AssertExpectations(t) } -func TestCaptureGoneWhileMovingTable(t *testing.T) { +func TestCaptureGoneWhileMovingKeySpan(t *testing.T) { t.Parallel() mockCaptureInfos := map[model.CaptureID]*model.CaptureInfo{ @@ -423,39 +423,39 @@ func TestCaptureGoneWhileMovingTable(t *testing.T) { ResolvedTs: 1550, }, } - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: 1, + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 1, CaptureID: "capture-1", - Status: util.RunningTable, + Status: util.RunningKeySpan, }) - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: 2, + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 2, CaptureID: "capture-2", - Status: util.RunningTable, + Status: util.RunningKeySpan, }) - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: 3, + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 3, CaptureID: "capture-1", - Status: util.RunningTable, + Status: util.RunningKeySpan, }) - dispatcher.MoveTable(1, "capture-2") - communicator.On("DispatchTable", mock.Anything, "cf-1", model.TableID(1), "capture-1", true). + dispatcher.MoveKeySpan(1, "capture-2") + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", model.KeySpanID(1), "capture-1", true). Return(true, nil) - checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1300, []model.TableID{1, 2, 3}, mockCaptureInfos) + checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1300, []model.KeySpanID{1, 2, 3}, mockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.AssertExpectations(t) delete(mockCaptureInfos, "capture-2") - dispatcher.OnAgentFinishedTableOperation("capture-1", 1) + dispatcher.OnAgentFinishedKeySpanOperation("capture-1", 1) communicator.Reset() - communicator.On("DispatchTable", mock.Anything, "cf-1", model.TableID(1), mock.Anything, false). + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", model.KeySpanID(1), mock.Anything, false). Return(true, nil) - communicator.On("DispatchTable", mock.Anything, "cf-1", model.TableID(2), mock.Anything, false). + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", model.KeySpanID(2), mock.Anything, false). Return(true, nil) - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1300, []model.TableID{1, 2, 3}, mockCaptureInfos) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1300, []model.KeySpanID{1, 2, 3}, mockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) @@ -501,31 +501,31 @@ func TestRebalance(t *testing.T) { }, } for i := 1; i <= 6; i++ { - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: model.TableID(i), + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: model.KeySpanID(i), CaptureID: fmt.Sprintf("capture-%d", (i+1)%2+1), - Status: util.RunningTable, + Status: util.RunningKeySpan, }) } dispatcher.Rebalance() - communicator.On("DispatchTable", mock.Anything, "cf-1", mock.Anything, mock.Anything, true). + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", mock.Anything, mock.Anything, true). Return(false, nil) - checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1300, []model.TableID{1, 2, 3, 4, 5, 6}, mockCaptureInfos) + checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1300, []model.KeySpanID{1, 2, 3, 4, 5, 6}, mockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.AssertExpectations(t) - communicator.AssertNumberOfCalls(t, "DispatchTable", 1) + communicator.AssertNumberOfCalls(t, "DispatchKeySpan", 1) communicator.Reset() - communicator.On("DispatchTable", mock.Anything, "cf-1", mock.Anything, mock.Anything, true). + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", mock.Anything, mock.Anything, true). Return(true, nil) - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1300, []model.TableID{1, 2, 3, 4, 5, 6}, mockCaptureInfos) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1300, []model.KeySpanID{1, 2, 3, 4, 5, 6}, mockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) - communicator.AssertNumberOfCalls(t, "DispatchTable", 2) + communicator.AssertNumberOfCalls(t, "DispatchKeySpan", 2) communicator.AssertExpectations(t) } @@ -568,14 +568,14 @@ func TestIgnoreEmptyCapture(t *testing.T) { }, } for i := 1; i <= 6; i++ { - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: model.TableID(i), + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: model.KeySpanID(i), CaptureID: fmt.Sprintf("capture-%d", (i+1)%2+1), - Status: util.RunningTable, + Status: util.RunningKeySpan, }) } - checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1300, []model.TableID{1, 2, 3, 4, 5, 6}, mockCaptureInfos) + checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1300, []model.KeySpanID{1, 2, 3, 4, 5, 6}, mockCaptureInfos) require.NoError(t, err) require.Equal(t, model.Ts(1300), checkpointTs) require.Equal(t, model.Ts(1550), resolvedTs) @@ -601,17 +601,17 @@ func TestIgnoreDeadCapture(t *testing.T) { }, } for i := 1; i <= 6; i++ { - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: model.TableID(i), + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: model.KeySpanID(i), CaptureID: fmt.Sprintf("capture-%d", (i+1)%2+1), - Status: util.RunningTable, + Status: util.RunningKeySpan, }) } // A dead capture sends very old watermarks. // They should be ignored. dispatcher.OnAgentCheckpoint("capture-3", 1000, 1000) - checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1300, []model.TableID{1, 2, 3, 4, 5, 6}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1300, []model.KeySpanID{1, 2, 3, 4, 5, 6}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, model.Ts(1300), checkpointTs) require.Equal(t, model.Ts(1550), resolvedTs) @@ -638,29 +638,29 @@ func TestIgnoreUnsyncedCaptures(t *testing.T) { } for i := 1; i <= 6; i++ { - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: model.TableID(i), + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: model.KeySpanID(i), CaptureID: fmt.Sprintf("capture-%d", (i+1)%2+1), - Status: util.RunningTable, + Status: util.RunningKeySpan, }) } dispatcher.OnAgentCheckpoint("capture-2", 1000, 1000) - checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1300, []model.TableID{1, 2, 3, 4, 5, 6}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1300, []model.KeySpanID{1, 2, 3, 4, 5, 6}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.Reset() - dispatcher.OnAgentSyncTaskStatuses("capture-2", []model.TableID{2, 4, 6}, []model.TableID{}, []model.TableID{}) - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1300, []model.TableID{1, 2, 3, 4, 5, 6}, defaultMockCaptureInfos) + dispatcher.OnAgentSyncTaskStatuses("capture-2", []model.KeySpanID{2, 4, 6}, []model.KeySpanID{}, []model.KeySpanID{}) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1300, []model.KeySpanID{1, 2, 3, 4, 5, 6}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, model.Ts(1300), checkpointTs) require.Equal(t, model.Ts(1500), resolvedTs) communicator.AssertExpectations(t) } -func TestRebalanceWhileAddingTable(t *testing.T) { +func TestRebalanceWhileAddingKeySpan(t *testing.T) { t.Parallel() ctx := cdcContext.NewBackendContext4Test(false) @@ -679,16 +679,16 @@ func TestRebalanceWhileAddingTable(t *testing.T) { }, } for i := 1; i <= 6; i++ { - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: model.TableID(i), + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: model.KeySpanID(i), CaptureID: "capture-1", - Status: util.RunningTable, + Status: util.RunningKeySpan, }) } - communicator.On("DispatchTable", mock.Anything, "cf-1", model.TableID(7), "capture-2", false). + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", model.KeySpanID(7), "capture-2", false). Return(true, nil) - checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1300, []model.TableID{1, 2, 3, 4, 5, 6, 7}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1300, []model.KeySpanID{1, 2, 3, 4, 5, 6, 7}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) @@ -696,25 +696,25 @@ func TestRebalanceWhileAddingTable(t *testing.T) { dispatcher.Rebalance() communicator.Reset() - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1300, []model.TableID{1, 2, 3, 4, 5, 6, 7}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1300, []model.KeySpanID{1, 2, 3, 4, 5, 6, 7}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.AssertExpectations(t) - dispatcher.OnAgentFinishedTableOperation("capture-2", model.TableID(7)) + dispatcher.OnAgentFinishedKeySpanOperation("capture-2", model.KeySpanID(7)) communicator.Reset() - communicator.On("DispatchTable", mock.Anything, "cf-1", mock.Anything, mock.Anything, true). + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", mock.Anything, mock.Anything, true). Return(true, nil) - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1300, []model.TableID{1, 2, 3, 4, 5, 6, 7}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1300, []model.KeySpanID{1, 2, 3, 4, 5, 6, 7}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) - communicator.AssertNumberOfCalls(t, "DispatchTable", 2) + communicator.AssertNumberOfCalls(t, "DispatchKeySpan", 2) communicator.AssertExpectations(t) } -func TestManualMoveTableWhileAddingTable(t *testing.T) { +func TestManualMoveKeySpanWhileAddingKeySpan(t *testing.T) { t.Parallel() ctx := cdcContext.NewBackendContext4Test(false) @@ -732,46 +732,46 @@ func TestManualMoveTableWhileAddingTable(t *testing.T) { ResolvedTs: 1550, }, } - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: 2, + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 2, CaptureID: "capture-1", - Status: util.RunningTable, + Status: util.RunningKeySpan, }) - dispatcher.tables.AddTableRecord(&util.TableRecord{ - TableID: 3, + dispatcher.keyspans.AddKeySpanRecord(&util.KeySpanRecord{ + KeySpanID: 3, CaptureID: "capture-1", - Status: util.RunningTable, + Status: util.RunningKeySpan, }) - communicator.On("DispatchTable", mock.Anything, "cf-1", model.TableID(1), "capture-2", false). + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", model.KeySpanID(1), "capture-2", false). Return(true, nil) - checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1300, []model.TableID{1, 2, 3}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1300, []model.KeySpanID{1, 2, 3}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) - dispatcher.MoveTable(1, "capture-1") - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1300, []model.TableID{1, 2, 3}, defaultMockCaptureInfos) + dispatcher.MoveKeySpan(1, "capture-1") + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1300, []model.KeySpanID{1, 2, 3}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.AssertExpectations(t) - dispatcher.OnAgentFinishedTableOperation("capture-2", 1) + dispatcher.OnAgentFinishedKeySpanOperation("capture-2", 1) communicator.Reset() - communicator.On("DispatchTable", mock.Anything, "cf-1", model.TableID(1), "capture-2", true). + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", model.KeySpanID(1), "capture-2", true). Return(true, nil) - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1300, []model.TableID{1, 2, 3}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1300, []model.KeySpanID{1, 2, 3}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.AssertExpectations(t) - dispatcher.OnAgentFinishedTableOperation("capture-2", 1) + dispatcher.OnAgentFinishedKeySpanOperation("capture-2", 1) communicator.Reset() - communicator.On("DispatchTable", mock.Anything, "cf-1", model.TableID(1), "capture-1", false). + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", model.KeySpanID(1), "capture-1", false). Return(true, nil) - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1300, []model.TableID{1, 2, 3}, defaultMockCaptureInfos) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1300, []model.KeySpanID{1, 2, 3}, defaultMockCaptureInfos) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) @@ -781,12 +781,12 @@ func TestManualMoveTableWhileAddingTable(t *testing.T) { func TestAutoRebalanceOnCaptureOnline(t *testing.T) { // This test case tests the following scenario: // 1. Capture-1 and Capture-2 are online. - // 2. Owner dispatches three tables to these two captures. + // 2. Owner dispatches three keyspans to these two captures. // 3. While the pending dispatches are in progress, Capture-3 goes online. // 4. Capture-1 and Capture-2 finish the dispatches. // // We expect that the workload is eventually balanced by migrating - // a table to Capture-3. + // a keyspan to Capture-3. t.Parallel() @@ -807,37 +807,37 @@ func TestAutoRebalanceOnCaptureOnline(t *testing.T) { communicator.On("Announce", mock.Anything, "cf-1", "capture-1").Return(true, nil) communicator.On("Announce", mock.Anything, "cf-1", "capture-2").Return(true, nil) - checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1000, []model.TableID{1, 2, 3}, captureList) + checkpointTs, resolvedTs, err := dispatcher.Tick(ctx, 1000, []model.KeySpanID{1, 2, 3}, captureList) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.AssertExpectations(t) - dispatcher.OnAgentSyncTaskStatuses("capture-1", []model.TableID{}, []model.TableID{}, []model.TableID{}) - dispatcher.OnAgentSyncTaskStatuses("capture-2", []model.TableID{}, []model.TableID{}, []model.TableID{}) + dispatcher.OnAgentSyncTaskStatuses("capture-1", []model.KeySpanID{}, []model.KeySpanID{}, []model.KeySpanID{}) + dispatcher.OnAgentSyncTaskStatuses("capture-2", []model.KeySpanID{}, []model.KeySpanID{}, []model.KeySpanID{}) communicator.Reset() - communicator.On("DispatchTable", mock.Anything, "cf-1", model.TableID(1), mock.Anything, false). + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", model.KeySpanID(1), mock.Anything, false). Return(true, nil) - communicator.On("DispatchTable", mock.Anything, "cf-1", model.TableID(2), mock.Anything, false). + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", model.KeySpanID(2), mock.Anything, false). Return(true, nil) - communicator.On("DispatchTable", mock.Anything, "cf-1", model.TableID(3), mock.Anything, false). + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", model.KeySpanID(3), mock.Anything, false). Return(true, nil) - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.TableID{1, 2, 3}, captureList) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.KeySpanID{1, 2, 3}, captureList) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.AssertExpectations(t) - require.NotEqual(t, 0, len(communicator.addTableRecords["capture-1"])) - require.NotEqual(t, 0, len(communicator.addTableRecords["capture-2"])) - require.Equal(t, 0, len(communicator.removeTableRecords["capture-1"])) - require.Equal(t, 0, len(communicator.removeTableRecords["capture-2"])) + require.NotEqual(t, 0, len(communicator.addKeySpanRecords["capture-1"])) + require.NotEqual(t, 0, len(communicator.addKeySpanRecords["capture-2"])) + require.Equal(t, 0, len(communicator.removeKeySpanRecords["capture-1"])) + require.Equal(t, 0, len(communicator.removeKeySpanRecords["capture-2"])) dispatcher.OnAgentCheckpoint("capture-1", 2000, 2000) dispatcher.OnAgentCheckpoint("capture-1", 2001, 2001) communicator.ExpectedCalls = nil - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.TableID{1, 2, 3}, captureList) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.KeySpanID{1, 2, 3}, captureList) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) @@ -850,47 +850,47 @@ func TestAutoRebalanceOnCaptureOnline(t *testing.T) { } communicator.ExpectedCalls = nil communicator.On("Announce", mock.Anything, "cf-1", "capture-3").Return(true, nil) - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.TableID{1, 2, 3}, captureList) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.KeySpanID{1, 2, 3}, captureList) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.AssertExpectations(t) communicator.ExpectedCalls = nil - dispatcher.OnAgentSyncTaskStatuses("capture-3", []model.TableID{}, []model.TableID{}, []model.TableID{}) - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.TableID{1, 2, 3}, captureList) + dispatcher.OnAgentSyncTaskStatuses("capture-3", []model.KeySpanID{}, []model.KeySpanID{}, []model.KeySpanID{}) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.KeySpanID{1, 2, 3}, captureList) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.AssertExpectations(t) - for captureID, tables := range communicator.addTableRecords { - for _, tableID := range tables { - dispatcher.OnAgentFinishedTableOperation(captureID, tableID) + for captureID, keyspans := range communicator.addKeySpanRecords { + for _, keyspanID := range keyspans { + dispatcher.OnAgentFinishedKeySpanOperation(captureID, keyspanID) } } communicator.Reset() - var removeTableFromCapture model.CaptureID - communicator.On("DispatchTable", mock.Anything, "cf-1", mock.Anything, mock.Anything, true). + var removeKeySpanFromCapture model.CaptureID + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", mock.Anything, mock.Anything, true). Return(true, nil).Run(func(args mock.Arguments) { - removeTableFromCapture = args.Get(3).(model.CaptureID) + removeKeySpanFromCapture = args.Get(3).(model.CaptureID) }) - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.TableID{1, 2, 3}, captureList) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.KeySpanID{1, 2, 3}, captureList) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) communicator.AssertExpectations(t) - removedTableID := communicator.removeTableRecords[removeTableFromCapture][0] + removedKeySpanID := communicator.removeKeySpanRecords[removeKeySpanFromCapture][0] - dispatcher.OnAgentFinishedTableOperation(removeTableFromCapture, removedTableID) + dispatcher.OnAgentFinishedKeySpanOperation(removeKeySpanFromCapture, removedKeySpanID) dispatcher.OnAgentCheckpoint("capture-1", 1100, 1400) dispatcher.OnAgentCheckpoint("capture-2", 1200, 1300) communicator.ExpectedCalls = nil - communicator.On("DispatchTable", mock.Anything, "cf-1", removedTableID, "capture-3", false). + communicator.On("DispatchKeySpan", mock.Anything, "cf-1", removedKeySpanID, "capture-3", false). Return(true, nil) - checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.TableID{1, 2, 3}, captureList) + checkpointTs, resolvedTs, err = dispatcher.Tick(ctx, 1000, []model.KeySpanID{1, 2, 3}, captureList) require.NoError(t, err) require.Equal(t, CheckpointCannotProceed, checkpointTs) require.Equal(t, CheckpointCannotProceed, resolvedTs) diff --git a/cdc/cdc/scheduler/util/keyspan_set.go b/cdc/cdc/scheduler/util/keyspan_set.go new file mode 100644 index 00000000..bd0c361a --- /dev/null +++ b/cdc/cdc/scheduler/util/keyspan_set.go @@ -0,0 +1,211 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "github.com/pingcap/log" + "github.com/tikv/migration/cdc/cdc/model" + "go.uber.org/zap" +) + +// KeySpanSet provides a data structure to store the keyspans' states for the +// scheduler. +type KeySpanSet struct { + // all keyspans' records + keyspanIDMap map[model.KeySpanID]*KeySpanRecord + + // a non-unique index to facilitate looking up keyspans + // assigned to a given capture. + captureIndex map[model.CaptureID]map[model.KeySpanID]*KeySpanRecord +} + +// keyspanRecord is a record to be inserted into keyspanSet. +type KeySpanRecord struct { + KeySpanID model.KeySpanID + CaptureID model.CaptureID + Status KeySpanStatus +} + +// Clone returns a copy of the KeySpanSet. +// This method is future-proof in case we add +// something not trivially copyable. +func (r *KeySpanRecord) Clone() *KeySpanRecord { + return &KeySpanRecord{ + KeySpanID: r.KeySpanID, + CaptureID: r.CaptureID, + Status: r.Status, + } +} + +// KeySpanStatus is a type representing the keyspan's replication status. +type KeySpanStatus int32 + +const ( + AddingKeySpan = KeySpanStatus(iota) + 1 + RemovingKeySpan + RunningKeySpan +) + +// NewKeySpanSet creates a new KeySpanSet. +func NewKeySpanSet() *KeySpanSet { + return &KeySpanSet{ + keyspanIDMap: map[model.KeySpanID]*KeySpanRecord{}, + captureIndex: map[model.CaptureID]map[model.KeySpanID]*KeySpanRecord{}, + } +} + +// AddKeySpanRecord inserts a new KeySpanRecord. +// It returns true if it succeeds. Returns false if there is a duplicate. +func (s *KeySpanSet) AddKeySpanRecord(record *KeySpanRecord) (successful bool) { + if _, ok := s.keyspanIDMap[record.KeySpanID]; ok { + // duplicate KeySpanID + return false + } + recordCloned := record.Clone() + s.keyspanIDMap[record.KeySpanID] = recordCloned + + captureIndexEntry := s.captureIndex[record.CaptureID] + if captureIndexEntry == nil { + captureIndexEntry = make(map[model.KeySpanID]*KeySpanRecord) + s.captureIndex[record.CaptureID] = captureIndexEntry + } + + captureIndexEntry[record.KeySpanID] = recordCloned + return true +} + +// UpdateKeySpanRecord updates an existing KeySpanRecord. +// All modifications to a keyspan's status should be done by this method. +func (s *KeySpanSet) UpdateKeySpanRecord(record *KeySpanRecord) (successful bool) { + oldRecord, ok := s.keyspanIDMap[record.KeySpanID] + if !ok { + // keyspan does not exist + return false + } + + // If there is no need to modify the CaptureID, we simply + // update the record. + if record.CaptureID == oldRecord.CaptureID { + recordCloned := record.Clone() + s.keyspanIDMap[record.KeySpanID] = recordCloned + s.captureIndex[record.CaptureID][record.KeySpanID] = recordCloned + return true + } + + // If the CaptureID is changed, we do a proper RemoveKeySpanRecord followed + // by AddKeySpanRecord. + if record.CaptureID != oldRecord.CaptureID { + if ok := s.RemoveKeySpanRecord(record.KeySpanID); !ok { + log.Panic("unreachable", zap.Any("record", record)) + } + if ok := s.AddKeySpanRecord(record); !ok { + log.Panic("unreachable", zap.Any("record", record)) + } + } + return true +} + +// GetKeySpanRecord tries to obtain a record with the specified keyspanID. +func (s *KeySpanSet) GetKeySpanRecord(keyspanID model.KeySpanID) (*KeySpanRecord, bool) { + rec, ok := s.keyspanIDMap[keyspanID] + if ok { + return rec.Clone(), ok + } + return nil, false +} + +// RemoveKeySpanRecord removes the record with keyspanID. Returns false +// if none exists. +func (s *KeySpanSet) RemoveKeySpanRecord(keyspanID model.KeySpanID) bool { + record, ok := s.keyspanIDMap[keyspanID] + if !ok { + return false + } + delete(s.keyspanIDMap, record.KeySpanID) + + captureIndexEntry, ok := s.captureIndex[record.CaptureID] + if !ok { + log.Panic("unreachable", zap.Uint64("keyspan-id", keyspanID)) + } + delete(captureIndexEntry, record.KeySpanID) + if len(captureIndexEntry) == 0 { + delete(s.captureIndex, record.CaptureID) + } + return true +} + +// RemoveKeySpanRecordByCaptureID removes all keyspan records associated with +// captureID. +func (s *KeySpanSet) RemoveKeySpanRecordByCaptureID(captureID model.CaptureID) []*KeySpanRecord { + captureIndexEntry, ok := s.captureIndex[captureID] + if !ok { + return nil + } + + var ret []*KeySpanRecord + for keyspanID, record := range captureIndexEntry { + delete(s.keyspanIDMap, keyspanID) + // Since the record has been removed, + // there is no need to clone it before returning. + ret = append(ret, record) + } + delete(s.captureIndex, captureID) + return ret +} + +// CountKeySpanByCaptureID counts the number of keyspans associated with the captureID. +func (s *KeySpanSet) CountKeySpanByCaptureID(captureID model.CaptureID) int { + return len(s.captureIndex[captureID]) +} + +// GetDistinctCaptures counts distinct captures with keyspans. +func (s *KeySpanSet) GetDistinctCaptures() []model.CaptureID { + var ret []model.CaptureID + for captureID := range s.captureIndex { + ret = append(ret, captureID) + } + return ret +} + +// GetAllKeySpans returns all stored information on all keyspans. +func (s *KeySpanSet) GetAllKeySpans() map[model.KeySpanID]*KeySpanRecord { + ret := make(map[model.KeySpanID]*KeySpanRecord) + for keyspanID, record := range s.keyspanIDMap { + ret[keyspanID] = record.Clone() + } + return ret +} + +// GetAllKeySpansGroupedByCaptures returns all stored information grouped by associated CaptureID. +func (s *KeySpanSet) GetAllKeySpansGroupedByCaptures() map[model.CaptureID]map[model.KeySpanID]*KeySpanRecord { + ret := make(map[model.CaptureID]map[model.KeySpanID]*KeySpanRecord) + for captureID, keyspanIDMap := range s.captureIndex { + keyspanIDMapCloned := make(map[model.KeySpanID]*KeySpanRecord) + for keyspanID, record := range keyspanIDMap { + keyspanIDMapCloned[keyspanID] = record.Clone() + } + ret[captureID] = keyspanIDMapCloned + } + return ret +} + +// CountKeySpanByStatus counts the number of keyspans with the given status. +func (s *KeySpanSet) CountKeySpanByStatus(status KeySpanStatus) (count int) { + for _, record := range s.keyspanIDMap { + if record.Status == status { + count++ + } + } + return +} diff --git a/cdc/cdc/scheduler/util/keyspan_set_test.go b/cdc/cdc/scheduler/util/keyspan_set_test.go new file mode 100644 index 00000000..33953e0c --- /dev/null +++ b/cdc/cdc/scheduler/util/keyspan_set_test.go @@ -0,0 +1,270 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/tikv/migration/cdc/cdc/model" +) + +func TestKeySpanSetBasics(t *testing.T) { + ts := NewKeySpanSet() + ok := ts.AddKeySpanRecord(&KeySpanRecord{ + KeySpanID: 1, + CaptureID: "capture-1", + Status: AddingKeySpan, + }) + require.True(t, ok) + + ok = ts.AddKeySpanRecord(&KeySpanRecord{ + KeySpanID: 1, + CaptureID: "capture-2", + Status: AddingKeySpan, + }) + // Adding a duplicate keyspan record should fail + require.False(t, ok) + + record, ok := ts.GetKeySpanRecord(1) + require.True(t, ok) + require.Equal(t, &KeySpanRecord{ + KeySpanID: 1, + CaptureID: "capture-1", + Status: AddingKeySpan, + }, record) + + ok = ts.RemoveKeySpanRecord(1) + require.True(t, ok) + + ok = ts.RemoveKeySpanRecord(2) + require.False(t, ok) +} + +func TestKeySpanSetCaptures(t *testing.T) { + ts := NewKeySpanSet() + ok := ts.AddKeySpanRecord(&KeySpanRecord{ + KeySpanID: 1, + CaptureID: "capture-1", + Status: AddingKeySpan, + }) + require.True(t, ok) + + ok = ts.AddKeySpanRecord(&KeySpanRecord{ + KeySpanID: 2, + CaptureID: "capture-1", + Status: AddingKeySpan, + }) + require.True(t, ok) + + ok = ts.AddKeySpanRecord(&KeySpanRecord{ + KeySpanID: 3, + CaptureID: "capture-2", + Status: AddingKeySpan, + }) + require.True(t, ok) + + ok = ts.AddKeySpanRecord(&KeySpanRecord{ + KeySpanID: 4, + CaptureID: "capture-2", + Status: AddingKeySpan, + }) + require.True(t, ok) + + ok = ts.AddKeySpanRecord(&KeySpanRecord{ + KeySpanID: 5, + CaptureID: "capture-3", + Status: AddingKeySpan, + }) + require.True(t, ok) + + require.Equal(t, 2, ts.CountKeySpanByCaptureID("capture-1")) + require.Equal(t, 2, ts.CountKeySpanByCaptureID("capture-2")) + require.Equal(t, 1, ts.CountKeySpanByCaptureID("capture-3")) + + ok = ts.AddKeySpanRecord(&KeySpanRecord{ + KeySpanID: 6, + CaptureID: "capture-3", + Status: AddingKeySpan, + }) + require.True(t, ok) + require.Equal(t, 2, ts.CountKeySpanByCaptureID("capture-3")) + + captures := ts.GetDistinctCaptures() + require.Len(t, captures, 3) + require.Contains(t, captures, "capture-1") + require.Contains(t, captures, "capture-2") + require.Contains(t, captures, "capture-3") + + ok = ts.RemoveKeySpanRecord(3) + require.True(t, ok) + ok = ts.RemoveKeySpanRecord(4) + require.True(t, ok) + + captures = ts.GetDistinctCaptures() + require.Len(t, captures, 2) + require.Contains(t, captures, "capture-1") + require.Contains(t, captures, "capture-3") + + captureToKeySpanMap := ts.GetAllKeySpansGroupedByCaptures() + require.Equal(t, map[model.CaptureID]map[model.KeySpanID]*KeySpanRecord{ + "capture-1": { + 1: &KeySpanRecord{ + KeySpanID: 1, + CaptureID: "capture-1", + Status: AddingKeySpan, + }, + 2: &KeySpanRecord{ + KeySpanID: 2, + CaptureID: "capture-1", + Status: AddingKeySpan, + }, + }, + "capture-3": { + 5: &KeySpanRecord{ + KeySpanID: 5, + CaptureID: "capture-3", + Status: AddingKeySpan, + }, + 6: &KeySpanRecord{ + KeySpanID: 6, + CaptureID: "capture-3", + Status: AddingKeySpan, + }, + }, + }, captureToKeySpanMap) + + removed := ts.RemoveKeySpanRecordByCaptureID("capture-3") + require.Len(t, removed, 2) + require.Contains(t, removed, &KeySpanRecord{ + KeySpanID: 5, + CaptureID: "capture-3", + Status: AddingKeySpan, + }) + require.Contains(t, removed, &KeySpanRecord{ + KeySpanID: 6, + CaptureID: "capture-3", + Status: AddingKeySpan, + }) + + _, ok = ts.GetKeySpanRecord(5) + require.False(t, ok) + _, ok = ts.GetKeySpanRecord(6) + require.False(t, ok) + + allKeySpans := ts.GetAllKeySpans() + require.Equal(t, map[model.KeySpanID]*KeySpanRecord{ + 1: { + KeySpanID: 1, + CaptureID: "capture-1", + Status: AddingKeySpan, + }, + 2: { + KeySpanID: 2, + CaptureID: "capture-1", + Status: AddingKeySpan, + }, + }, allKeySpans) + + ok = ts.RemoveKeySpanRecord(1) + require.True(t, ok) + ok = ts.RemoveKeySpanRecord(2) + require.True(t, ok) + + captureToKeySpanMap = ts.GetAllKeySpansGroupedByCaptures() + require.Len(t, captureToKeySpanMap, 0) +} + +func TestCountKeySpanByStatus(t *testing.T) { + ts := NewKeySpanSet() + ok := ts.AddKeySpanRecord(&KeySpanRecord{ + KeySpanID: 1, + CaptureID: "capture-1", + Status: AddingKeySpan, + }) + require.True(t, ok) + + ok = ts.AddKeySpanRecord(&KeySpanRecord{ + KeySpanID: 2, + CaptureID: "capture-1", + Status: RunningKeySpan, + }) + require.True(t, ok) + + ok = ts.AddKeySpanRecord(&KeySpanRecord{ + KeySpanID: 3, + CaptureID: "capture-2", + Status: RemovingKeySpan, + }) + require.True(t, ok) + + ok = ts.AddKeySpanRecord(&KeySpanRecord{ + KeySpanID: 4, + CaptureID: "capture-2", + Status: AddingKeySpan, + }) + require.True(t, ok) + + ok = ts.AddKeySpanRecord(&KeySpanRecord{ + KeySpanID: 5, + CaptureID: "capture-3", + Status: RunningKeySpan, + }) + require.True(t, ok) + + require.Equal(t, 2, ts.CountKeySpanByStatus(AddingKeySpan)) + require.Equal(t, 2, ts.CountKeySpanByStatus(RunningKeySpan)) + require.Equal(t, 1, ts.CountKeySpanByStatus(RemovingKeySpan)) +} + +func TestUpdateKeySpanRecord(t *testing.T) { + ts := NewKeySpanSet() + ok := ts.AddKeySpanRecord(&KeySpanRecord{ + KeySpanID: 4, + CaptureID: "capture-2", + Status: AddingKeySpan, + }) + require.True(t, ok) + + ok = ts.AddKeySpanRecord(&KeySpanRecord{ + KeySpanID: 5, + CaptureID: "capture-3", + Status: AddingKeySpan, + }) + require.True(t, ok) + + ok = ts.UpdateKeySpanRecord(&KeySpanRecord{ + KeySpanID: 5, + CaptureID: "capture-3", + Status: RunningKeySpan, + }) + require.True(t, ok) + + rec, ok := ts.GetKeySpanRecord(5) + require.True(t, ok) + require.Equal(t, RunningKeySpan, rec.Status) + require.Equal(t, RunningKeySpan, ts.GetAllKeySpansGroupedByCaptures()["capture-3"][5].Status) + + ok = ts.UpdateKeySpanRecord(&KeySpanRecord{ + KeySpanID: 4, + CaptureID: "capture-3", + Status: RunningKeySpan, + }) + require.True(t, ok) + rec, ok = ts.GetKeySpanRecord(4) + require.True(t, ok) + require.Equal(t, RunningKeySpan, rec.Status) + require.Equal(t, "capture-3", rec.CaptureID) + require.Equal(t, RunningKeySpan, ts.GetAllKeySpansGroupedByCaptures()["capture-3"][4].Status) +} diff --git a/cdc/cdc/scheduler/util/sort_table_ids.go b/cdc/cdc/scheduler/util/sort_keyspan_ids.go similarity index 74% rename from cdc/cdc/scheduler/util/sort_table_ids.go rename to cdc/cdc/scheduler/util/sort_keyspan_ids.go index 96238e45..cf4cfcd3 100644 --- a/cdc/cdc/scheduler/util/sort_table_ids.go +++ b/cdc/cdc/scheduler/util/sort_keyspan_ids.go @@ -19,9 +19,9 @@ import ( "github.com/tikv/migration/cdc/cdc/model" ) -// SortTableIDs sorts a slice of table IDs in ascending order. -func SortTableIDs(tableIDs []model.TableID) { - sort.Slice(tableIDs, func(i, j int) bool { - return tableIDs[i] < tableIDs[j] +// SortKeySpanIDs sorts a slice of keyspan IDs in ascending order. +func SortKeySpanIDs(keyspanIDs []model.KeySpanID) { + sort.Slice(keyspanIDs, func(i, j int) bool { + return keyspanIDs[i] < keyspanIDs[j] }) } diff --git a/cdc/cdc/scheduler/util/table_set.go b/cdc/cdc/scheduler/util/table_set.go deleted file mode 100644 index ec728d8e..00000000 --- a/cdc/cdc/scheduler/util/table_set.go +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package util - -import ( - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" - "go.uber.org/zap" -) - -// TableSet provides a data structure to store the tables' states for the -// scheduler. -type TableSet struct { - // all tables' records - tableIDMap map[model.TableID]*TableRecord - - // a non-unique index to facilitate looking up tables - // assigned to a given capture. - captureIndex map[model.CaptureID]map[model.TableID]*TableRecord -} - -// TableRecord is a record to be inserted into TableSet. -type TableRecord struct { - TableID model.TableID - CaptureID model.CaptureID - Status TableStatus -} - -// Clone returns a copy of the TableSet. -// This method is future-proof in case we add -// something not trivially copyable. -func (r *TableRecord) Clone() *TableRecord { - return &TableRecord{ - TableID: r.TableID, - CaptureID: r.CaptureID, - Status: r.Status, - } -} - -// TableStatus is a type representing the table's replication status. -type TableStatus int32 - -const ( - AddingTable = TableStatus(iota) + 1 - RemovingTable - RunningTable -) - -// NewTableSet creates a new TableSet. -func NewTableSet() *TableSet { - return &TableSet{ - tableIDMap: map[model.TableID]*TableRecord{}, - captureIndex: map[model.CaptureID]map[model.TableID]*TableRecord{}, - } -} - -// AddTableRecord inserts a new TableRecord. -// It returns true if it succeeds. Returns false if there is a duplicate. -func (s *TableSet) AddTableRecord(record *TableRecord) (successful bool) { - if _, ok := s.tableIDMap[record.TableID]; ok { - // duplicate tableID - return false - } - recordCloned := record.Clone() - s.tableIDMap[record.TableID] = recordCloned - - captureIndexEntry := s.captureIndex[record.CaptureID] - if captureIndexEntry == nil { - captureIndexEntry = make(map[model.TableID]*TableRecord) - s.captureIndex[record.CaptureID] = captureIndexEntry - } - - captureIndexEntry[record.TableID] = recordCloned - return true -} - -// UpdateTableRecord updates an existing TableRecord. -// All modifications to a table's status should be done by this method. -func (s *TableSet) UpdateTableRecord(record *TableRecord) (successful bool) { - oldRecord, ok := s.tableIDMap[record.TableID] - if !ok { - // table does not exist - return false - } - - // If there is no need to modify the CaptureID, we simply - // update the record. - if record.CaptureID == oldRecord.CaptureID { - recordCloned := record.Clone() - s.tableIDMap[record.TableID] = recordCloned - s.captureIndex[record.CaptureID][record.TableID] = recordCloned - return true - } - - // If the CaptureID is changed, we do a proper RemoveTableRecord followed - // by AddTableRecord. - if record.CaptureID != oldRecord.CaptureID { - if ok := s.RemoveTableRecord(record.TableID); !ok { - log.Panic("unreachable", zap.Any("record", record)) - } - if ok := s.AddTableRecord(record); !ok { - log.Panic("unreachable", zap.Any("record", record)) - } - } - return true -} - -// GetTableRecord tries to obtain a record with the specified tableID. -func (s *TableSet) GetTableRecord(tableID model.TableID) (*TableRecord, bool) { - rec, ok := s.tableIDMap[tableID] - if ok { - return rec.Clone(), ok - } - return nil, false -} - -// RemoveTableRecord removes the record with tableID. Returns false -// if none exists. -func (s *TableSet) RemoveTableRecord(tableID model.TableID) bool { - record, ok := s.tableIDMap[tableID] - if !ok { - return false - } - delete(s.tableIDMap, record.TableID) - - captureIndexEntry, ok := s.captureIndex[record.CaptureID] - if !ok { - log.Panic("unreachable", zap.Int64("table-id", tableID)) - } - delete(captureIndexEntry, record.TableID) - if len(captureIndexEntry) == 0 { - delete(s.captureIndex, record.CaptureID) - } - return true -} - -// RemoveTableRecordByCaptureID removes all table records associated with -// captureID. -func (s *TableSet) RemoveTableRecordByCaptureID(captureID model.CaptureID) []*TableRecord { - captureIndexEntry, ok := s.captureIndex[captureID] - if !ok { - return nil - } - - var ret []*TableRecord - for tableID, record := range captureIndexEntry { - delete(s.tableIDMap, tableID) - // Since the record has been removed, - // there is no need to clone it before returning. - ret = append(ret, record) - } - delete(s.captureIndex, captureID) - return ret -} - -// CountTableByCaptureID counts the number of tables associated with the captureID. -func (s *TableSet) CountTableByCaptureID(captureID model.CaptureID) int { - return len(s.captureIndex[captureID]) -} - -// GetDistinctCaptures counts distinct captures with tables. -func (s *TableSet) GetDistinctCaptures() []model.CaptureID { - var ret []model.CaptureID - for captureID := range s.captureIndex { - ret = append(ret, captureID) - } - return ret -} - -// GetAllTables returns all stored information on all tables. -func (s *TableSet) GetAllTables() map[model.TableID]*TableRecord { - ret := make(map[model.TableID]*TableRecord) - for tableID, record := range s.tableIDMap { - ret[tableID] = record.Clone() - } - return ret -} - -// GetAllTablesGroupedByCaptures returns all stored information grouped by associated CaptureID. -func (s *TableSet) GetAllTablesGroupedByCaptures() map[model.CaptureID]map[model.TableID]*TableRecord { - ret := make(map[model.CaptureID]map[model.TableID]*TableRecord) - for captureID, tableIDMap := range s.captureIndex { - tableIDMapCloned := make(map[model.TableID]*TableRecord) - for tableID, record := range tableIDMap { - tableIDMapCloned[tableID] = record.Clone() - } - ret[captureID] = tableIDMapCloned - } - return ret -} - -// CountTableByStatus counts the number of tables with the given status. -func (s *TableSet) CountTableByStatus(status TableStatus) (count int) { - for _, record := range s.tableIDMap { - if record.Status == status { - count++ - } - } - return -} diff --git a/cdc/cdc/scheduler/util/table_set_test.go b/cdc/cdc/scheduler/util/table_set_test.go deleted file mode 100644 index 2da90be4..00000000 --- a/cdc/cdc/scheduler/util/table_set_test.go +++ /dev/null @@ -1,270 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package util - -import ( - "testing" - - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" -) - -func TestTableSetBasics(t *testing.T) { - ts := NewTableSet() - ok := ts.AddTableRecord(&TableRecord{ - TableID: 1, - CaptureID: "capture-1", - Status: AddingTable, - }) - require.True(t, ok) - - ok = ts.AddTableRecord(&TableRecord{ - TableID: 1, - CaptureID: "capture-2", - Status: AddingTable, - }) - // Adding a duplicate table record should fail - require.False(t, ok) - - record, ok := ts.GetTableRecord(1) - require.True(t, ok) - require.Equal(t, &TableRecord{ - TableID: 1, - CaptureID: "capture-1", - Status: AddingTable, - }, record) - - ok = ts.RemoveTableRecord(1) - require.True(t, ok) - - ok = ts.RemoveTableRecord(2) - require.False(t, ok) -} - -func TestTableSetCaptures(t *testing.T) { - ts := NewTableSet() - ok := ts.AddTableRecord(&TableRecord{ - TableID: 1, - CaptureID: "capture-1", - Status: AddingTable, - }) - require.True(t, ok) - - ok = ts.AddTableRecord(&TableRecord{ - TableID: 2, - CaptureID: "capture-1", - Status: AddingTable, - }) - require.True(t, ok) - - ok = ts.AddTableRecord(&TableRecord{ - TableID: 3, - CaptureID: "capture-2", - Status: AddingTable, - }) - require.True(t, ok) - - ok = ts.AddTableRecord(&TableRecord{ - TableID: 4, - CaptureID: "capture-2", - Status: AddingTable, - }) - require.True(t, ok) - - ok = ts.AddTableRecord(&TableRecord{ - TableID: 5, - CaptureID: "capture-3", - Status: AddingTable, - }) - require.True(t, ok) - - require.Equal(t, 2, ts.CountTableByCaptureID("capture-1")) - require.Equal(t, 2, ts.CountTableByCaptureID("capture-2")) - require.Equal(t, 1, ts.CountTableByCaptureID("capture-3")) - - ok = ts.AddTableRecord(&TableRecord{ - TableID: 6, - CaptureID: "capture-3", - Status: AddingTable, - }) - require.True(t, ok) - require.Equal(t, 2, ts.CountTableByCaptureID("capture-3")) - - captures := ts.GetDistinctCaptures() - require.Len(t, captures, 3) - require.Contains(t, captures, "capture-1") - require.Contains(t, captures, "capture-2") - require.Contains(t, captures, "capture-3") - - ok = ts.RemoveTableRecord(3) - require.True(t, ok) - ok = ts.RemoveTableRecord(4) - require.True(t, ok) - - captures = ts.GetDistinctCaptures() - require.Len(t, captures, 2) - require.Contains(t, captures, "capture-1") - require.Contains(t, captures, "capture-3") - - captureToTableMap := ts.GetAllTablesGroupedByCaptures() - require.Equal(t, map[model.CaptureID]map[model.TableID]*TableRecord{ - "capture-1": { - 1: &TableRecord{ - TableID: 1, - CaptureID: "capture-1", - Status: AddingTable, - }, - 2: &TableRecord{ - TableID: 2, - CaptureID: "capture-1", - Status: AddingTable, - }, - }, - "capture-3": { - 5: &TableRecord{ - TableID: 5, - CaptureID: "capture-3", - Status: AddingTable, - }, - 6: &TableRecord{ - TableID: 6, - CaptureID: "capture-3", - Status: AddingTable, - }, - }, - }, captureToTableMap) - - removed := ts.RemoveTableRecordByCaptureID("capture-3") - require.Len(t, removed, 2) - require.Contains(t, removed, &TableRecord{ - TableID: 5, - CaptureID: "capture-3", - Status: AddingTable, - }) - require.Contains(t, removed, &TableRecord{ - TableID: 6, - CaptureID: "capture-3", - Status: AddingTable, - }) - - _, ok = ts.GetTableRecord(5) - require.False(t, ok) - _, ok = ts.GetTableRecord(6) - require.False(t, ok) - - allTables := ts.GetAllTables() - require.Equal(t, map[model.TableID]*TableRecord{ - 1: { - TableID: 1, - CaptureID: "capture-1", - Status: AddingTable, - }, - 2: { - TableID: 2, - CaptureID: "capture-1", - Status: AddingTable, - }, - }, allTables) - - ok = ts.RemoveTableRecord(1) - require.True(t, ok) - ok = ts.RemoveTableRecord(2) - require.True(t, ok) - - captureToTableMap = ts.GetAllTablesGroupedByCaptures() - require.Len(t, captureToTableMap, 0) -} - -func TestCountTableByStatus(t *testing.T) { - ts := NewTableSet() - ok := ts.AddTableRecord(&TableRecord{ - TableID: 1, - CaptureID: "capture-1", - Status: AddingTable, - }) - require.True(t, ok) - - ok = ts.AddTableRecord(&TableRecord{ - TableID: 2, - CaptureID: "capture-1", - Status: RunningTable, - }) - require.True(t, ok) - - ok = ts.AddTableRecord(&TableRecord{ - TableID: 3, - CaptureID: "capture-2", - Status: RemovingTable, - }) - require.True(t, ok) - - ok = ts.AddTableRecord(&TableRecord{ - TableID: 4, - CaptureID: "capture-2", - Status: AddingTable, - }) - require.True(t, ok) - - ok = ts.AddTableRecord(&TableRecord{ - TableID: 5, - CaptureID: "capture-3", - Status: RunningTable, - }) - require.True(t, ok) - - require.Equal(t, 2, ts.CountTableByStatus(AddingTable)) - require.Equal(t, 2, ts.CountTableByStatus(RunningTable)) - require.Equal(t, 1, ts.CountTableByStatus(RemovingTable)) -} - -func TestUpdateTableRecord(t *testing.T) { - ts := NewTableSet() - ok := ts.AddTableRecord(&TableRecord{ - TableID: 4, - CaptureID: "capture-2", - Status: AddingTable, - }) - require.True(t, ok) - - ok = ts.AddTableRecord(&TableRecord{ - TableID: 5, - CaptureID: "capture-3", - Status: AddingTable, - }) - require.True(t, ok) - - ok = ts.UpdateTableRecord(&TableRecord{ - TableID: 5, - CaptureID: "capture-3", - Status: RunningTable, - }) - require.True(t, ok) - - rec, ok := ts.GetTableRecord(5) - require.True(t, ok) - require.Equal(t, RunningTable, rec.Status) - require.Equal(t, RunningTable, ts.GetAllTablesGroupedByCaptures()["capture-3"][5].Status) - - ok = ts.UpdateTableRecord(&TableRecord{ - TableID: 4, - CaptureID: "capture-3", - Status: RunningTable, - }) - require.True(t, ok) - rec, ok = ts.GetTableRecord(4) - require.True(t, ok) - require.Equal(t, RunningTable, rec.Status) - require.Equal(t, "capture-3", rec.CaptureID) - require.Equal(t, RunningTable, ts.GetAllTablesGroupedByCaptures()["capture-3"][4].Status) -} diff --git a/cdc/cdc/server.go b/cdc/cdc/server.go index 8f9b3038..689081ca 100644 --- a/cdc/cdc/server.go +++ b/cdc/cdc/server.go @@ -28,7 +28,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/tikv/migration/cdc/cdc/capture" "github.com/tikv/migration/cdc/cdc/kv" - "github.com/tikv/migration/cdc/cdc/sorter/unified" "github.com/tikv/migration/cdc/pkg/config" cerror "github.com/tikv/migration/cdc/pkg/errors" "github.com/tikv/migration/cdc/pkg/etcd" @@ -262,9 +261,11 @@ func (s *Server) run(ctx context.Context) (err error) { return s.etcdHealthChecker(cctx) }) - wg.Go(func() error { - return unified.RunWorkerPool(cctx) - }) + /* + wg.Go(func() error { + return unified.RunWorkerPool(cctx) + }) + */ wg.Go(func() error { return kv.RunWorkerPool(cctx) diff --git a/cdc/cdc/sink/black_hole.go b/cdc/cdc/sink/black_hole.go index cf9dc21f..79d2b4af 100644 --- a/cdc/cdc/sink/black_hole.go +++ b/cdc/cdc/sink/black_hole.go @@ -35,17 +35,17 @@ type blackHoleSink struct { lastAccumulated uint64 } -func (b *blackHoleSink) EmitRowChangedEvents(ctx context.Context, rows ...*model.RowChangedEvent) error { - for _, row := range rows { - log.Debug("BlockHoleSink: EmitRowChangedEvents", zap.Any("row", row)) +func (b *blackHoleSink) EmitChangedEvents(ctx context.Context, rawKVEntries ...*model.RawKVEntry) error { + for _, rawKVEntry := range rawKVEntries { + log.Debug("BlockHoleSink: EmitRowChangedEvents", zap.Any("row", rawKVEntry)) } - rowsCount := len(rows) + rowsCount := len(rawKVEntries) atomic.AddUint64(&b.accumulated, uint64(rowsCount)) b.statistics.AddRowsCount(rowsCount) return nil } -func (b *blackHoleSink) FlushRowChangedEvents(ctx context.Context, _ model.TableID, resolvedTs uint64) (uint64, error) { +func (b *blackHoleSink) FlushChangedEvents(ctx context.Context, _ model.KeySpanID, resolvedTs uint64) (uint64, error) { log.Debug("BlockHoleSink: FlushRowChangedEvents", zap.Uint64("resolvedTs", resolvedTs)) err := b.statistics.RecordBatchExecution(func() (int, error) { // TODO: add some random replication latency @@ -72,6 +72,6 @@ func (b *blackHoleSink) Close(ctx context.Context) error { return nil } -func (b *blackHoleSink) Barrier(ctx context.Context, tableID model.TableID) error { +func (b *blackHoleSink) Barrier(ctx context.Context, keyspanID model.KeySpanID) error { return nil } diff --git a/cdc/cdc/sink/buffer_sink.go b/cdc/cdc/sink/buffer_sink.go index 9debba7e..7f43ed56 100644 --- a/cdc/cdc/sink/buffer_sink.go +++ b/cdc/cdc/sink/buffer_sink.go @@ -35,8 +35,8 @@ const maxFlushBatchSize = 512 type bufferSink struct { Sink changeFeedCheckpointTs uint64 - tableCheckpointTsMap sync.Map - buffer map[model.TableID][]*model.RowChangedEvent + keyspanCheckpointTsMap sync.Map + buffer map[model.KeySpanID][]*model.RawKVEntry bufferMu sync.Mutex flushTsChan chan flushMsg drawbackChan chan drawbackMsg @@ -49,8 +49,8 @@ func newBufferSink( ) *bufferSink { sink := &bufferSink{ Sink: backendSink, - // buffer shares the same flow control with table sink - buffer: make(map[model.TableID][]*model.RowChangedEvent), + // buffer shares the same flow control with keyspan sink + buffer: make(map[model.KeySpanID][]*model.RawKVEntry), changeFeedCheckpointTs: checkpointTs, flushTsChan: make(chan flushMsg, maxFlushBatchSize), drawbackChan: drawbackChan, @@ -103,7 +103,7 @@ func (b *bufferSink) runOnce(ctx context.Context, state *runState) (bool, error) return false, ctx.Err() case drawback := <-b.drawbackChan: b.bufferMu.Lock() - delete(b.buffer, drawback.tableID) + delete(b.buffer, drawback.keyspanID) b.bufferMu.Unlock() close(drawback.callback) case event := <-b.flushTsChan: @@ -123,37 +123,37 @@ func (b *bufferSink) runOnce(ctx context.Context, state *runState) (bool, error) startEmit := time.Now() // find all rows before resolvedTs and emit to backend sink for i := 0; i < batchSize; i++ { - tableID, resolvedTs := batch[i].tableID, batch[i].resolvedTs - rows := b.buffer[tableID] - i := sort.Search(len(rows), func(i int) bool { - return rows[i].CommitTs > resolvedTs + keyspanID, resolvedTs := batch[i].keyspanID, batch[i].resolvedTs + rawKVEntries := b.buffer[keyspanID] + + i := sort.Search(len(rawKVEntries), func(i int) bool { + return rawKVEntries[i].CRTs > resolvedTs }) if i == 0 { continue } state.metricTotalRows.Add(float64(i)) - err := b.Sink.EmitRowChangedEvents(ctx, rows[:i]...) + err := b.Sink.EmitChangedEvents(ctx, rawKVEntries...) if err != nil { b.bufferMu.Unlock() return false, errors.Trace(err) } - - // put remaining rows back to buffer + // put remaining rawKVEntries back to buffer // append to a new, fixed slice to avoid lazy GC - b.buffer[tableID] = append(make([]*model.RowChangedEvent, 0, len(rows[i:])), rows[i:]...) + b.buffer[keyspanID] = append(make([]*model.RawKVEntry, 0, len(rawKVEntries[i:])), rawKVEntries[i:]...) } b.bufferMu.Unlock() state.metricEmitRowDuration.Observe(time.Since(startEmit).Seconds()) startFlush := time.Now() for i := 0; i < batchSize; i++ { - tableID, resolvedTs := batch[i].tableID, batch[i].resolvedTs - checkpointTs, err := b.Sink.FlushRowChangedEvents(ctx, tableID, resolvedTs) + keyspanID, resolvedTs := batch[i].keyspanID, batch[i].resolvedTs + checkpointTs, err := b.Sink.FlushChangedEvents(ctx, keyspanID, resolvedTs) if err != nil { return false, errors.Trace(err) } - b.tableCheckpointTsMap.Store(tableID, checkpointTs) + b.keyspanCheckpointTsMap.Store(keyspanID, checkpointTs) } now := time.Now() state.metricFlushDuration.Observe(now.Sub(startFlush).Seconds()) @@ -167,41 +167,41 @@ func (b *bufferSink) runOnce(ctx context.Context, state *runState) (bool, error) return true, nil } -func (b *bufferSink) EmitRowChangedEvents(ctx context.Context, rows ...*model.RowChangedEvent) error { +func (b *bufferSink) EmitChangedEvents(ctx context.Context, rawKVEntries ...*model.RawKVEntry) error { select { case <-ctx.Done(): return ctx.Err() default: - if len(rows) == 0 { + if len(rawKVEntries) == 0 { return nil } - tableID := rows[0].Table.TableID + keyspanID := rawKVEntries[0].KeySpanID b.bufferMu.Lock() - b.buffer[tableID] = append(b.buffer[tableID], rows...) + b.buffer[keyspanID] = append(b.buffer[keyspanID], rawKVEntries...) b.bufferMu.Unlock() } return nil } -func (b *bufferSink) FlushRowChangedEvents(ctx context.Context, tableID model.TableID, resolvedTs uint64) (uint64, error) { +func (b *bufferSink) FlushChangedEvents(ctx context.Context, keyspanID model.KeySpanID, resolvedTs uint64) (uint64, error) { select { case <-ctx.Done(): - return b.getTableCheckpointTs(tableID), ctx.Err() + return b.getKeySpanCheckpointTs(keyspanID), ctx.Err() case b.flushTsChan <- flushMsg{ - tableID: tableID, + keyspanID: keyspanID, resolvedTs: resolvedTs, }: } - return b.getTableCheckpointTs(tableID), nil + return b.getKeySpanCheckpointTs(keyspanID), nil } type flushMsg struct { - tableID model.TableID + keyspanID model.KeySpanID resolvedTs uint64 } -func (b *bufferSink) getTableCheckpointTs(tableID model.TableID) uint64 { - checkPoints, ok := b.tableCheckpointTsMap.Load(tableID) +func (b *bufferSink) getKeySpanCheckpointTs(keyspanID model.KeySpanID) uint64 { + checkPoints, ok := b.keyspanCheckpointTsMap.Load(keyspanID) if ok { return checkPoints.(uint64) } diff --git a/cdc/cdc/sink/buffer_sink_test.go b/cdc/cdc/sink/buffer_sink_test.go index bdf09e65..e8faddcf 100644 --- a/cdc/cdc/sink/buffer_sink_test.go +++ b/cdc/cdc/sink/buffer_sink_test.go @@ -24,16 +24,16 @@ import ( "github.com/tikv/migration/cdc/cdc/model" ) -func TestTableIsNotFlushed(t *testing.T) { +func TestKeySpanIsNotFlushed(t *testing.T) { t.Parallel() b := bufferSink{changeFeedCheckpointTs: 1} - require.Equal(t, uint64(1), b.getTableCheckpointTs(2)) + require.Equal(t, uint64(1), b.getKeySpanCheckpointTs(2)) b.UpdateChangeFeedCheckpointTs(3) - require.Equal(t, uint64(3), b.getTableCheckpointTs(2)) + require.Equal(t, uint64(3), b.getKeySpanCheckpointTs(2)) } -func TestFlushTable(t *testing.T) { +func TestFlushKeySpan(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.TODO()) @@ -41,41 +41,37 @@ func TestFlushTable(t *testing.T) { b := newBufferSink(newBlackHoleSink(ctx, make(map[string]string)), 5, make(chan drawbackMsg)) go b.run(ctx, make(chan error)) - require.Equal(t, uint64(5), b.getTableCheckpointTs(2)) - require.Nil(t, b.EmitRowChangedEvents(ctx)) - tbl1 := &model.TableName{TableID: 1} - tbl2 := &model.TableName{TableID: 2} - tbl3 := &model.TableName{TableID: 3} - tbl4 := &model.TableName{TableID: 4} - require.Nil(t, b.EmitRowChangedEvents(ctx, []*model.RowChangedEvent{ - {CommitTs: 6, Table: tbl1}, - {CommitTs: 6, Table: tbl2}, - {CommitTs: 6, Table: tbl3}, - {CommitTs: 6, Table: tbl4}, - {CommitTs: 10, Table: tbl1}, - {CommitTs: 10, Table: tbl2}, - {CommitTs: 10, Table: tbl3}, - {CommitTs: 10, Table: tbl4}, + require.Equal(t, uint64(5), b.getKeySpanCheckpointTs(2)) + require.Nil(t, b.EmitChangedEvents(ctx)) + require.Nil(t, b.EmitChangedEvents(ctx, []*model.RawKVEntry{ + {KeySpanID: 1}, + {KeySpanID: 2}, + {KeySpanID: 3}, + {KeySpanID: 4}, + {KeySpanID: 1}, + {KeySpanID: 2}, + {KeySpanID: 3}, + {KeySpanID: 4}, }...)) - checkpoint, err := b.FlushRowChangedEvents(ctx, 1, 7) + checkpoint, err := b.FlushChangedEvents(ctx, 1, 7) require.True(t, checkpoint <= 7) require.Nil(t, err) - checkpoint, err = b.FlushRowChangedEvents(ctx, 2, 6) + checkpoint, err = b.FlushChangedEvents(ctx, 2, 6) require.True(t, checkpoint <= 6) require.Nil(t, err) - checkpoint, err = b.FlushRowChangedEvents(ctx, 3, 8) + checkpoint, err = b.FlushChangedEvents(ctx, 3, 8) require.True(t, checkpoint <= 8) require.Nil(t, err) time.Sleep(200 * time.Millisecond) - require.Equal(t, uint64(7), b.getTableCheckpointTs(1)) - require.Equal(t, uint64(6), b.getTableCheckpointTs(2)) - require.Equal(t, uint64(8), b.getTableCheckpointTs(3)) - require.Equal(t, uint64(5), b.getTableCheckpointTs(4)) + require.Equal(t, uint64(7), b.getKeySpanCheckpointTs(1)) + require.Equal(t, uint64(6), b.getKeySpanCheckpointTs(2)) + require.Equal(t, uint64(8), b.getKeySpanCheckpointTs(3)) + require.Equal(t, uint64(5), b.getKeySpanCheckpointTs(4)) b.UpdateChangeFeedCheckpointTs(6) - require.Equal(t, uint64(7), b.getTableCheckpointTs(1)) - require.Equal(t, uint64(6), b.getTableCheckpointTs(2)) - require.Equal(t, uint64(8), b.getTableCheckpointTs(3)) - require.Equal(t, uint64(6), b.getTableCheckpointTs(4)) + require.Equal(t, uint64(7), b.getKeySpanCheckpointTs(1)) + require.Equal(t, uint64(6), b.getKeySpanCheckpointTs(2)) + require.Equal(t, uint64(8), b.getKeySpanCheckpointTs(3)) + require.Equal(t, uint64(6), b.getKeySpanCheckpointTs(4)) } func TestFlushFailed(t *testing.T) { @@ -85,19 +81,19 @@ func TestFlushFailed(t *testing.T) { b := newBufferSink(newBlackHoleSink(ctx, make(map[string]string)), 5, make(chan drawbackMsg)) go b.run(ctx, make(chan error)) - checkpoint, err := b.FlushRowChangedEvents(ctx, 3, 8) + checkpoint, err := b.FlushChangedEvents(ctx, 3, 8) require.True(t, checkpoint <= 8) require.Nil(t, err) time.Sleep(200 * time.Millisecond) - require.Equal(t, uint64(8), b.getTableCheckpointTs(3)) + require.Equal(t, uint64(8), b.getKeySpanCheckpointTs(3)) cancel() - checkpoint, _ = b.FlushRowChangedEvents(ctx, 3, 18) + checkpoint, _ = b.FlushChangedEvents(ctx, 3, 18) require.Equal(t, uint64(8), checkpoint) - checkpoint, _ = b.FlushRowChangedEvents(ctx, 1, 18) + checkpoint, _ = b.FlushChangedEvents(ctx, 1, 18) require.Equal(t, uint64(5), checkpoint) time.Sleep(200 * time.Millisecond) - require.Equal(t, uint64(8), b.getTableCheckpointTs(3)) - require.Equal(t, uint64(5), b.getTableCheckpointTs(1)) + require.Equal(t, uint64(8), b.getKeySpanCheckpointTs(3)) + require.Equal(t, uint64(5), b.getKeySpanCheckpointTs(1)) } type benchSink struct { @@ -111,7 +107,7 @@ func (b *benchSink) EmitRowChangedEvents( } func (b *benchSink) FlushRowChangedEvents( - ctx context.Context, tableID model.TableID, resolvedTs uint64, + ctx context.Context, keyspanID model.KeySpanID, resolvedTs uint64, ) (uint64, error) { return 0, nil } @@ -131,14 +127,14 @@ func BenchmarkRun(b *testing.B) { s := newBufferSink(&benchSink{}, 5, make(chan drawbackMsg)) s.flushTsChan = make(chan flushMsg, count) for i := 0; i < count; i++ { - s.buffer[int64(i)] = []*model.RowChangedEvent{{CommitTs: 5}} + s.buffer[uint64(i)] = []*model.RawKVEntry{} } b.ResetTimer() - b.Run(fmt.Sprintf("%d table(s)", count), func(b *testing.B) { + b.Run(fmt.Sprintf("%d keyspan(s)", count), func(b *testing.B) { for i := 0; i < b.N; i++ { for j := 0; j < count; j++ { - s.flushTsChan <- flushMsg{tableID: int64(0)} + s.flushTsChan <- flushMsg{keyspanID: uint64(0)} } for len(s.flushTsChan) != 0 { keepRun, err := s.runOnce(ctx, &state) diff --git a/cdc/cdc/sink/causality.go b/cdc/cdc/sink/causality.go deleted file mode 100644 index b155beae..00000000 --- a/cdc/cdc/sink/causality.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package sink - -import ( - "encoding/binary" - - "github.com/pingcap/log" - "go.uber.org/zap" - - "github.com/tikv/migration/cdc/cdc/model" -) - -// causality provides a simple mechanism to improve the concurrency of SQLs execution under the premise of ensuring correctness. -// causality groups sqls that maybe contain causal relationships, and syncer executes them linearly. -// if some conflicts exist in more than one groups, then syncer waits all SQLs that are grouped be executed and reset causality. -// this mechanism meets quiescent consistency to ensure correctness. -type causality struct { - relations map[string]int -} - -func newCausality() *causality { - return &causality{ - relations: make(map[string]int), - } -} - -func (c *causality) add(keys [][]byte, idx int) { - if len(keys) == 0 { - return - } - - for _, key := range keys { - c.relations[string(key)] = idx - } -} - -func (c *causality) reset() { - c.relations = make(map[string]int) -} - -// detectConflict detects whether there is a conflict -func (c *causality) detectConflict(keys [][]byte) (bool, int) { - if len(keys) == 0 { - return false, 0 - } - - firstIdx := -1 - for _, key := range keys { - if idx, ok := c.relations[string(key)]; ok { - if firstIdx == -1 { - firstIdx = idx - } else if firstIdx != idx { - return true, -1 - } - } - } - - return firstIdx != -1, firstIdx -} - -func genTxnKeys(txn *model.SingleTableTxn) [][]byte { - if len(txn.Rows) == 0 { - return nil - } - keysSet := make(map[string]struct{}, len(txn.Rows)) - for _, row := range txn.Rows { - rowKeys := genRowKeys(row) - for _, key := range rowKeys { - keysSet[string(key)] = struct{}{} - } - } - keys := make([][]byte, 0, len(keysSet)) - for key := range keysSet { - keys = append(keys, []byte(key)) - } - return keys -} - -func genRowKeys(row *model.RowChangedEvent) [][]byte { - var keys [][]byte - if len(row.Columns) != 0 { - for iIdx, idxCol := range row.IndexColumns { - key := genKeyList(row.Columns, iIdx, idxCol, row.Table.TableID) - if len(key) == 0 { - continue - } - keys = append(keys, key) - } - } - if len(row.PreColumns) != 0 { - for iIdx, idxCol := range row.IndexColumns { - key := genKeyList(row.PreColumns, iIdx, idxCol, row.Table.TableID) - if len(key) == 0 { - continue - } - keys = append(keys, key) - } - } - if len(keys) == 0 { - // use table ID as key if no key generated (no PK/UK), - // no concurrence for rows in the same table. - log.Debug("use table id as the key", zap.Int64("tableID", row.Table.TableID)) - tableKey := make([]byte, 8) - binary.BigEndian.PutUint64(tableKey, uint64(row.Table.TableID)) - keys = [][]byte{tableKey} - } - return keys -} - -func genKeyList(columns []*model.Column, iIdx int, colIdx []int, tableID int64) []byte { - var key []byte - for _, i := range colIdx { - // if a column value is null, we can ignore this index - // If the index contain generated column, we can't use this key to detect conflict with other DML, - // Because such as insert can't specified the generated value. - if columns[i] == nil || columns[i].Value == nil || columns[i].Flag.IsGeneratedColumn() { - return nil - } - key = append(key, []byte(model.ColumnValueString(columns[i].Value))...) - key = append(key, 0) - } - if len(key) == 0 { - return nil - } - tableKey := make([]byte, 16) - binary.BigEndian.PutUint64(tableKey[:8], uint64(iIdx)) - binary.BigEndian.PutUint64(tableKey[8:], uint64(tableID)) - key = append(key, tableKey...) - return key -} diff --git a/cdc/cdc/sink/causality_test.go b/cdc/cdc/sink/causality_test.go deleted file mode 100644 index a50edf8c..00000000 --- a/cdc/cdc/sink/causality_test.go +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package sink - -import ( - "bytes" - "sort" - - "github.com/pingcap/check" - "github.com/pingcap/tidb/parser/mysql" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/util/testleak" -) - -type testCausalitySuite struct{} - -var _ = check.Suite(&testCausalitySuite{}) - -func (s *testCausalitySuite) TestCausality(c *check.C) { - defer testleak.AfterTest(c)() - rows := [][][]byte{ - {[]byte("a")}, - {[]byte("b")}, - {[]byte("c")}, - } - ca := newCausality() - for i, row := range rows { - conflict, idx := ca.detectConflict(row) - c.Assert(conflict, check.IsFalse) - c.Assert(idx, check.Equals, -1) - ca.add(row, i) - // Test for single key index conflict. - conflict, idx = ca.detectConflict(row) - c.Assert(conflict, check.IsTrue) - c.Assert(idx, check.Equals, i) - } - c.Assert(len(ca.relations), check.Equals, 3) - cases := []struct { - keys [][]byte - conflict bool - idx int - }{ - // Test for single key index conflict. - {[][]byte{[]byte("a"), []byte("ab")}, true, 0}, - {[][]byte{[]byte("b"), []byte("ba")}, true, 1}, - {[][]byte{[]byte("a"), []byte("a")}, true, 0}, - {[][]byte{[]byte("b"), []byte("b")}, true, 1}, - {[][]byte{[]byte("c"), []byte("c")}, true, 2}, - // Test for multi-key index conflict. - {[][]byte{[]byte("a"), []byte("b")}, true, -1}, - {[][]byte{[]byte("b"), []byte("a")}, true, -1}, - {[][]byte{[]byte("b"), []byte("c")}, true, -1}, - } - for _, cas := range cases { - conflict, idx := ca.detectConflict(cas.keys) - comment := check.Commentf("keys: %v", cas.keys) - c.Assert(conflict, check.Equals, cas.conflict, comment) - c.Assert(idx, check.Equals, cas.idx, comment) - } - ca.reset() - c.Assert(len(ca.relations), check.Equals, 0) -} - -func (s *testCausalitySuite) TestGenKeys(c *check.C) { - defer testleak.AfterTest(c)() - testCases := []struct { - txn *model.SingleTableTxn - expected [][]byte - }{{ - txn: &model.SingleTableTxn{}, - expected: nil, - }, { - txn: &model.SingleTableTxn{ - Rows: []*model.RowChangedEvent{ - { - StartTs: 418658114257813514, - CommitTs: 418658114257813515, - Table: &model.TableName{Schema: "common_1", Table: "uk_without_pk", TableID: 47}, - PreColumns: []*model.Column{nil, { - Name: "a1", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.MultipleKeyFlag | model.HandleKeyFlag, - Value: 12, - }, { - Name: "a3", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.MultipleKeyFlag | model.HandleKeyFlag, - Value: 1, - }}, - IndexColumns: [][]int{{1, 2}}, - }, { - StartTs: 418658114257813514, - CommitTs: 418658114257813515, - Table: &model.TableName{Schema: "common_1", Table: "uk_without_pk", TableID: 47}, - PreColumns: []*model.Column{nil, { - Name: "a1", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.MultipleKeyFlag | model.HandleKeyFlag, - Value: 1, - }, { - Name: "a3", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.MultipleKeyFlag | model.HandleKeyFlag, - Value: 21, - }}, - IndexColumns: [][]int{{1, 2}}, - }, - }, - }, - expected: [][]byte{ - {'1', '2', 0x0, '1', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 47}, - {'1', 0x0, '2', '1', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 47}, - }, - }, { - txn: &model.SingleTableTxn{ - Rows: []*model.RowChangedEvent{ - { - StartTs: 418658114257813514, - CommitTs: 418658114257813515, - Table: &model.TableName{Schema: "common_1", Table: "uk_without_pk", TableID: 47}, - PreColumns: []*model.Column{nil, { - Name: "a1", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.HandleKeyFlag, - Value: 12, - }, { - Name: "a3", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.HandleKeyFlag, - Value: 1, - }}, - IndexColumns: [][]int{{1}, {2}}, - }, { - StartTs: 418658114257813514, - CommitTs: 418658114257813515, - Table: &model.TableName{Schema: "common_1", Table: "uk_without_pk", TableID: 47}, - PreColumns: []*model.Column{nil, { - Name: "a1", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.HandleKeyFlag, - Value: 1, - }, { - Name: "a3", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.HandleKeyFlag, - Value: 21, - }}, - IndexColumns: [][]int{{1}, {2}}, - }, - }, - }, - expected: [][]byte{ - {'2', '1', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 47}, - {'1', '2', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 47}, - {'1', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 47}, - {'1', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 47}, - }, - }, { - txn: &model.SingleTableTxn{ - Rows: []*model.RowChangedEvent{ - { - StartTs: 418658114257813514, - CommitTs: 418658114257813515, - Table: &model.TableName{Schema: "common_1", Table: "uk_without_pk", TableID: 47}, - PreColumns: []*model.Column{nil, { - Name: "a1", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.NullableFlag, - Value: nil, - }, { - Name: "a3", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.NullableFlag, - Value: nil, - }}, - IndexColumns: [][]int{{1}, {2}}, - }, { - StartTs: 418658114257813514, - CommitTs: 418658114257813515, - Table: &model.TableName{Schema: "common_1", Table: "uk_without_pk", TableID: 47}, - PreColumns: []*model.Column{nil, { - Name: "a1", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.HandleKeyFlag, - Value: 1, - }, { - Name: "a3", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.HandleKeyFlag, - Value: 21, - }}, - IndexColumns: [][]int{{1}, {2}}, - }, - }, - }, - expected: [][]uint8{ - {'2', '1', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 47}, - {'1', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 47}, - {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 47}, - }, - }} - for _, tc := range testCases { - keys := genTxnKeys(tc.txn) - sort.Slice(keys, func(i, j int) bool { - return bytes.Compare(keys[i], keys[j]) > 0 - }) - c.Assert(keys, check.DeepEquals, tc.expected) - } -} diff --git a/cdc/cdc/sink/common/common.go b/cdc/cdc/sink/common/common.go deleted file mode 100644 index 27064f94..00000000 --- a/cdc/cdc/sink/common/common.go +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package common - -import ( - "sort" - "sync" - - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/filter" - "go.uber.org/zap" -) - -type txnsWithTheSameCommitTs struct { - txns map[model.Ts]*model.SingleTableTxn - commitTs model.Ts -} - -func (t *txnsWithTheSameCommitTs) Append(row *model.RowChangedEvent) { - if row.CommitTs != t.commitTs { - log.Panic("unexpected row change event", - zap.Uint64("commitTs of txn", t.commitTs), - zap.Any("row", row)) - } - if t.txns == nil { - t.txns = make(map[model.Ts]*model.SingleTableTxn) - } - txn, exist := t.txns[row.StartTs] - if !exist { - txn = &model.SingleTableTxn{ - StartTs: row.StartTs, - CommitTs: row.CommitTs, - Table: row.Table, - ReplicaID: row.ReplicaID, - } - t.txns[row.StartTs] = txn - } - txn.Append(row) -} - -// UnresolvedTxnCache caches unresolved txns -type UnresolvedTxnCache struct { - unresolvedTxnsMu sync.Mutex - unresolvedTxns map[model.TableID][]*txnsWithTheSameCommitTs -} - -// NewUnresolvedTxnCache returns a new UnresolvedTxnCache -func NewUnresolvedTxnCache() *UnresolvedTxnCache { - return &UnresolvedTxnCache{ - unresolvedTxns: make(map[model.TableID][]*txnsWithTheSameCommitTs), - } -} - -// Append adds unresolved rows to cache -// the rows inputed into this function will go through the following handling logic -// 1. group by tableID from one input stream -// 2. for each tableID stream, the callers of this function should **make sure** that the CommitTs of rows is **strictly increasing** -// 3. group by CommitTs, according to CommitTs cut the rows into many group of rows in the same CommitTs -// 4. group by StartTs, cause the StartTs is the unique identifier of the transaction, according to StartTs cut the rows into many txns -func (c *UnresolvedTxnCache) Append(filter *filter.Filter, rows ...*model.RowChangedEvent) int { - c.unresolvedTxnsMu.Lock() - defer c.unresolvedTxnsMu.Unlock() - appendRows := 0 - for _, row := range rows { - if filter != nil && filter.ShouldIgnoreDMLEvent(row.StartTs, row.Table.Schema, row.Table.Table) { - log.Info("Row changed event ignored", zap.Uint64("start-ts", row.StartTs)) - continue - } - txns := c.unresolvedTxns[row.Table.TableID] - if len(txns) == 0 || txns[len(txns)-1].commitTs != row.CommitTs { - // fail-fast check - if len(txns) != 0 && txns[len(txns)-1].commitTs > row.CommitTs { - log.Panic("the commitTs of the emit row is less than the received row", - zap.Stringer("table", row.Table), - zap.Uint64("emit row startTs", row.StartTs), - zap.Uint64("emit row commitTs", row.CommitTs), - zap.Uint64("last received row commitTs", txns[len(txns)-1].commitTs)) - } - txns = append(txns, &txnsWithTheSameCommitTs{ - commitTs: row.CommitTs, - }) - c.unresolvedTxns[row.Table.TableID] = txns - } - txns[len(txns)-1].Append(row) - appendRows++ - } - return appendRows -} - -// Resolved returns resolved txns according to resolvedTs -// The returned map contains many txns grouped by tableID. for each table, the each commitTs of txn in txns slice is strictly increasing -func (c *UnresolvedTxnCache) Resolved(resolvedTsMap *sync.Map) (map[model.TableID]uint64, map[model.TableID][]*model.SingleTableTxn) { - c.unresolvedTxnsMu.Lock() - defer c.unresolvedTxnsMu.Unlock() - if len(c.unresolvedTxns) == 0 { - return nil, nil - } - - return splitResolvedTxn(resolvedTsMap, c.unresolvedTxns) -} - -func splitResolvedTxn( - resolvedTsMap *sync.Map, unresolvedTxns map[model.TableID][]*txnsWithTheSameCommitTs, -) (flushedResolvedTsMap map[model.TableID]uint64, resolvedRowsMap map[model.TableID][]*model.SingleTableTxn) { - resolvedRowsMap = make(map[model.TableID][]*model.SingleTableTxn, len(unresolvedTxns)) - flushedResolvedTsMap = make(map[model.TableID]uint64, len(unresolvedTxns)) - for tableID, txns := range unresolvedTxns { - v, ok := resolvedTsMap.Load(tableID) - if !ok { - continue - } - resolvedTs := v.(uint64) - i := sort.Search(len(txns), func(i int) bool { - return txns[i].commitTs > resolvedTs - }) - if i == 0 { - continue - } - var resolvedTxnsWithTheSameCommitTs []*txnsWithTheSameCommitTs - if i == len(txns) { - resolvedTxnsWithTheSameCommitTs = txns - delete(unresolvedTxns, tableID) - } else { - resolvedTxnsWithTheSameCommitTs = txns[:i] - unresolvedTxns[tableID] = txns[i:] - } - var txnsLength int - for _, txns := range resolvedTxnsWithTheSameCommitTs { - txnsLength += len(txns.txns) - } - resolvedTxns := make([]*model.SingleTableTxn, 0, txnsLength) - for _, txns := range resolvedTxnsWithTheSameCommitTs { - for _, txn := range txns.txns { - resolvedTxns = append(resolvedTxns, txn) - } - } - resolvedRowsMap[tableID] = resolvedTxns - flushedResolvedTsMap[tableID] = resolvedTs - } - return -} diff --git a/cdc/cdc/sink/common/common_test.go b/cdc/cdc/sink/common/common_test.go deleted file mode 100644 index 5c5be8f0..00000000 --- a/cdc/cdc/sink/common/common_test.go +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package common - -import ( - "sort" - "sync" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/util/testleak" -) - -func TestSplitResolvedTxn(test *testing.T) { - defer testleak.AfterTestT(test)() - - testCases := [][]struct { - input []*model.RowChangedEvent - resolvedTsMap map[model.TableID]uint64 - expected map[model.TableID][]*model.SingleTableTxn - }{{{ // Testing basic transaction collocation, no txns with the same commitTs - input: []*model.RowChangedEvent{ - {StartTs: 1, CommitTs: 5, Table: &model.TableName{TableID: 1}}, - {StartTs: 1, CommitTs: 5, Table: &model.TableName{TableID: 1}}, - {StartTs: 1, CommitTs: 6, Table: &model.TableName{TableID: 2}}, - {StartTs: 1, CommitTs: 7, Table: &model.TableName{TableID: 3}}, - {StartTs: 1, CommitTs: 8, Table: &model.TableName{TableID: 1}}, - {StartTs: 1, CommitTs: 11, Table: &model.TableName{TableID: 1}}, - {StartTs: 1, CommitTs: 12, Table: &model.TableName{TableID: 2}}, - }, - resolvedTsMap: map[model.TableID]uint64{ - 1: uint64(6), - 2: uint64(6), - }, - expected: map[model.TableID][]*model.SingleTableTxn{ - 1: {{Table: &model.TableName{TableID: 1}, StartTs: 1, CommitTs: 5, Rows: []*model.RowChangedEvent{ - {StartTs: 1, CommitTs: 5, Table: &model.TableName{TableID: 1}}, - {StartTs: 1, CommitTs: 5, Table: &model.TableName{TableID: 1}}, - }}}, - 2: {{Table: &model.TableName{TableID: 2}, StartTs: 1, CommitTs: 6, Rows: []*model.RowChangedEvent{ - {StartTs: 1, CommitTs: 6, Table: &model.TableName{TableID: 2}}, - }}}, - }, - }, { - input: []*model.RowChangedEvent{ - {StartTs: 1, CommitTs: 8, Table: &model.TableName{TableID: 3}}, - }, - resolvedTsMap: map[model.TableID]uint64{ - 1: uint64(13), - 2: uint64(13), - 3: uint64(13), - }, - expected: map[model.TableID][]*model.SingleTableTxn{ - 1: {{Table: &model.TableName{TableID: 1}, StartTs: 1, CommitTs: 8, Rows: []*model.RowChangedEvent{ - {StartTs: 1, CommitTs: 8, Table: &model.TableName{TableID: 1}}, - }}, {Table: &model.TableName{TableID: 1}, StartTs: 1, CommitTs: 11, Rows: []*model.RowChangedEvent{ - {StartTs: 1, CommitTs: 11, Table: &model.TableName{TableID: 1}}, - }}}, - 2: {{Table: &model.TableName{TableID: 2}, StartTs: 1, CommitTs: 12, Rows: []*model.RowChangedEvent{ - {StartTs: 1, CommitTs: 12, Table: &model.TableName{TableID: 2}}, - }}}, - 3: {{Table: &model.TableName{TableID: 3}, StartTs: 1, CommitTs: 7, Rows: []*model.RowChangedEvent{ - {StartTs: 1, CommitTs: 7, Table: &model.TableName{TableID: 3}}, - }}, {Table: &model.TableName{TableID: 3}, StartTs: 1, CommitTs: 8, Rows: []*model.RowChangedEvent{ - {StartTs: 1, CommitTs: 8, Table: &model.TableName{TableID: 3}}, - }}}, - }, - }}, {{ // Testing the short circuit path - input: []*model.RowChangedEvent{}, - resolvedTsMap: map[model.TableID]uint64{ - 1: uint64(13), - 2: uint64(13), - 3: uint64(13), - }, - expected: nil, - }, { - input: []*model.RowChangedEvent{ - {StartTs: 1, CommitTs: 11, Table: &model.TableName{TableID: 1}}, - {StartTs: 1, CommitTs: 12, Table: &model.TableName{TableID: 1}}, - {StartTs: 1, CommitTs: 13, Table: &model.TableName{TableID: 2}}, - }, - resolvedTsMap: map[model.TableID]uint64{ - 1: uint64(6), - 2: uint64(6), - }, - expected: map[model.TableID][]*model.SingleTableTxn{}, - }}, {{ // Testing the txns with the same commitTs - input: []*model.RowChangedEvent{ - {StartTs: 1, CommitTs: 5, Table: &model.TableName{TableID: 1}}, - {StartTs: 1, CommitTs: 8, Table: &model.TableName{TableID: 1}}, - {StartTs: 1, CommitTs: 6, Table: &model.TableName{TableID: 2}}, - {StartTs: 2, CommitTs: 6, Table: &model.TableName{TableID: 2}}, - {StartTs: 2, CommitTs: 8, Table: &model.TableName{TableID: 1}}, - {StartTs: 1, CommitTs: 8, Table: &model.TableName{TableID: 1}}, - {StartTs: 2, CommitTs: 8, Table: &model.TableName{TableID: 1}}, - {StartTs: 1, CommitTs: 6, Table: &model.TableName{TableID: 2}}, - {StartTs: 1, CommitTs: 7, Table: &model.TableName{TableID: 2}}, - }, - resolvedTsMap: map[model.TableID]uint64{ - 1: uint64(6), - 2: uint64(6), - }, - expected: map[model.TableID][]*model.SingleTableTxn{ - 1: {{Table: &model.TableName{TableID: 1}, StartTs: 1, CommitTs: 5, Rows: []*model.RowChangedEvent{ - {StartTs: 1, CommitTs: 5, Table: &model.TableName{TableID: 1}}, - }}}, - 2: {{Table: &model.TableName{TableID: 2}, StartTs: 1, CommitTs: 6, Rows: []*model.RowChangedEvent{ - {StartTs: 1, CommitTs: 6, Table: &model.TableName{TableID: 2}}, - {StartTs: 1, CommitTs: 6, Table: &model.TableName{TableID: 2}}, - }}, {Table: &model.TableName{TableID: 2}, StartTs: 2, CommitTs: 6, Rows: []*model.RowChangedEvent{ - {StartTs: 2, CommitTs: 6, Table: &model.TableName{TableID: 2}}, - }}}, - }, - }, { - input: []*model.RowChangedEvent{ - {StartTs: 2, CommitTs: 7, Table: &model.TableName{TableID: 2}}, - {StartTs: 1, CommitTs: 7, Table: &model.TableName{TableID: 2}}, - {StartTs: 1, CommitTs: 8, Table: &model.TableName{TableID: 1}}, - {StartTs: 2, CommitTs: 8, Table: &model.TableName{TableID: 1}}, - {StartTs: 1, CommitTs: 9, Table: &model.TableName{TableID: 1}}, - }, - resolvedTsMap: map[model.TableID]uint64{ - 1: uint64(13), - 2: uint64(13), - }, - expected: map[model.TableID][]*model.SingleTableTxn{ - 1: {{Table: &model.TableName{TableID: 1}, StartTs: 1, CommitTs: 8, Rows: []*model.RowChangedEvent{ - {StartTs: 1, CommitTs: 8, Table: &model.TableName{TableID: 1}}, - {StartTs: 1, CommitTs: 8, Table: &model.TableName{TableID: 1}}, - {StartTs: 1, CommitTs: 8, Table: &model.TableName{TableID: 1}}, - }}, {Table: &model.TableName{TableID: 1}, StartTs: 2, CommitTs: 8, Rows: []*model.RowChangedEvent{ - {StartTs: 2, CommitTs: 8, Table: &model.TableName{TableID: 1}}, - {StartTs: 2, CommitTs: 8, Table: &model.TableName{TableID: 1}}, - {StartTs: 2, CommitTs: 8, Table: &model.TableName{TableID: 1}}, - }}, {Table: &model.TableName{TableID: 1}, StartTs: 1, CommitTs: 9, Rows: []*model.RowChangedEvent{ - {StartTs: 1, CommitTs: 9, Table: &model.TableName{TableID: 1}}, - }}}, - 2: {{Table: &model.TableName{TableID: 2}, StartTs: 1, CommitTs: 7, Rows: []*model.RowChangedEvent{ - {StartTs: 1, CommitTs: 7, Table: &model.TableName{TableID: 2}}, - {StartTs: 1, CommitTs: 7, Table: &model.TableName{TableID: 2}}, - }}, {Table: &model.TableName{TableID: 2}, StartTs: 2, CommitTs: 7, Rows: []*model.RowChangedEvent{ - {StartTs: 2, CommitTs: 7, Table: &model.TableName{TableID: 2}}, - }}}, - }, - }}} - for _, tc := range testCases { - cache := NewUnresolvedTxnCache() - for _, t := range tc { - cache.Append(nil, t.input...) - resolvedTsMap := sync.Map{} - for tableID, ts := range t.resolvedTsMap { - resolvedTsMap.Store(tableID, ts) - } - _, resolved := cache.Resolved(&resolvedTsMap) - for tableID, txns := range resolved { - sort.Slice(txns, func(i, j int) bool { - if txns[i].CommitTs != txns[j].CommitTs { - return txns[i].CommitTs < txns[j].CommitTs - } - return txns[i].StartTs < txns[j].StartTs - }) - resolved[tableID] = txns - } - require.Equal(test, t.expected, resolved, cmp.Diff(resolved, t.expected)) - } - } -} diff --git a/cdc/cdc/sink/common/flow_control.go b/cdc/cdc/sink/common/flow_control.go index 1696b59d..7855dba5 100644 --- a/cdc/cdc/sink/common/flow_control.go +++ b/cdc/cdc/sink/common/flow_control.go @@ -24,10 +24,10 @@ import ( "go.uber.org/zap" ) -// TableMemoryQuota is designed to curb the total memory consumption of processing -// the event streams in a table. -// A higher-level controller more suitable for direct use by the processor is TableFlowController. -type TableMemoryQuota struct { +// KeySpanMemoryQuota is designed to curb the total memory consumption of processing +// the event streams in a keyspan. +// A higher-level controller more suikeyspan for direct use by the processor is KeySpanFlowController. +type KeySpanMemoryQuota struct { Quota uint64 // should not be changed once intialized IsAborted uint32 @@ -38,10 +38,10 @@ type TableMemoryQuota struct { cond *sync.Cond } -// NewTableMemoryQuota creates a new TableMemoryQuota +// NewKeySpanMemoryQuota creates a new KeySpanMemoryQuota // quota: max advised memory consumption in bytes. -func NewTableMemoryQuota(quota uint64) *TableMemoryQuota { - ret := &TableMemoryQuota{ +func NewKeySpanMemoryQuota(quota uint64) *KeySpanMemoryQuota { + ret := &KeySpanMemoryQuota{ Quota: quota, mu: sync.Mutex{}, Consumed: 0, @@ -55,7 +55,7 @@ func NewTableMemoryQuota(quota uint64) *TableMemoryQuota { // block until enough memory has been freed up by Release. // blockCallBack will be called if the function will block. // Should be used with care to prevent deadlock. -func (c *TableMemoryQuota) ConsumeWithBlocking(nBytes uint64, blockCallBack func() error) error { +func (c *KeySpanMemoryQuota) ConsumeWithBlocking(nBytes uint64, blockCallBack func() error) error { if nBytes >= c.Quota { return cerrors.ErrFlowControllerEventLargerThanQuota.GenWithStackByArgs(nBytes, c.Quota) } @@ -91,7 +91,7 @@ func (c *TableMemoryQuota) ConsumeWithBlocking(nBytes uint64, blockCallBack func // ForceConsume is called when blocking is not acceptable and the limit can be violated // for the sake of avoid deadlock. It merely records the increased memory consumption. -func (c *TableMemoryQuota) ForceConsume(nBytes uint64) error { +func (c *KeySpanMemoryQuota) ForceConsume(nBytes uint64) error { c.mu.Lock() defer c.mu.Unlock() @@ -104,12 +104,12 @@ func (c *TableMemoryQuota) ForceConsume(nBytes uint64) error { } // Release is called when a chuck of memory is done being used. -func (c *TableMemoryQuota) Release(nBytes uint64) { +func (c *KeySpanMemoryQuota) Release(nBytes uint64) { c.mu.Lock() if c.Consumed < nBytes { c.mu.Unlock() - log.Panic("TableMemoryQuota: releasing more than consumed, report a bug", + log.Panic("KeySpanMemoryQuota: releasing more than consumed, report a bug", zap.Uint64("consumed", c.Consumed), zap.Uint64("released", nBytes)) } @@ -125,22 +125,22 @@ func (c *TableMemoryQuota) Release(nBytes uint64) { } // Abort interrupts any ongoing ConsumeWithBlocking call -func (c *TableMemoryQuota) Abort() { +func (c *KeySpanMemoryQuota) Abort() { atomic.StoreUint32(&c.IsAborted, 1) c.cond.Signal() } // GetConsumption returns the current memory consumption -func (c *TableMemoryQuota) GetConsumption() uint64 { +func (c *KeySpanMemoryQuota) GetConsumption() uint64 { c.mu.Lock() defer c.mu.Unlock() return c.Consumed } -// TableFlowController provides a convenient interface to control the memory consumption of a per table event stream -type TableFlowController struct { - memoryQuota *TableMemoryQuota +// KeySpanFlowController provides a convenient interface to control the memory consumption of a per keyspan event stream +type KeySpanFlowController struct { + memoryQuota *KeySpanMemoryQuota mu sync.Mutex queue deque.Deque @@ -153,17 +153,17 @@ type commitTsSizeEntry struct { Size uint64 } -// NewTableFlowController creates a new TableFlowController -func NewTableFlowController(quota uint64) *TableFlowController { - return &TableFlowController{ - memoryQuota: NewTableMemoryQuota(quota), +// NewKeySpanFlowController creates a new KeySpanFlowController +func NewKeySpanFlowController(quota uint64) *KeySpanFlowController { + return &KeySpanFlowController{ + memoryQuota: NewKeySpanMemoryQuota(quota), queue: deque.NewDeque(), } } // Consume is called when an event has arrived for being processed by the sink. // It will handle transaction boundaries automatically, and will not block intra-transaction. -func (c *TableFlowController) Consume(commitTs uint64, size uint64, blockCallBack func() error) error { +func (c *KeySpanFlowController) Consume(commitTs uint64, size uint64, blockCallBack func() error) error { lastCommitTs := atomic.LoadUint64(&c.lastCommitTs) if commitTs < lastCommitTs { @@ -201,7 +201,7 @@ func (c *TableFlowController) Consume(commitTs uint64, size uint64, blockCallBac } // Release is called when all events committed before resolvedTs has been freed from memory. -func (c *TableFlowController) Release(resolvedTs uint64) { +func (c *KeySpanFlowController) Release(resolvedTs uint64) { var nBytesToRelease uint64 c.mu.Lock() @@ -219,11 +219,11 @@ func (c *TableFlowController) Release(resolvedTs uint64) { } // Abort interrupts any ongoing Consume call -func (c *TableFlowController) Abort() { +func (c *KeySpanFlowController) Abort() { c.memoryQuota.Abort() } // GetConsumption returns the current memory consumption -func (c *TableFlowController) GetConsumption() uint64 { +func (c *KeySpanFlowController) GetConsumption() uint64 { return c.memoryQuota.GetConsumption() } diff --git a/cdc/cdc/sink/common/flow_control_test.go b/cdc/cdc/sink/common/flow_control_test.go index e742bd32..a810b5d2 100644 --- a/cdc/cdc/sink/common/flow_control_test.go +++ b/cdc/cdc/sink/common/flow_control_test.go @@ -47,7 +47,7 @@ func (c *mockCallBacker) cb() error { func (s *flowControlSuite) TestMemoryQuotaBasic(c *check.C) { defer testleak.AfterTest(c)() - controller := NewTableMemoryQuota(1024) + controller := NewKeySpanMemoryQuota(1024) sizeCh := make(chan uint64, 1024) var ( wg sync.WaitGroup @@ -89,7 +89,7 @@ func (s *flowControlSuite) TestMemoryQuotaBasic(c *check.C) { func (s *flowControlSuite) TestMemoryQuotaForceConsume(c *check.C) { defer testleak.AfterTest(c)() - controller := NewTableMemoryQuota(1024) + controller := NewKeySpanMemoryQuota(1024) sizeCh := make(chan uint64, 1024) var ( wg sync.WaitGroup @@ -137,7 +137,7 @@ func (s *flowControlSuite) TestMemoryQuotaForceConsume(c *check.C) { func (s *flowControlSuite) TestMemoryQuotaAbort(c *check.C) { defer testleak.AfterTest(c)() - controller := NewTableMemoryQuota(1024) + controller := NewKeySpanMemoryQuota(1024) var wg sync.WaitGroup wg.Add(1) go func() { @@ -162,7 +162,7 @@ func (s *flowControlSuite) TestMemoryQuotaAbort(c *check.C) { func (s *flowControlSuite) TestMemoryQuotaReleaseZero(c *check.C) { defer testleak.AfterTest(c)() - controller := NewTableMemoryQuota(1024) + controller := NewKeySpanMemoryQuota(1024) controller.Release(0) } @@ -178,7 +178,7 @@ func (s *flowControlSuite) TestFlowControlBasic(c *check.C) { defer cancel() errg, ctx := errgroup.WithContext(ctx) mockedRowsCh := make(chan *commitTsSizeEntry, 1024) - flowController := NewTableFlowController(2048) + flowController := NewKeySpanFlowController(2048) errg.Go(func() error { lastCommitTs := uint64(1) @@ -288,7 +288,7 @@ func (s *flowControlSuite) TestFlowControlAbort(c *check.C) { defer testleak.AfterTest(c)() callBacker := &mockCallBacker{} - controller := NewTableFlowController(1024) + controller := NewKeySpanFlowController(1024) var wg sync.WaitGroup wg.Add(1) go func() { @@ -318,7 +318,7 @@ func (s *flowControlSuite) TestFlowControlCallBack(c *check.C) { defer cancel() errg, ctx := errgroup.WithContext(ctx) mockedRowsCh := make(chan *commitTsSizeEntry, 1024) - flowController := NewTableFlowController(512) + flowController := NewKeySpanFlowController(512) errg.Go(func() error { lastCommitTs := uint64(1) @@ -421,7 +421,7 @@ func (s *flowControlSuite) TestFlowControlCallBackNotBlockingRelease(c *check.C) defer testleak.AfterTest(c)() var wg sync.WaitGroup - controller := NewTableFlowController(512) + controller := NewKeySpanFlowController(512) wg.Add(1) ctx, cancel := context.WithCancel(context.TODO()) @@ -463,7 +463,7 @@ func (s *flowControlSuite) TestFlowControlCallBackError(c *check.C) { defer testleak.AfterTest(c)() var wg sync.WaitGroup - controller := NewTableFlowController(512) + controller := NewKeySpanFlowController(512) wg.Add(1) ctx, cancel := context.WithCancel(context.TODO()) @@ -492,7 +492,7 @@ func (s *flowControlSuite) TestFlowControlCallBackError(c *check.C) { func (s *flowControlSuite) TestFlowControlConsumeLargerThanQuota(c *check.C) { defer testleak.AfterTest(c)() - controller := NewTableFlowController(1024) + controller := NewKeySpanFlowController(1024) err := controller.Consume(1, 2048, func() error { c.Fatalf("unreachable") return nil @@ -500,12 +500,12 @@ func (s *flowControlSuite) TestFlowControlConsumeLargerThanQuota(c *check.C) { c.Assert(err, check.ErrorMatches, ".*ErrFlowControllerEventLargerThanQuota.*") } -func BenchmarkTableFlowController(B *testing.B) { +func BenchmarkKeySpanFlowController(B *testing.B) { ctx, cancel := context.WithTimeout(context.TODO(), time.Second*5) defer cancel() errg, ctx := errgroup.WithContext(ctx) mockedRowsCh := make(chan *commitTsSizeEntry, 102400) - flowController := NewTableFlowController(20 * 1024 * 1024) // 20M + flowController := NewKeySpanFlowController(20 * 1024 * 1024) // 20M errg.Go(func() error { lastCommitTs := uint64(1) diff --git a/cdc/cdc/sink/dispatcher/default.go b/cdc/cdc/sink/dispatcher/default.go deleted file mode 100644 index 0c7715db..00000000 --- a/cdc/cdc/sink/dispatcher/default.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package dispatcher - -import ( - "github.com/tikv/migration/cdc/cdc/model" -) - -type defaultDispatcher struct { - partitionNum int32 - tbd *tableDispatcher - ivd *indexValueDispatcher - enableOldValue bool -} - -func newDefaultDispatcher(partitionNum int32, enableOldValue bool) *defaultDispatcher { - return &defaultDispatcher{ - partitionNum: partitionNum, - tbd: newTableDispatcher(partitionNum), - ivd: newIndexValueDispatcher(partitionNum), - enableOldValue: enableOldValue, - } -} - -func (d *defaultDispatcher) Dispatch(row *model.RowChangedEvent) int32 { - if d.enableOldValue { - return d.tbd.Dispatch(row) - } - if len(row.IndexColumns) != 1 { - return d.tbd.Dispatch(row) - } - return d.ivd.Dispatch(row) -} diff --git a/cdc/cdc/sink/dispatcher/default_test.go b/cdc/cdc/sink/dispatcher/default_test.go deleted file mode 100644 index 347e1c63..00000000 --- a/cdc/cdc/sink/dispatcher/default_test.go +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package dispatcher - -import ( - "github.com/pingcap/check" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/util/testleak" -) - -type DefaultDispatcherSuite struct{} - -var _ = check.Suite(&DefaultDispatcherSuite{}) - -func (s DefaultDispatcherSuite) TestDefaultDispatcher(c *check.C) { - defer testleak.AfterTest(c)() - testCases := []struct { - row *model.RowChangedEvent - exceptPartition int32 - }{ - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t1", - }, - Columns: []*model.Column{ - { - Name: "id", - Value: 1, - Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, - }, - }, - IndexColumns: [][]int{{0}}, - }, exceptPartition: 11}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t1", - }, - Columns: []*model.Column{ - { - Name: "id", - Value: 2, - Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, - }, - }, - IndexColumns: [][]int{{0}}, - }, exceptPartition: 1}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t1", - }, - Columns: []*model.Column{ - { - Name: "id", - Value: 3, - Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, - }, - }, - IndexColumns: [][]int{{0}}, - }, exceptPartition: 7}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t2", - }, - Columns: []*model.Column{ - { - Name: "id", - Value: 1, - Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, - }, { - Name: "a", - Value: 1, - }, - }, - IndexColumns: [][]int{{0}}, - }, exceptPartition: 1}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t2", - }, - Columns: []*model.Column{ - { - Name: "id", - Value: 2, - Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, - }, { - Name: "a", - Value: 2, - }, - }, - IndexColumns: [][]int{{0}}, - }, exceptPartition: 11}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t2", - }, - Columns: []*model.Column{ - { - Name: "id", - Value: 3, - Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, - }, { - Name: "a", - Value: 3, - }, - }, - IndexColumns: [][]int{{0}}, - }, exceptPartition: 13}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t2", - }, - Columns: []*model.Column{ - { - Name: "id", - Value: 3, - Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, - }, { - Name: "a", - Value: 4, - }, - }, - IndexColumns: [][]int{{0}}, - }, exceptPartition: 13}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t3", - }, - Columns: []*model.Column{ - { - Name: "id", - Value: 1, - Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, - }, - { - Name: "a", - Value: 2, - Flag: model.UniqueKeyFlag, - }, - }, - IndexColumns: [][]int{{0}, {1}}, - }, exceptPartition: 3}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t3", - }, - Columns: []*model.Column{ - { - Name: "id", - Value: 2, - Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, - }, { - Name: "a", - Value: 3, - Flag: model.UniqueKeyFlag, - }, - }, - IndexColumns: [][]int{{0}, {1}}, - }, exceptPartition: 3}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t3", - }, - Columns: []*model.Column{ - { - Name: "id", - Value: 3, - Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, - }, { - Name: "a", - Value: 4, - Flag: model.UniqueKeyFlag, - }, - }, - IndexColumns: [][]int{{0}, {1}}, - }, exceptPartition: 3}, - } - p := newDefaultDispatcher(16, false) - for _, tc := range testCases { - c.Assert(p.Dispatch(tc.row), check.Equals, tc.exceptPartition) - } -} diff --git a/cdc/cdc/sink/dispatcher/dispatcher.go b/cdc/cdc/sink/dispatcher/dispatcher.go deleted file mode 100644 index 2b53397e..00000000 --- a/cdc/cdc/sink/dispatcher/dispatcher.go +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package dispatcher - -import ( - "strings" - - "github.com/pingcap/log" - filter "github.com/pingcap/tidb-tools/pkg/table-filter" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/config" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "go.uber.org/zap" -) - -// Dispatcher is an abstraction for dispatching rows into different partitions -type Dispatcher interface { - // Dispatch returns an index of partitions according to RowChangedEvent - Dispatch(row *model.RowChangedEvent) int32 -} - -type dispatchRule int - -const ( - dispatchRuleDefault dispatchRule = iota - dispatchRuleRowID - dispatchRuleTS - dispatchRuleTable - dispatchRuleIndexValue -) - -func (r *dispatchRule) fromString(rule string) { - switch strings.ToLower(rule) { - case "default": - *r = dispatchRuleDefault - case "rowid": - *r = dispatchRuleRowID - case "ts": - *r = dispatchRuleTS - case "table": - *r = dispatchRuleTable - case "index-value": - *r = dispatchRuleIndexValue - default: - *r = dispatchRuleDefault - log.Warn("can't support dispatch rule, using default rule", zap.String("rule", rule)) - } -} - -type dispatcherSwitcher struct { - rules []struct { - Dispatcher - filter.Filter - } -} - -func (s *dispatcherSwitcher) Dispatch(row *model.RowChangedEvent) int32 { - return s.matchDispatcher(row).Dispatch(row) -} - -func (s *dispatcherSwitcher) matchDispatcher(row *model.RowChangedEvent) Dispatcher { - for _, rule := range s.rules { - if !rule.MatchTable(row.Table.Schema, row.Table.Table) { - continue - } - return rule.Dispatcher - } - log.Panic("the dispatch rule must cover all tables") - return nil -} - -// NewDispatcher creates a new dispatcher -func NewDispatcher(cfg *config.ReplicaConfig, partitionNum int32) (Dispatcher, error) { - ruleConfigs := append(cfg.Sink.DispatchRules, &config.DispatchRule{ - Matcher: []string{"*.*"}, - Dispatcher: "default", - }) - rules := make([]struct { - Dispatcher - filter.Filter - }, 0, len(ruleConfigs)) - - for _, ruleConfig := range ruleConfigs { - f, err := filter.Parse(ruleConfig.Matcher) - if err != nil { - return nil, cerror.WrapError(cerror.ErrFilterRuleInvalid, err) - } - if !cfg.CaseSensitive { - f = filter.CaseInsensitive(f) - } - var d Dispatcher - var rule dispatchRule - rule.fromString(ruleConfig.Dispatcher) - switch rule { - case dispatchRuleRowID, dispatchRuleIndexValue: - if cfg.EnableOldValue { - log.Warn("This index-value distribution mode " + - "does not guarantee row-level orderliness when " + - "switching on the old value, so please use caution!") - } - d = newIndexValueDispatcher(partitionNum) - case dispatchRuleTS: - d = newTsDispatcher(partitionNum) - case dispatchRuleTable: - d = newTableDispatcher(partitionNum) - case dispatchRuleDefault: - d = newDefaultDispatcher(partitionNum, cfg.EnableOldValue) - } - rules = append(rules, struct { - Dispatcher - filter.Filter - }{Dispatcher: d, Filter: f}) - } - return &dispatcherSwitcher{ - rules: rules, - }, nil -} diff --git a/cdc/cdc/sink/dispatcher/index_value.go b/cdc/cdc/sink/dispatcher/index_value.go deleted file mode 100644 index f3cb5a51..00000000 --- a/cdc/cdc/sink/dispatcher/index_value.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package dispatcher - -import ( - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/hash" -) - -type indexValueDispatcher struct { - partitionNum int32 - hasher *hash.PositionInertia -} - -func newIndexValueDispatcher(partitionNum int32) *indexValueDispatcher { - return &indexValueDispatcher{ - partitionNum: partitionNum, - hasher: hash.NewPositionInertia(), - } -} - -func (r *indexValueDispatcher) Dispatch(row *model.RowChangedEvent) int32 { - r.hasher.Reset() - r.hasher.Write([]byte(row.Table.Schema), []byte(row.Table.Table)) - // FIXME(leoppro): if the row events includes both pre-cols and cols - // the dispatch logic here is wrong - - // distribute partition by rowid or unique column value - dispatchCols := row.Columns - if len(row.Columns) == 0 { - dispatchCols = row.PreColumns - } - for _, col := range dispatchCols { - if col == nil { - continue - } - if col.Flag.IsHandleKey() { - r.hasher.Write([]byte(col.Name), []byte(model.ColumnValueString(col.Value))) - } - } - return int32(r.hasher.Sum32() % uint32(r.partitionNum)) -} diff --git a/cdc/cdc/sink/dispatcher/index_value_test.go b/cdc/cdc/sink/dispatcher/index_value_test.go deleted file mode 100644 index 1a93a580..00000000 --- a/cdc/cdc/sink/dispatcher/index_value_test.go +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package dispatcher - -import ( - "github.com/pingcap/check" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/util/testleak" -) - -type IndexValueDispatcherSuite struct{} - -var _ = check.Suite(&IndexValueDispatcherSuite{}) - -func (s IndexValueDispatcherSuite) TestIndexValueDispatcher(c *check.C) { - defer testleak.AfterTest(c)() - testCases := []struct { - row *model.RowChangedEvent - exceptPartition int32 - }{ - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t1", - }, - Columns: []*model.Column{ - { - Name: "a", - Value: 11, - Flag: model.HandleKeyFlag, - }, { - Name: "b", - Value: 22, - Flag: 0, - }, - }, - }, exceptPartition: 2}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t1", - }, - Columns: []*model.Column{ - { - Name: "a", - Value: 22, - Flag: model.HandleKeyFlag, - }, { - Name: "b", - Value: 22, - Flag: 0, - }, - }, - }, exceptPartition: 11}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t1", - }, - Columns: []*model.Column{ - { - Name: "a", - Value: 11, - Flag: model.HandleKeyFlag, - }, { - Name: "b", - Value: 33, - Flag: 0, - }, - }, - }, exceptPartition: 2}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t2", - }, - Columns: []*model.Column{ - { - Name: "a", - Value: 11, - Flag: model.HandleKeyFlag, - }, { - Name: "b", - Value: 22, - Flag: model.HandleKeyFlag, - }, - }, - }, exceptPartition: 5}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t2", - }, - Columns: []*model.Column{ - { - Name: "b", - Value: 22, - Flag: model.HandleKeyFlag, - }, { - Name: "a", - Value: 11, - Flag: model.HandleKeyFlag, - }, - }, - }, exceptPartition: 5}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t2", - }, - Columns: []*model.Column{ - { - Name: "a", - Value: 11, - Flag: model.HandleKeyFlag, - }, { - Name: "b", - Value: 0, - Flag: model.HandleKeyFlag, - }, - }, - }, exceptPartition: 14}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t2", - }, - Columns: []*model.Column{ - { - Name: "a", - Value: 11, - Flag: model.HandleKeyFlag, - }, { - Name: "b", - Value: 33, - Flag: model.HandleKeyFlag, - }, - }, - }, exceptPartition: 2}, - } - p := newIndexValueDispatcher(16) - for _, tc := range testCases { - c.Assert(p.Dispatch(tc.row), check.Equals, tc.exceptPartition) - } -} diff --git a/cdc/cdc/sink/dispatcher/switcher_test.go b/cdc/cdc/sink/dispatcher/switcher_test.go deleted file mode 100644 index e38f7f22..00000000 --- a/cdc/cdc/sink/dispatcher/switcher_test.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package dispatcher - -import ( - "github.com/pingcap/check" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/util/testleak" -) - -type SwitcherSuite struct{} - -var _ = check.Suite(&SwitcherSuite{}) - -func (s SwitcherSuite) TestSwitcher(c *check.C) { - defer testleak.AfterTest(c)() - d, err := NewDispatcher(config.GetDefaultReplicaConfig(), 4) - c.Assert(err, check.IsNil) - c.Assert(d.(*dispatcherSwitcher).matchDispatcher(&model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", Table: "test", - }, - }), check.FitsTypeOf, &defaultDispatcher{}) - - d, err = NewDispatcher(&config.ReplicaConfig{ - Sink: &config.SinkConfig{ - DispatchRules: []*config.DispatchRule{ - {Matcher: []string{"test_default.*"}, Dispatcher: "default"}, - {Matcher: []string{"test_table.*"}, Dispatcher: "table"}, - {Matcher: []string{"test_index_value.*"}, Dispatcher: "index-value"}, - {Matcher: []string{"test.*"}, Dispatcher: "rowid"}, - {Matcher: []string{"*.*", "!*.test"}, Dispatcher: "ts"}, - }, - }, - }, 4) - c.Assert(err, check.IsNil) - c.Assert(d.(*dispatcherSwitcher).matchDispatcher(&model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", Table: "table1", - }, - }), check.FitsTypeOf, &indexValueDispatcher{}) - c.Assert(d.(*dispatcherSwitcher).matchDispatcher(&model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "sbs", Table: "table2", - }, - }), check.FitsTypeOf, &tsDispatcher{}) - c.Assert(d.(*dispatcherSwitcher).matchDispatcher(&model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "sbs", Table: "test", - }, - }), check.FitsTypeOf, &defaultDispatcher{}) - c.Assert(d.(*dispatcherSwitcher).matchDispatcher(&model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test_default", Table: "test", - }, - }), check.FitsTypeOf, &defaultDispatcher{}) - c.Assert(d.(*dispatcherSwitcher).matchDispatcher(&model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test_table", Table: "test", - }, - }), check.FitsTypeOf, &tableDispatcher{}) - c.Assert(d.(*dispatcherSwitcher).matchDispatcher(&model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test_index_value", Table: "test", - }, - }), check.FitsTypeOf, &indexValueDispatcher{}) -} diff --git a/cdc/cdc/sink/dispatcher/table.go b/cdc/cdc/sink/dispatcher/table.go deleted file mode 100644 index 8385ddc5..00000000 --- a/cdc/cdc/sink/dispatcher/table.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package dispatcher - -import ( - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/hash" -) - -type tableDispatcher struct { - partitionNum int32 - hasher *hash.PositionInertia -} - -func newTableDispatcher(partitionNum int32) *tableDispatcher { - return &tableDispatcher{ - partitionNum: partitionNum, - hasher: hash.NewPositionInertia(), - } -} - -func (t *tableDispatcher) Dispatch(row *model.RowChangedEvent) int32 { - t.hasher.Reset() - // distribute partition by table - t.hasher.Write([]byte(row.Table.Schema), []byte(row.Table.Table)) - return int32(t.hasher.Sum32() % uint32(t.partitionNum)) -} diff --git a/cdc/cdc/sink/dispatcher/table_test.go b/cdc/cdc/sink/dispatcher/table_test.go deleted file mode 100644 index 7d74b3c9..00000000 --- a/cdc/cdc/sink/dispatcher/table_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package dispatcher - -import ( - "github.com/pingcap/check" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/util/testleak" -) - -type TableDispatcherSuite struct{} - -var _ = check.Suite(&TableDispatcherSuite{}) - -func (s TableDispatcherSuite) TestTableDispatcher(c *check.C) { - defer testleak.AfterTest(c)() - testCases := []struct { - row *model.RowChangedEvent - exceptPartition int32 - }{ - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t1", - }, - CommitTs: 1, - }, exceptPartition: 15}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t1", - }, - CommitTs: 2, - }, exceptPartition: 15}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t1", - }, - CommitTs: 3, - }, exceptPartition: 15}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t2", - }, - CommitTs: 1, - }, exceptPartition: 5}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t2", - }, - CommitTs: 2, - }, exceptPartition: 5}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t2", - }, - CommitTs: 3, - }, exceptPartition: 5}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t3", - }, - CommitTs: 3, - }, exceptPartition: 3}, - } - p := newTableDispatcher(16) - for _, tc := range testCases { - c.Assert(p.Dispatch(tc.row), check.Equals, tc.exceptPartition) - } -} diff --git a/cdc/cdc/sink/dispatcher/ts.go b/cdc/cdc/sink/dispatcher/ts.go deleted file mode 100644 index 171b8bbd..00000000 --- a/cdc/cdc/sink/dispatcher/ts.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package dispatcher - -import "github.com/tikv/migration/cdc/cdc/model" - -type tsDispatcher struct { - partitionNum int32 -} - -func newTsDispatcher(partitionNum int32) *tsDispatcher { - return &tsDispatcher{ - partitionNum: partitionNum, - } -} - -func (t *tsDispatcher) Dispatch(row *model.RowChangedEvent) int32 { - return int32(row.CommitTs % uint64(t.partitionNum)) -} diff --git a/cdc/cdc/sink/dispatcher/ts_test.go b/cdc/cdc/sink/dispatcher/ts_test.go deleted file mode 100644 index f9bee2d6..00000000 --- a/cdc/cdc/sink/dispatcher/ts_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package dispatcher - -import ( - "testing" - - "github.com/pingcap/check" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/util/testleak" -) - -func Test(t *testing.T) { check.TestingT(t) } - -type TsDispatcherSuite struct{} - -var _ = check.Suite(&TsDispatcherSuite{}) - -func (s TsDispatcherSuite) TestTsDispatcher(c *check.C) { - defer testleak.AfterTest(c)() - testCases := []struct { - row *model.RowChangedEvent - exceptPartition int32 - }{ - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t1", - }, - CommitTs: 1, - }, exceptPartition: 1}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t1", - }, - CommitTs: 2, - }, exceptPartition: 2}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t1", - }, - CommitTs: 3, - }, exceptPartition: 3}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t2", - }, - CommitTs: 1, - }, exceptPartition: 1}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t2", - }, - CommitTs: 2, - }, exceptPartition: 2}, - {row: &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t2", - }, - CommitTs: 3, - }, exceptPartition: 3}, - } - p := &tsDispatcher{partitionNum: 16} - for _, tc := range testCases { - c.Assert(p.Dispatch(tc.row), check.Equals, tc.exceptPartition) - } -} diff --git a/cdc/cdc/sink/keyspan_sink.go b/cdc/cdc/sink/keyspan_sink.go new file mode 100644 index 00000000..34d29555 --- /dev/null +++ b/cdc/cdc/sink/keyspan_sink.go @@ -0,0 +1,74 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package sink + +import ( + "context" + + "github.com/pingcap/errors" + "github.com/pingcap/log" + "github.com/tikv/migration/cdc/cdc/model" + "go.uber.org/zap" +) + +type keyspanSink struct { + keyspanID model.KeySpanID + manager *Manager + buffer []*model.RawKVEntry +} + +var _ Sink = (*keyspanSink)(nil) + +func (t *keyspanSink) EmitChangedEvents(ctx context.Context, rawKVEntries ...*model.RawKVEntry) error { + t.buffer = append(t.buffer, rawKVEntries...) + t.manager.metricsKeySpanSinkTotalEvents.Add(float64(len(rawKVEntries))) + return nil +} + +// FlushRowChangedEvents flushes sorted rows to sink manager, note the resolvedTs +// is required to be no more than global resolvedTs, keyspan barrierTs and keyspan +// redo log watermarkTs. +func (t *keyspanSink) FlushChangedEvents(ctx context.Context, keyspanID model.KeySpanID, resolvedTs uint64) (uint64, error) { + if keyspanID != t.keyspanID { + log.Panic("inconsistent keyspan sink", + zap.Uint64("keyspanID", keyspanID), zap.Uint64("sinkKeySpanID", t.keyspanID)) + } + resolvedRawKVEntries := t.buffer + t.buffer = []*model.RawKVEntry{} + + err := t.manager.bufSink.EmitChangedEvents(ctx, resolvedRawKVEntries...) + if err != nil { + return t.manager.getCheckpointTs(keyspanID), errors.Trace(err) + } + return t.flushResolvedTs(ctx, resolvedTs) +} + +func (t *keyspanSink) flushResolvedTs(ctx context.Context, resolvedTs uint64) (uint64, error) { + return t.manager.flushBackendSink(ctx, t.keyspanID, resolvedTs) +} + +func (t *keyspanSink) EmitCheckpointTs(ctx context.Context, ts uint64) error { + // the keyspan sink doesn't receive the checkpoint event + return nil +} + +// Close once the method is called, no more events can be written to this keyspan sink +func (t *keyspanSink) Close(ctx context.Context) error { + return t.manager.destroyKeySpanSink(ctx, t.keyspanID) +} + +// Barrier is not used in keyspan sink +func (t *keyspanSink) Barrier(ctx context.Context, keyspanID model.KeySpanID) error { + return nil +} diff --git a/cdc/cdc/sink/manager.go b/cdc/cdc/sink/manager.go index d671f5a0..1758f204 100644 --- a/cdc/cdc/sink/manager.go +++ b/cdc/cdc/sink/manager.go @@ -22,25 +22,24 @@ import ( "github.com/pingcap/log" "github.com/prometheus/client_golang/prometheus" "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/redo" "go.uber.org/zap" ) -// Manager manages table sinks, maintains the relationship between table sinks +// Manager manages keyspan sinks, maintains the relationship between keyspan sinks // and backendSink. // Manager is thread-safe. type Manager struct { bufSink *bufferSink - tableCheckpointTsMap sync.Map - tableSinks map[model.TableID]*tableSink - tableSinksMu sync.Mutex + keyspanCheckpointTsMap sync.Map + keyspanSinks map[model.KeySpanID]*keyspanSink + keyspanSinksMu sync.Mutex changeFeedCheckpointTs uint64 drawbackChan chan drawbackMsg - captureAddr string - changefeedID model.ChangeFeedID - metricsTableSinkTotalRows prometheus.Counter + captureAddr string + changefeedID model.ChangeFeedID + metricsKeySpanSinkTotalEvents prometheus.Counter } // NewManager creates a new Sink manager @@ -52,77 +51,76 @@ func NewManager( bufSink := newBufferSink(backendSink, checkpointTs, drawbackChan) go bufSink.run(ctx, errCh) return &Manager{ - bufSink: bufSink, - changeFeedCheckpointTs: checkpointTs, - tableSinks: make(map[model.TableID]*tableSink), - drawbackChan: drawbackChan, - captureAddr: captureAddr, - changefeedID: changefeedID, - metricsTableSinkTotalRows: tableSinkTotalRowsCountCounter.WithLabelValues(captureAddr, changefeedID), + bufSink: bufSink, + changeFeedCheckpointTs: checkpointTs, + keyspanSinks: make(map[model.KeySpanID]*keyspanSink), + drawbackChan: drawbackChan, + captureAddr: captureAddr, + changefeedID: changefeedID, + metricsKeySpanSinkTotalEvents: keyspanSinkTotalEventsCountCounter.WithLabelValues(captureAddr, changefeedID), } } -// CreateTableSink creates a table sink -func (m *Manager) CreateTableSink(tableID model.TableID, checkpointTs model.Ts, redoManager redo.LogManager) Sink { - m.tableSinksMu.Lock() - defer m.tableSinksMu.Unlock() - if _, exist := m.tableSinks[tableID]; exist { - log.Panic("the table sink already exists", zap.Uint64("tableID", uint64(tableID))) +// CreateKeySpanSink creates a keyspan sink +func (m *Manager) CreateKeySpanSink(keyspanID model.KeySpanID, checkpointTs model.Ts) Sink { + m.keyspanSinksMu.Lock() + defer m.keyspanSinksMu.Unlock() + if _, exist := m.keyspanSinks[keyspanID]; exist { + log.Panic("the keyspan sink already exists", zap.Uint64("keyspanID", keyspanID)) } - sink := &tableSink{ - tableID: tableID, - manager: m, - buffer: make([]*model.RowChangedEvent, 0, 128), - redoManager: redoManager, + sink := &keyspanSink{ + keyspanID: keyspanID, + manager: m, + buffer: make([]*model.RawKVEntry, 0, 128), } - m.tableSinks[tableID] = sink + m.keyspanSinks[keyspanID] = sink return sink } // Close closes the Sink manager and backend Sink, this method can be reentrantly called func (m *Manager) Close(ctx context.Context) error { - m.tableSinksMu.Lock() - defer m.tableSinksMu.Unlock() - tableSinkTotalRowsCountCounter.DeleteLabelValues(m.captureAddr, m.changefeedID) + m.keyspanSinksMu.Lock() + defer m.keyspanSinksMu.Unlock() + keyspanSinkTotalEventsCountCounter.DeleteLabelValues(m.captureAddr, m.changefeedID) if m.bufSink != nil { return m.bufSink.Close(ctx) } return nil } -func (m *Manager) flushBackendSink(ctx context.Context, tableID model.TableID, resolvedTs uint64) (model.Ts, error) { - checkpointTs, err := m.bufSink.FlushRowChangedEvents(ctx, tableID, resolvedTs) +func (m *Manager) flushBackendSink(ctx context.Context, keyspanID model.KeySpanID, resolvedTs uint64) (model.Ts, error) { + checkpointTs, err := m.bufSink.FlushChangedEvents(ctx, keyspanID, resolvedTs) if err != nil { - return m.getCheckpointTs(tableID), errors.Trace(err) + return m.getCheckpointTs(keyspanID), errors.Trace(err) } - m.tableCheckpointTsMap.Store(tableID, checkpointTs) + m.keyspanCheckpointTsMap.Store(keyspanID, checkpointTs) return checkpointTs, nil } -func (m *Manager) destroyTableSink(ctx context.Context, tableID model.TableID) error { - m.tableSinksMu.Lock() - delete(m.tableSinks, tableID) - m.tableSinksMu.Unlock() +func (m *Manager) destroyKeySpanSink(ctx context.Context, keyspanID model.KeySpanID) error { + m.keyspanSinksMu.Lock() + delete(m.keyspanSinks, keyspanID) + m.keyspanSinksMu.Unlock() callback := make(chan struct{}) select { case <-ctx.Done(): return ctx.Err() - case m.drawbackChan <- drawbackMsg{tableID: tableID, callback: callback}: + case m.drawbackChan <- drawbackMsg{keyspanID: keyspanID, callback: callback}: } select { case <-ctx.Done(): return ctx.Err() case <-callback: } - return m.bufSink.Barrier(ctx, tableID) + return m.bufSink.Barrier(ctx, keyspanID) } -func (m *Manager) getCheckpointTs(tableID model.TableID) uint64 { - checkPoints, ok := m.tableCheckpointTsMap.Load(tableID) +func (m *Manager) getCheckpointTs(keyspanID model.KeySpanID) uint64 { + checkPoints, ok := m.keyspanCheckpointTsMap.Load(keyspanID) if ok { return checkPoints.(uint64) } - // cannot find table level checkpointTs because of no table level resolvedTs flush task finished successfully, + // cannot find keyspan level checkpointTs because of no keyspan level resolvedTs flush task finished successfully, // for example: first time to flush resolvedTs but cannot get the flush lock, return changefeed level checkpointTs is safe return atomic.LoadUint64(&m.changeFeedCheckpointTs) } @@ -135,6 +133,6 @@ func (m *Manager) UpdateChangeFeedCheckpointTs(checkpointTs uint64) { } type drawbackMsg struct { - tableID model.TableID - callback chan struct{} + keyspanID model.KeySpanID + callback chan struct{} } diff --git a/cdc/cdc/sink/manager_test.go b/cdc/cdc/sink/manager_test.go index 194a00b1..1774a9b4 100644 --- a/cdc/cdc/sink/manager_test.go +++ b/cdc/cdc/sink/manager_test.go @@ -25,7 +25,6 @@ import ( "github.com/pingcap/check" "github.com/pingcap/errors" "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/redo" "github.com/tikv/migration/cdc/pkg/util/testleak" ) @@ -35,51 +34,40 @@ var _ = check.Suite(&managerSuite{}) type checkSink struct { *check.C - rows map[model.TableID][]*model.RowChangedEvent + entries map[model.KeySpanID][]*model.RawKVEntry rowsMu sync.Mutex - lastResolvedTs map[model.TableID]uint64 + lastResolvedTs map[model.KeySpanID]uint64 } func newCheckSink(c *check.C) *checkSink { return &checkSink{ C: c, - rows: make(map[model.TableID][]*model.RowChangedEvent), - lastResolvedTs: make(map[model.TableID]uint64), + entries: make(map[model.KeySpanID][]*model.RawKVEntry), + lastResolvedTs: make(map[model.KeySpanID]uint64), } } -func (c *checkSink) EmitRowChangedEvents(ctx context.Context, rows ...*model.RowChangedEvent) error { +func (c *checkSink) EmitChangedEvents(ctx context.Context, entries ...*model.RawKVEntry) error { c.rowsMu.Lock() defer c.rowsMu.Unlock() - for _, row := range rows { - c.rows[row.Table.TableID] = append(c.rows[row.Table.TableID], row) + for _, entry := range entries { + c.entries[entry.KeySpanID] = append(c.entries[entry.KeySpanID], entry) } return nil } -func (c *checkSink) EmitDDLEvent(ctx context.Context, ddl *model.DDLEvent) error { - panic("unreachable") -} - -func (c *checkSink) FlushRowChangedEvents(ctx context.Context, tableID model.TableID, resolvedTs uint64) (uint64, error) { +func (c *checkSink) FlushChangedEvents(ctx context.Context, keyspanID model.KeySpanID, resolvedTs uint64) (uint64, error) { c.rowsMu.Lock() defer c.rowsMu.Unlock() - var newRows []*model.RowChangedEvent - rows := c.rows[tableID] - for _, row := range rows { - if row.CommitTs <= c.lastResolvedTs[tableID] { - return c.lastResolvedTs[tableID], errors.Errorf("commit-ts(%d) is not greater than lastResolvedTs(%d)", row.CommitTs, c.lastResolvedTs) - } - if row.CommitTs > resolvedTs { - newRows = append(newRows, row) - } - } + var newEntries []*model.RawKVEntry + entries := c.entries[keyspanID] + newEntries = append(newEntries, entries...) - c.Assert(c.lastResolvedTs[tableID], check.LessEqual, resolvedTs) - c.lastResolvedTs[tableID] = resolvedTs - c.rows[tableID] = newRows + c.Assert(c.lastResolvedTs[keyspanID], check.LessEqual, resolvedTs) + c.lastResolvedTs[keyspanID] = resolvedTs + c.entries[keyspanID] = newEntries - return c.lastResolvedTs[tableID], nil + return c.lastResolvedTs[keyspanID], nil } func (c *checkSink) EmitCheckpointTs(ctx context.Context, ts uint64) error { @@ -90,7 +78,7 @@ func (c *checkSink) Close(ctx context.Context) error { return nil } -func (c *checkSink) Barrier(ctx context.Context, tableID model.TableID) error { +func (c *checkSink) Barrier(ctx context.Context, keyspanID model.KeySpanID) error { return nil } @@ -104,19 +92,19 @@ func (s *managerSuite) TestManagerRandom(c *check.C) { goroutineNum := 10 rowNum := 100 var wg sync.WaitGroup - tableSinks := make([]Sink, goroutineNum) + keyspanSinks := make([]Sink, goroutineNum) for i := 0; i < goroutineNum; i++ { i := i wg.Add(1) go func() { defer wg.Done() - tableSinks[i] = manager.CreateTableSink(model.TableID(i), 0, redo.NewDisabledManager()) + keyspanSinks[i] = manager.CreateKeySpanSink(model.KeySpanID(i), 0) }() } wg.Wait() for i := 0; i < goroutineNum; i++ { i := i - tableSink := tableSinks[i] + keyspanSink := keyspanSinks[i] wg.Add(1) go func() { defer wg.Done() @@ -125,18 +113,17 @@ func (s *managerSuite) TestManagerRandom(c *check.C) { for j := 1; j < rowNum; j++ { if rand.Intn(10) == 0 { resolvedTs := lastResolvedTs + uint64(rand.Intn(j-int(lastResolvedTs))) - _, err := tableSink.FlushRowChangedEvents(ctx, model.TableID(i), resolvedTs) + _, err := keyspanSink.FlushChangedEvents(ctx, model.KeySpanID(i), resolvedTs) c.Assert(err, check.IsNil) lastResolvedTs = resolvedTs } else { - err := tableSink.EmitRowChangedEvents(ctx, &model.RowChangedEvent{ - Table: &model.TableName{TableID: int64(i)}, - CommitTs: uint64(j), + err := keyspanSink.EmitChangedEvents(ctx, &model.RawKVEntry{ + KeySpanID: uint64(i), }) c.Assert(err, check.IsNil) } } - _, err := tableSink.FlushRowChangedEvents(ctx, model.TableID(i), uint64(rowNum)) + _, err := keyspanSink.FlushChangedEvents(ctx, model.KeySpanID(i), uint64(rowNum)) c.Assert(err, check.IsNil) }() } @@ -149,7 +136,7 @@ func (s *managerSuite) TestManagerRandom(c *check.C) { } } -func (s *managerSuite) TestManagerAddRemoveTable(c *check.C) { +func (s *managerSuite) TestManagerAddRemoveKeySpan(c *check.C) { defer testleak.AfterTest(c)() ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -161,9 +148,9 @@ func (s *managerSuite) TestManagerAddRemoveTable(c *check.C) { const ExitSignal = uint64(math.MaxUint64) var maxResolvedTs uint64 - tableSinks := make([]Sink, 0, goroutineNum) - tableCancels := make([]context.CancelFunc, 0, goroutineNum) - runTableSink := func(ctx context.Context, index int64, sink Sink, startTs uint64) { + keyspanSinks := make([]Sink, 0, goroutineNum) + keyspanCancels := make([]context.CancelFunc, 0, goroutineNum) + runKeySpanSink := func(ctx context.Context, index uint64, sink Sink, startTs uint64) { defer wg.Done() lastResolvedTs := startTs for { @@ -181,13 +168,12 @@ func (s *managerSuite) TestManagerAddRemoveTable(c *check.C) { continue } for i := lastResolvedTs + 1; i <= resolvedTs; i++ { - err := sink.EmitRowChangedEvents(ctx, &model.RowChangedEvent{ - Table: &model.TableName{TableID: index}, - CommitTs: i, + err := sink.EmitChangedEvents(ctx, &model.RawKVEntry{ + KeySpanID: index, }) c.Assert(err, check.IsNil) } - _, err := sink.FlushRowChangedEvents(ctx, sink.(*tableSink).tableID, resolvedTs) + _, err := sink.FlushChangedEvents(ctx, sink.(*keyspanSink).keyspanID, resolvedTs) if err != nil { c.Assert(errors.Cause(err), check.Equals, context.Canceled) } @@ -195,31 +181,30 @@ func (s *managerSuite) TestManagerAddRemoveTable(c *check.C) { } } - redoManager := redo.NewDisabledManager() wg.Add(1) go func() { defer wg.Done() - // add three table and then remote one table + // add three keyspan and then remote one keyspan for i := 0; i < goroutineNum; i++ { if i%4 != 3 { - // add table - table := manager.CreateTableSink(model.TableID(i), maxResolvedTs, redoManager) + // add keyspan + keyspan := manager.CreateKeySpanSink(model.KeySpanID(i), maxResolvedTs) ctx, cancel := context.WithCancel(ctx) - tableCancels = append(tableCancels, cancel) - tableSinks = append(tableSinks, table) + keyspanCancels = append(keyspanCancels, cancel) + keyspanSinks = append(keyspanSinks, keyspan) atomic.AddUint64(&maxResolvedTs, 20) wg.Add(1) - go runTableSink(ctx, int64(i), table, maxResolvedTs) + go runKeySpanSink(ctx, uint64(i), keyspan, maxResolvedTs) } else { - // remove table - table := tableSinks[0] - // note when a table is removed, no more data can be sent to the - // backend sink, so we cancel the context of this table sink. - tableCancels[0]() - c.Assert(table.Close(ctx), check.IsNil) - tableSinks = tableSinks[1:] - tableCancels = tableCancels[1:] + // remove keyspan + keyspan := keyspanSinks[0] + // note when a keyspan is removed, no more data can be sent to the + // backend sink, so we cancel the context of this keyspan sink. + keyspanCancels[0]() + c.Assert(keyspan.Close(ctx), check.IsNil) + keyspanSinks = keyspanSinks[1:] + keyspanCancels = keyspanCancels[1:] } time.Sleep(10 * time.Millisecond) } @@ -235,7 +220,7 @@ func (s *managerSuite) TestManagerAddRemoveTable(c *check.C) { } } -func (s *managerSuite) TestManagerDestroyTableSink(c *check.C) { +func (s *managerSuite) TestManagerDestroyKeySpanSink(c *check.C) { defer testleak.AfterTest(c)() ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -244,16 +229,15 @@ func (s *managerSuite) TestManagerDestroyTableSink(c *check.C) { manager := NewManager(ctx, newCheckSink(c), errCh, 0, "", "") defer manager.Close(ctx) - tableID := int64(49) - tableSink := manager.CreateTableSink(tableID, 100, redo.NewDisabledManager()) - err := tableSink.EmitRowChangedEvents(ctx, &model.RowChangedEvent{ - Table: &model.TableName{TableID: tableID}, - CommitTs: uint64(110), + keyspanID := uint64(49) + keyspanSink := manager.CreateKeySpanSink(keyspanID, 100) + err := keyspanSink.EmitChangedEvents(ctx, &model.RawKVEntry{ + KeySpanID: keyspanID, }) c.Assert(err, check.IsNil) - _, err = tableSink.FlushRowChangedEvents(ctx, tableID, 110) + _, err = keyspanSink.FlushChangedEvents(ctx, keyspanID, 110) c.Assert(err, check.IsNil) - err = manager.destroyTableSink(ctx, tableID) + err = manager.destroyKeySpanSink(ctx, keyspanID) c.Assert(err, check.IsNil) } @@ -264,17 +248,17 @@ func BenchmarkManagerFlushing(b *testing.B) { errCh := make(chan error, 16) manager := NewManager(ctx, newCheckSink(nil), errCh, 0, "", "") - // Init table sinks. + // Init keyspan sinks. goroutineNum := 2000 rowNum := 2000 var wg sync.WaitGroup - tableSinks := make([]Sink, goroutineNum) + keyspanSinks := make([]Sink, goroutineNum) for i := 0; i < goroutineNum; i++ { i := i wg.Add(1) go func() { defer wg.Done() - tableSinks[i] = manager.CreateTableSink(model.TableID(i), 0, redo.NewDisabledManager()) + keyspanSinks[i] = manager.CreateKeySpanSink(model.KeySpanID(i), 0) }() } wg.Wait() @@ -282,14 +266,13 @@ func BenchmarkManagerFlushing(b *testing.B) { // Concurrent emit events. for i := 0; i < goroutineNum; i++ { i := i - tableSink := tableSinks[i] + keyspanSink := keyspanSinks[i] wg.Add(1) go func() { defer wg.Done() for j := 1; j < rowNum; j++ { - err := tableSink.EmitRowChangedEvents(context.Background(), &model.RowChangedEvent{ - Table: &model.TableName{TableID: int64(i)}, - CommitTs: uint64(j), + err := keyspanSink.EmitChangedEvents(context.Background(), &model.RawKVEntry{ + KeySpanID: uint64(i), }) if err != nil { b.Error(err) @@ -299,14 +282,14 @@ func BenchmarkManagerFlushing(b *testing.B) { } wg.Wait() - // All tables are flushed concurrently, except table 0. + // All keyspans are flushed concurrently, except keyspan 0. for i := 1; i < goroutineNum; i++ { i := i - tblSink := tableSinks[i] + tblSink := keyspanSinks[i] go func() { for j := 1; j < rowNum; j++ { if j%2 == 0 { - _, err := tblSink.FlushRowChangedEvents(context.Background(), tblSink.(*tableSink).tableID, uint64(j)) + _, err := tblSink.FlushChangedEvents(context.Background(), tblSink.(*keyspanSink).keyspanID, uint64(j)) if err != nil { b.Error(err) } @@ -316,10 +299,10 @@ func BenchmarkManagerFlushing(b *testing.B) { } b.ResetTimer() - // Table 0 flush. - tblSink := tableSinks[0] + // KeySpan 0 flush. + tblSink := keyspanSinks[0] for i := 0; i < b.N; i++ { - _, err := tblSink.FlushRowChangedEvents(context.Background(), tblSink.(*tableSink).tableID, uint64(rowNum)) + _, err := tblSink.FlushChangedEvents(context.Background(), tblSink.(*keyspanSink).keyspanID, uint64(rowNum)) if err != nil { b.Error(err) } @@ -340,15 +323,11 @@ type errorSink struct { *check.C } -func (e *errorSink) EmitRowChangedEvents(ctx context.Context, rows ...*model.RowChangedEvent) error { +func (e *errorSink) EmitChangedEvents(ctx context.Context, rows ...*model.RawKVEntry) error { return errors.New("error in emit row changed events") } -func (e *errorSink) EmitDDLEvent(ctx context.Context, ddl *model.DDLEvent) error { - panic("unreachable") -} - -func (e *errorSink) FlushRowChangedEvents(ctx context.Context, tableID model.TableID, resolvedTs uint64) (uint64, error) { +func (e *errorSink) FlushChangedEvents(ctx context.Context, keyspanID model.KeySpanID, resolvedTs uint64) (uint64, error) { return 0, errors.New("error in flush row changed events") } @@ -360,7 +339,7 @@ func (e *errorSink) Close(ctx context.Context) error { return nil } -func (e *errorSink) Barrier(ctx context.Context, tableID model.TableID) error { +func (e *errorSink) Barrier(ctx context.Context, keyspanID model.KeySpanID) error { return nil } @@ -371,13 +350,12 @@ func (s *managerSuite) TestManagerError(c *check.C) { errCh := make(chan error, 16) manager := NewManager(ctx, &errorSink{C: c}, errCh, 0, "", "") defer manager.Close(ctx) - sink := manager.CreateTableSink(1, 0, redo.NewDisabledManager()) - err := sink.EmitRowChangedEvents(ctx, &model.RowChangedEvent{ - CommitTs: 1, - Table: &model.TableName{TableID: 1}, + sink := manager.CreateKeySpanSink(1, 0) + err := sink.EmitChangedEvents(ctx, &model.RawKVEntry{ + KeySpanID: 1, }) c.Assert(err, check.IsNil) - _, err = sink.FlushRowChangedEvents(ctx, 1, 2) + _, err = sink.FlushChangedEvents(ctx, 1, 2) c.Assert(err, check.IsNil) err = <-errCh c.Assert(err.Error(), check.Equals, "error in emit row changed events") diff --git a/cdc/cdc/sink/metrics.go b/cdc/cdc/sink/metrics.go index 6d728fa0..f41e55cf 100644 --- a/cdc/cdc/sink/metrics.go +++ b/cdc/cdc/sink/metrics.go @@ -20,7 +20,7 @@ import ( var ( execBatchHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "sink", Name: "txn_batch_size", Help: "Bucketed histogram of batch size of a txn.", @@ -28,7 +28,7 @@ var ( }, []string{"capture", "changefeed"}) execTxnHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "sink", Name: "txn_exec_duration", Help: "Bucketed histogram of processing time (s) of a txn.", @@ -36,7 +36,7 @@ var ( }, []string{"capture", "changefeed"}) execDDLHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "sink", Name: "ddl_exec_duration", Help: "Bucketed histogram of processing time (s) of a ddl.", @@ -44,14 +44,14 @@ var ( }, []string{"capture", "changefeed"}) executionErrorCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "sink", Name: "execution_error", Help: "total count of execution errors", }, []string{"capture", "changefeed"}) conflictDetectDurationHis = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "sink", Name: "conflict_detect_duration", Help: "Bucketed histogram of conflict detect time (s) for single DML statement", @@ -59,45 +59,45 @@ var ( }, []string{"capture", "changefeed"}) bucketSizeCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "sink", Name: "bucket_size", Help: "size of the DML bucket", }, []string{"capture", "changefeed", "bucket"}) totalRowsCountGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "sink", Name: "total_rows_count", Help: "The total count of rows that are processed by sink", }, []string{"capture", "changefeed"}) totalFlushedRowsCountGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "sink", Name: "total_flushed_rows_count", Help: "The total count of rows that are flushed by sink", }, []string{"capture", "changefeed"}) flushRowChangedDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "sink", Name: "flush_event_duration_seconds", Help: "Bucketed histogram of processing time (s) of flushing events in processor", Buckets: prometheus.ExponentialBuckets(0.002 /* 2ms */, 2, 20), }, []string{"capture", "changefeed", "type"}) - tableSinkTotalRowsCountCounter = prometheus.NewCounterVec( + keyspanSinkTotalEventsCountCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "sink", - Name: "table_sink_total_rows_count", - Help: "The total count of rows that are processed by table sink", + Name: "keyspan_sink_total_event_count", + Help: "The total count of rows that are processed by keyspan sink", }, []string{"capture", "changefeed"}) bufferSinkTotalRowsCountCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "sink", Name: "buffer_sink_total_rows_count", Help: "The total count of rows that are processed by buffer sink", @@ -115,6 +115,6 @@ func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(totalRowsCountGauge) registry.MustRegister(totalFlushedRowsCountGauge) registry.MustRegister(flushRowChangedDuration) - registry.MustRegister(tableSinkTotalRowsCountCounter) + registry.MustRegister(keyspanSinkTotalEventsCountCounter) registry.MustRegister(bufferSinkTotalRowsCountCounter) } diff --git a/cdc/cdc/sink/mq.go b/cdc/cdc/sink/mq.go deleted file mode 100644 index c3162689..00000000 --- a/cdc/cdc/sink/mq.go +++ /dev/null @@ -1,440 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package sink - -import ( - "context" - "net/url" - "strings" - "sync/atomic" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/sink/codec" - "github.com/tikv/migration/cdc/cdc/sink/dispatcher" - "github.com/tikv/migration/cdc/cdc/sink/producer" - "github.com/tikv/migration/cdc/cdc/sink/producer/kafka" - "github.com/tikv/migration/cdc/cdc/sink/producer/pulsar" - "github.com/tikv/migration/cdc/pkg/config" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/filter" - "github.com/tikv/migration/cdc/pkg/notify" - "github.com/tikv/migration/cdc/pkg/security" - "go.uber.org/zap" - "golang.org/x/sync/errgroup" -) - -type mqEvent struct { - row *model.RowChangedEvent - resolvedTs uint64 -} - -const ( - defaultPartitionInputChSize = 12800 - // -1 means broadcast to all partitions, it's the default for the default open protocol. - defaultDDLDispatchPartition = -1 -) - -type mqSink struct { - mqProducer producer.Producer - dispatcher dispatcher.Dispatcher - encoderBuilder codec.EncoderBuilder - filter *filter.Filter - protocol config.Protocol - - partitionNum int32 - partitionInput []chan mqEvent - partitionResolvedTs []uint64 - tableCheckpointTs map[model.TableID]uint64 - resolvedNotifier *notify.Notifier - resolvedReceiver *notify.Receiver - - statistics *Statistics -} - -func newMqSink( - ctx context.Context, credential *security.Credential, mqProducer producer.Producer, - filter *filter.Filter, replicaConfig *config.ReplicaConfig, opts map[string]string, errCh chan error, -) (*mqSink, error) { - var protocol config.Protocol - err := protocol.FromString(replicaConfig.Sink.Protocol) - if err != nil { - return nil, cerror.WrapError(cerror.ErrKafkaInvalidConfig, err) - } - encoderBuilder, err := codec.NewEventBatchEncoderBuilder(protocol, credential, opts) - if err != nil { - return nil, cerror.WrapError(cerror.ErrKafkaInvalidConfig, err) - } - // pre-flight verification of encoder parameters - if _, err := encoderBuilder.Build(ctx); err != nil { - return nil, cerror.WrapError(cerror.ErrKafkaInvalidConfig, err) - } - - partitionNum := mqProducer.GetPartitionNum() - d, err := dispatcher.NewDispatcher(replicaConfig, partitionNum) - if err != nil { - return nil, errors.Trace(err) - } - - partitionInput := make([]chan mqEvent, partitionNum) - for i := 0; i < int(partitionNum); i++ { - partitionInput[i] = make(chan mqEvent, defaultPartitionInputChSize) - } - - notifier := new(notify.Notifier) - resolvedReceiver, err := notifier.NewReceiver(50 * time.Millisecond) - if err != nil { - return nil, errors.Trace(err) - } - - s := &mqSink{ - mqProducer: mqProducer, - dispatcher: d, - encoderBuilder: encoderBuilder, - filter: filter, - protocol: protocol, - - partitionNum: partitionNum, - partitionInput: partitionInput, - partitionResolvedTs: make([]uint64, partitionNum), - tableCheckpointTs: make(map[model.TableID]uint64), - resolvedNotifier: notifier, - resolvedReceiver: resolvedReceiver, - - statistics: NewStatistics(ctx, "MQ", opts), - } - - go func() { - if err := s.run(ctx); err != nil && errors.Cause(err) != context.Canceled { - select { - case <-ctx.Done(): - return - case errCh <- err: - default: - log.Error("error channel is full", zap.Error(err)) - } - } - }() - return s, nil -} - -func (k *mqSink) EmitRowChangedEvents(ctx context.Context, rows ...*model.RowChangedEvent) error { - rowsCount := 0 - for _, row := range rows { - if k.filter.ShouldIgnoreDMLEvent(row.StartTs, row.Table.Schema, row.Table.Table) { - log.Info("Row changed event ignored", zap.Uint64("start-ts", row.StartTs)) - continue - } - partition := k.dispatcher.Dispatch(row) - select { - case <-ctx.Done(): - return ctx.Err() - case k.partitionInput[partition] <- struct { - row *model.RowChangedEvent - resolvedTs uint64 - }{row: row}: - } - rowsCount++ - } - k.statistics.AddRowsCount(rowsCount) - return nil -} - -func (k *mqSink) FlushRowChangedEvents(ctx context.Context, tableID model.TableID, resolvedTs uint64) (uint64, error) { - if checkpointTs, ok := k.tableCheckpointTs[tableID]; ok && resolvedTs <= checkpointTs { - return checkpointTs, nil - } - - for i := 0; i < int(k.partitionNum); i++ { - select { - case <-ctx.Done(): - return 0, ctx.Err() - case k.partitionInput[i] <- struct { - row *model.RowChangedEvent - resolvedTs uint64 - }{resolvedTs: resolvedTs}: - } - } - - // waiting for all row events are sent to mq producer -flushLoop: - for { - select { - case <-ctx.Done(): - return 0, ctx.Err() - case <-k.resolvedReceiver.C: - for i := 0; i < int(k.partitionNum); i++ { - if resolvedTs > atomic.LoadUint64(&k.partitionResolvedTs[i]) { - continue flushLoop - } - } - break flushLoop - } - } - err := k.mqProducer.Flush(ctx) - if err != nil { - return 0, errors.Trace(err) - } - k.tableCheckpointTs[tableID] = resolvedTs - k.statistics.PrintStatus(ctx) - return resolvedTs, nil -} - -func (k *mqSink) EmitCheckpointTs(ctx context.Context, ts uint64) error { - encoder, err := k.encoderBuilder.Build(ctx) - if err != nil { - return errors.Trace(err) - } - msg, err := encoder.EncodeCheckpointEvent(ts) - if err != nil { - return errors.Trace(err) - } - if msg == nil { - return nil - } - err = k.writeToProducer(ctx, msg, codec.EncoderNeedSyncWrite, -1) - return errors.Trace(err) -} - -func (k *mqSink) EmitDDLEvent(ctx context.Context, ddl *model.DDLEvent) error { - if k.filter.ShouldIgnoreDDLEvent(ddl.StartTs, ddl.Type, ddl.TableInfo.Schema, ddl.TableInfo.Table) { - log.Info( - "DDL event ignored", - zap.String("query", ddl.Query), - zap.Uint64("startTs", ddl.StartTs), - zap.Uint64("commitTs", ddl.CommitTs), - ) - return cerror.ErrDDLEventIgnored.GenWithStackByArgs() - } - encoder, err := k.encoderBuilder.Build(ctx) - if err != nil { - return errors.Trace(err) - } - msg, err := encoder.EncodeDDLEvent(ddl) - if err != nil { - return errors.Trace(err) - } - - if msg == nil { - return nil - } - - var partition int32 = defaultDDLDispatchPartition - // for Canal-JSON / Canal-PB, send to partition 0. - if _, ok := encoder.(*codec.CanalFlatEventBatchEncoder); ok { - partition = 0 - } - if _, ok := encoder.(*codec.CanalEventBatchEncoder); ok { - partition = 0 - } - - k.statistics.AddDDLCount() - log.Debug("emit ddl event", zap.String("query", ddl.Query), zap.Uint64("commit-ts", ddl.CommitTs), zap.Int32("partition", partition)) - err = k.writeToProducer(ctx, msg, codec.EncoderNeedSyncWrite, partition) - return errors.Trace(err) -} - -func (k *mqSink) Close(ctx context.Context) error { - err := k.mqProducer.Close() - return errors.Trace(err) -} - -func (k *mqSink) Barrier(cxt context.Context, tableID model.TableID) error { - // Barrier does nothing because FlushRowChangedEvents in mq sink has flushed - // all buffered events by force. - return nil -} - -func (k *mqSink) run(ctx context.Context) error { - defer k.resolvedReceiver.Stop() - wg, ctx := errgroup.WithContext(ctx) - for i := int32(0); i < k.partitionNum; i++ { - partition := i - wg.Go(func() error { - return k.runWorker(ctx, partition) - }) - } - return wg.Wait() -} - -const batchSizeLimit = 4 * 1024 * 1024 // 4MB - -func (k *mqSink) runWorker(ctx context.Context, partition int32) error { - input := k.partitionInput[partition] - encoder, err := k.encoderBuilder.Build(ctx) - if err != nil { - return errors.Trace(err) - } - tick := time.NewTicker(500 * time.Millisecond) - defer tick.Stop() - - flushToProducer := func(op codec.EncoderResult) error { - return k.statistics.RecordBatchExecution(func() (int, error) { - messages := encoder.Build() - thisBatchSize := 0 - if len(messages) == 0 { - return 0, nil - } - - for _, msg := range messages { - err := k.writeToProducer(ctx, msg, codec.EncoderNeedAsyncWrite, partition) - if err != nil { - return 0, err - } - thisBatchSize += msg.GetRowsCount() - } - - if op == codec.EncoderNeedSyncWrite { - err := k.mqProducer.Flush(ctx) - if err != nil { - return 0, err - } - } - log.Debug("MQSink flushed", zap.Int("thisBatchSize", thisBatchSize)) - return thisBatchSize, nil - }) - } - for { - var e mqEvent - select { - case <-ctx.Done(): - return ctx.Err() - case <-tick.C: - if err := flushToProducer(codec.EncoderNeedAsyncWrite); err != nil { - return errors.Trace(err) - } - continue - case e = <-input: - } - if e.row == nil { - if e.resolvedTs != 0 { - op, err := encoder.AppendResolvedEvent(e.resolvedTs) - if err != nil { - return errors.Trace(err) - } - - if err := flushToProducer(op); err != nil { - return errors.Trace(err) - } - - atomic.StoreUint64(&k.partitionResolvedTs[partition], e.resolvedTs) - k.resolvedNotifier.Notify() - } - continue - } - op, err := encoder.AppendRowChangedEvent(e.row) - if err != nil { - return errors.Trace(err) - } - - if encoder.Size() >= batchSizeLimit { - op = codec.EncoderNeedAsyncWrite - } - - if encoder.Size() >= batchSizeLimit || op != codec.EncoderNoOperation { - if err := flushToProducer(op); err != nil { - return errors.Trace(err) - } - } - } -} - -func (k *mqSink) writeToProducer(ctx context.Context, message *codec.MQMessage, op codec.EncoderResult, partition int32) error { - switch op { - case codec.EncoderNeedAsyncWrite: - if partition >= 0 { - return k.mqProducer.AsyncSendMessage(ctx, message, partition) - } - return cerror.ErrAsyncBroadcastNotSupport.GenWithStackByArgs() - case codec.EncoderNeedSyncWrite: - if partition >= 0 { - err := k.mqProducer.AsyncSendMessage(ctx, message, partition) - if err != nil { - return err - } - return k.mqProducer.Flush(ctx) - } - return k.mqProducer.SyncBroadcastMessage(ctx, message) - } - - log.Warn("writeToProducer called with no-op", - zap.ByteString("key", message.Key), - zap.ByteString("value", message.Value), - zap.Int32("partition", partition)) - return nil -} - -func newKafkaSaramaSink(ctx context.Context, sinkURI *url.URL, filter *filter.Filter, replicaConfig *config.ReplicaConfig, opts map[string]string, errCh chan error) (*mqSink, error) { - producerConfig := kafka.NewConfig() - if err := kafka.CompleteConfigsAndOpts(sinkURI, producerConfig, replicaConfig, opts); err != nil { - return nil, cerror.WrapError(cerror.ErrKafkaInvalidConfig, err) - } - // NOTICE: Please check after the completion, as we may get the configuration from the sinkURI. - err := replicaConfig.Validate() - if err != nil { - return nil, err - } - - topic := strings.TrimFunc(sinkURI.Path, func(r rune) bool { - return r == '/' - }) - if topic == "" { - return nil, cerror.ErrKafkaInvalidConfig.GenWithStack("no topic is specified in sink-uri") - } - - sProducer, err := kafka.NewKafkaSaramaProducer(ctx, topic, producerConfig, opts, errCh) - if err != nil { - return nil, errors.Trace(err) - } - sink, err := newMqSink(ctx, producerConfig.Credential, sProducer, filter, replicaConfig, opts, errCh) - if err != nil { - return nil, errors.Trace(err) - } - return sink, nil -} - -func newPulsarSink(ctx context.Context, sinkURI *url.URL, filter *filter.Filter, replicaConfig *config.ReplicaConfig, opts map[string]string, errCh chan error) (*mqSink, error) { - producer, err := pulsar.NewProducer(sinkURI, errCh) - if err != nil { - return nil, errors.Trace(err) - } - s := sinkURI.Query().Get(config.ProtocolKey) - if s != "" { - replicaConfig.Sink.Protocol = s - } - // These two options are not used by Pulsar producer itself, but the encoders - s = sinkURI.Query().Get("max-message-bytes") - if s != "" { - opts["max-message-bytes"] = s - } - - s = sinkURI.Query().Get("max-batch-size") - if s != "" { - opts["max-batch-size"] = s - } - err = replicaConfig.Validate() - if err != nil { - return nil, err - } - // For now, it's a placeholder. Avro format have to make connection to Schema Registry, - // and it may need credential. - credential := &security.Credential{} - sink, err := newMqSink(ctx, credential, producer, filter, replicaConfig, opts, errCh) - if err != nil { - return nil, errors.Trace(err) - } - return sink, nil -} diff --git a/cdc/cdc/sink/mq_test.go b/cdc/cdc/sink/mq_test.go deleted file mode 100644 index 134cfe15..00000000 --- a/cdc/cdc/sink/mq_test.go +++ /dev/null @@ -1,346 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package sink - -import ( - "context" - "fmt" - "net/url" - - "github.com/Shopify/sarama" - "github.com/pingcap/check" - "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/sink/codec" - kafkap "github.com/tikv/migration/cdc/cdc/sink/producer/kafka" - "github.com/tikv/migration/cdc/pkg/config" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/filter" - "github.com/tikv/migration/cdc/pkg/kafka" - "github.com/tikv/migration/cdc/pkg/util/testleak" -) - -type mqSinkSuite struct{} - -var _ = check.Suite(&mqSinkSuite{}) - -func (s mqSinkSuite) TestKafkaSink(c *check.C) { - defer testleak.AfterTest(c)() - ctx, cancel := context.WithCancel(context.Background()) - - topic := kafka.DefaultMockTopicName - leader := sarama.NewMockBroker(c, 1) - defer leader.Close() - metadataResponse := new(sarama.MetadataResponse) - metadataResponse.AddBroker(leader.Addr(), leader.BrokerID()) - metadataResponse.AddTopicPartition(topic, 0, leader.BrokerID(), nil, nil, nil, sarama.ErrNoError) - leader.Returns(metadataResponse) - leader.Returns(metadataResponse) - - prodSuccess := new(sarama.ProduceResponse) - prodSuccess.AddTopicPartition(topic, 0, sarama.ErrNoError) - - uriTemplate := "kafka://%s/%s?kafka-version=0.9.0.0&max-batch-size=1" + - "&max-message-bytes=1048576&partition-num=1" + - "&kafka-client-id=unit-test&auto-create-topic=false&compression=gzip&protocol=open-protocol" - uri := fmt.Sprintf(uriTemplate, leader.Addr(), topic) - sinkURI, err := url.Parse(uri) - c.Assert(err, check.IsNil) - replicaConfig := config.GetDefaultReplicaConfig() - fr, err := filter.NewFilter(replicaConfig) - c.Assert(err, check.IsNil) - opts := map[string]string{} - errCh := make(chan error, 1) - - kafkap.NewAdminClientImpl = kafka.NewMockAdminClient - defer func() { - kafkap.NewAdminClientImpl = kafka.NewSaramaAdminClient - }() - - sink, err := newKafkaSaramaSink(ctx, sinkURI, fr, replicaConfig, opts, errCh) - c.Assert(err, check.IsNil) - - encoder, err := sink.encoderBuilder.Build(ctx) - c.Assert(err, check.IsNil) - - c.Assert(encoder, check.FitsTypeOf, &codec.JSONEventBatchEncoder{}) - c.Assert(encoder.(*codec.JSONEventBatchEncoder).GetMaxBatchSize(), check.Equals, 1) - c.Assert(encoder.(*codec.JSONEventBatchEncoder).GetMaxMessageBytes(), check.Equals, 1048576) - - // mock kafka broker processes 1 row changed event - leader.Returns(prodSuccess) - tableID := model.TableID(1) - row := &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t1", - TableID: tableID, - }, - StartTs: 100, - CommitTs: 120, - Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}}, - } - err = sink.EmitRowChangedEvents(ctx, row) - c.Assert(err, check.IsNil) - checkpointTs, err := sink.FlushRowChangedEvents(ctx, tableID, uint64(120)) - c.Assert(err, check.IsNil) - c.Assert(checkpointTs, check.Equals, uint64(120)) - // flush older resolved ts - checkpointTs, err = sink.FlushRowChangedEvents(ctx, tableID, uint64(110)) - c.Assert(err, check.IsNil) - c.Assert(checkpointTs, check.Equals, uint64(120)) - - // mock kafka broker processes 1 checkpoint ts event - leader.Returns(prodSuccess) - err = sink.EmitCheckpointTs(ctx, uint64(120)) - c.Assert(err, check.IsNil) - - // mock kafka broker processes 1 ddl event - leader.Returns(prodSuccess) - ddl := &model.DDLEvent{ - StartTs: 130, - CommitTs: 140, - TableInfo: &model.SimpleTableInfo{ - Schema: "a", Table: "b", - }, - Query: "create table a", - Type: 1, - } - err = sink.EmitDDLEvent(ctx, ddl) - c.Assert(err, check.IsNil) - - cancel() - err = sink.EmitRowChangedEvents(ctx, row) - if err != nil { - c.Assert(errors.Cause(err), check.Equals, context.Canceled) - } - err = sink.EmitDDLEvent(ctx, ddl) - if err != nil { - c.Assert(errors.Cause(err), check.Equals, context.Canceled) - } - err = sink.EmitCheckpointTs(ctx, uint64(140)) - if err != nil { - c.Assert(errors.Cause(err), check.Equals, context.Canceled) - } - - err = sink.Close(ctx) - if err != nil { - c.Assert(errors.Cause(err), check.Equals, context.Canceled) - } -} - -func (s mqSinkSuite) TestKafkaSinkFilter(c *check.C) { - defer testleak.AfterTest(c)() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - topic := kafka.DefaultMockTopicName - leader := sarama.NewMockBroker(c, 1) - defer leader.Close() - metadataResponse := new(sarama.MetadataResponse) - metadataResponse.AddBroker(leader.Addr(), leader.BrokerID()) - metadataResponse.AddTopicPartition(topic, 0, leader.BrokerID(), nil, nil, nil, sarama.ErrNoError) - leader.Returns(metadataResponse) - leader.Returns(metadataResponse) - - prodSuccess := new(sarama.ProduceResponse) - prodSuccess.AddTopicPartition(topic, 0, sarama.ErrNoError) - - uriTemplate := "kafka://%s/%s?kafka-version=0.9.0.0&auto-create-topic=false&protocol=open-protocol" - uri := fmt.Sprintf(uriTemplate, leader.Addr(), topic) - sinkURI, err := url.Parse(uri) - c.Assert(err, check.IsNil) - replicaConfig := config.GetDefaultReplicaConfig() - replicaConfig.Filter = &config.FilterConfig{ - Rules: []string{"test.*"}, - } - fr, err := filter.NewFilter(replicaConfig) - c.Assert(err, check.IsNil) - opts := map[string]string{} - errCh := make(chan error, 1) - - kafkap.NewAdminClientImpl = kafka.NewMockAdminClient - defer func() { - kafkap.NewAdminClientImpl = kafka.NewSaramaAdminClient - }() - - sink, err := newKafkaSaramaSink(ctx, sinkURI, fr, replicaConfig, opts, errCh) - c.Assert(err, check.IsNil) - - row := &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "order", - Table: "t1", - }, - StartTs: 100, - CommitTs: 120, - } - err = sink.EmitRowChangedEvents(ctx, row) - c.Assert(err, check.IsNil) - c.Assert(sink.statistics.TotalRowsCount(), check.Equals, uint64(0)) - - ddl := &model.DDLEvent{ - StartTs: 130, - CommitTs: 140, - TableInfo: &model.SimpleTableInfo{ - Schema: "lineitem", Table: "t2", - }, - Query: "create table lineitem.t2", - Type: 1, - } - err = sink.EmitDDLEvent(ctx, ddl) - c.Assert(cerror.ErrDDLEventIgnored.Equal(err), check.IsTrue) - - err = sink.Close(ctx) - if err != nil { - c.Assert(errors.Cause(err), check.Equals, context.Canceled) - } -} - -func (s mqSinkSuite) TestPulsarSinkEncoderConfig(c *check.C) { - defer testleak.AfterTest(c)() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - err := failpoint.Enable("github.com/tikv/migration/cdc/cdc/sink/producer/pulsar/MockPulsar", "return(true)") - c.Assert(err, check.IsNil) - - uri := "pulsar://127.0.0.1:1234/kafka-test?" + - "max-message-bytes=4194304&max-batch-size=1" - - sinkURI, err := url.Parse(uri) - c.Assert(err, check.IsNil) - replicaConfig := config.GetDefaultReplicaConfig() - fr, err := filter.NewFilter(replicaConfig) - c.Assert(err, check.IsNil) - opts := map[string]string{} - errCh := make(chan error, 1) - sink, err := newPulsarSink(ctx, sinkURI, fr, replicaConfig, opts, errCh) - c.Assert(err, check.IsNil) - - encoder, err := sink.encoderBuilder.Build(ctx) - c.Assert(err, check.IsNil) - c.Assert(encoder, check.FitsTypeOf, &codec.JSONEventBatchEncoder{}) - c.Assert(encoder.(*codec.JSONEventBatchEncoder).GetMaxBatchSize(), check.Equals, 1) - c.Assert(encoder.(*codec.JSONEventBatchEncoder).GetMaxMessageBytes(), check.Equals, 4194304) -} - -func (s mqSinkSuite) TestFlushRowChangedEvents(c *check.C) { - defer testleak.AfterTest(c)() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - topic := kafka.DefaultMockTopicName - leader := sarama.NewMockBroker(c, 1) - defer leader.Close() - - metadataResponse := new(sarama.MetadataResponse) - metadataResponse.AddBroker(leader.Addr(), leader.BrokerID()) - metadataResponse.AddTopicPartition(topic, 0, leader.BrokerID(), nil, nil, nil, sarama.ErrNoError) - leader.Returns(metadataResponse) - leader.Returns(metadataResponse) - - prodSuccess := new(sarama.ProduceResponse) - prodSuccess.AddTopicPartition(topic, 0, sarama.ErrNoError) - - uriTemplate := "kafka://%s/%s?kafka-version=0.9.0.0&max-batch-size=1" + - "&max-message-bytes=1048576&partition-num=1" + - "&kafka-client-id=unit-test&auto-create-topic=false&compression=gzip&protocol=open-protocol" - uri := fmt.Sprintf(uriTemplate, leader.Addr(), topic) - sinkURI, err := url.Parse(uri) - c.Assert(err, check.IsNil) - replicaConfig := config.GetDefaultReplicaConfig() - fr, err := filter.NewFilter(replicaConfig) - c.Assert(err, check.IsNil) - opts := map[string]string{} - errCh := make(chan error, 1) - - kafkap.NewAdminClientImpl = kafka.NewMockAdminClient - defer func() { - kafkap.NewAdminClientImpl = kafka.NewSaramaAdminClient - }() - - sink, err := newKafkaSaramaSink(ctx, sinkURI, fr, replicaConfig, opts, errCh) - c.Assert(err, check.IsNil) - - // mock kafka broker processes 1 row changed event - leader.Returns(prodSuccess) - tableID1 := model.TableID(1) - row1 := &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t1", - TableID: tableID1, - }, - StartTs: 100, - CommitTs: 120, - Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}}, - } - err = sink.EmitRowChangedEvents(ctx, row1) - c.Assert(err, check.IsNil) - - tableID2 := model.TableID(2) - row2 := &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t2", - TableID: tableID2, - }, - StartTs: 90, - CommitTs: 110, - Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}}, - } - err = sink.EmitRowChangedEvents(ctx, row2) - c.Assert(err, check.IsNil) - - tableID3 := model.TableID(3) - row3 := &model.RowChangedEvent{ - Table: &model.TableName{ - Schema: "test", - Table: "t3", - TableID: tableID3, - }, - StartTs: 110, - CommitTs: 130, - Columns: []*model.Column{{Name: "col1", Type: 1, Value: "aa"}}, - } - - err = sink.EmitRowChangedEvents(ctx, row3) - c.Assert(err, check.IsNil) - - // mock kafka broker processes 1 row resolvedTs event - leader.Returns(prodSuccess) - checkpointTs1, err := sink.FlushRowChangedEvents(ctx, tableID1, row1.CommitTs) - c.Assert(err, check.IsNil) - c.Assert(checkpointTs1, check.Equals, row1.CommitTs) - - checkpointTs2, err := sink.FlushRowChangedEvents(ctx, tableID2, row2.CommitTs) - c.Assert(err, check.IsNil) - c.Assert(checkpointTs2, check.Equals, row2.CommitTs) - - checkpointTs3, err := sink.FlushRowChangedEvents(ctx, tableID3, row3.CommitTs) - c.Assert(err, check.IsNil) - c.Assert(checkpointTs3, check.Equals, row3.CommitTs) - - // flush older resolved ts - checkpointTsOld, err := sink.FlushRowChangedEvents(ctx, tableID1, uint64(110)) - c.Assert(err, check.IsNil) - c.Assert(checkpointTsOld, check.Equals, row1.CommitTs) - - err = sink.Close(ctx) - if err != nil { - c.Assert(errors.Cause(err), check.Equals, context.Canceled) - } -} diff --git a/cdc/cdc/sink/mysql.go b/cdc/cdc/sink/mysql.go deleted file mode 100644 index 2124ef36..00000000 --- a/cdc/cdc/sink/mysql.go +++ /dev/null @@ -1,922 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package sink - -import ( - "context" - "database/sql" - "fmt" - "net/url" - "strconv" - "strings" - "sync" - "time" - - dmysql "github.com/go-sql-driver/mysql" - "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/pingcap/log" - timodel "github.com/pingcap/tidb/parser/model" - "github.com/pingcap/tidb/parser/mysql" - "github.com/prometheus/client_golang/prometheus" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/sink/common" - "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/cyclic" - "github.com/tikv/migration/cdc/pkg/cyclic/mark" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/errorutil" - tifilter "github.com/tikv/migration/cdc/pkg/filter" - "github.com/tikv/migration/cdc/pkg/notify" - "github.com/tikv/migration/cdc/pkg/quotes" - "github.com/tikv/migration/cdc/pkg/retry" - "go.uber.org/zap" -) - -const ( - backoffBaseDelayInMs = 500 - // in previous/backoff retry pkg, the DefaultMaxInterval = 60 * time.Second - backoffMaxDelayInMs = 60 * 1000 -) - -type mysqlSink struct { - db *sql.DB - params *sinkParams - - filter *tifilter.Filter - cyclic *cyclic.Cyclic - - txnCache *common.UnresolvedTxnCache - workers []*mysqlSinkWorker - tableCheckpointTs sync.Map - tableMaxResolvedTs sync.Map - - execWaitNotifier *notify.Notifier - resolvedNotifier *notify.Notifier - errCh chan error - flushSyncWg sync.WaitGroup - - statistics *Statistics - - // metrics used by mysql sink only - metricConflictDetectDurationHis prometheus.Observer - metricBucketSizeCounters []prometheus.Counter - - forceReplicate bool - cancel func() -} - -var _ Sink = &mysqlSink{} - -// newMySQLSink creates a new MySQL sink using schema storage -func newMySQLSink( - ctx context.Context, - changefeedID model.ChangeFeedID, - sinkURI *url.URL, - filter *tifilter.Filter, - replicaConfig *config.ReplicaConfig, - opts map[string]string, -) (Sink, error) { - opts[OptChangefeedID] = changefeedID - params, err := parseSinkURIToParams(ctx, sinkURI, opts) - if err != nil { - return nil, err - } - - params.enableOldValue = replicaConfig.EnableOldValue - - // dsn format of the driver: - // [username[:password]@][protocol[(address)]]/dbname[?param1=value1&...¶mN=valueN] - username := sinkURI.User.Username() - password, _ := sinkURI.User.Password() - port := sinkURI.Port() - if username == "" { - username = "root" - } - if port == "" { - port = "4000" - } - - dsnStr := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s", username, password, sinkURI.Hostname(), port, params.tls) - dsn, err := dmysql.ParseDSN(dsnStr) - if err != nil { - return nil, cerror.WrapError(cerror.ErrMySQLInvalidConfig, err) - } - - // create test db used for parameter detection - if dsn.Params == nil { - dsn.Params = make(map[string]string, 1) - } - if params.timezone != "" { - dsn.Params["time_zone"] = params.timezone - } - dsn.Params["readTimeout"] = params.readTimeout - dsn.Params["writeTimeout"] = params.writeTimeout - dsn.Params["timeout"] = params.dialTimeout - testDB, err := GetDBConnImpl(ctx, dsn.FormatDSN()) - if err != nil { - return nil, err - } - defer testDB.Close() - - dsnStr, err = generateDSNByParams(ctx, dsn, params, testDB) - if err != nil { - return nil, errors.Trace(err) - } - db, err := GetDBConnImpl(ctx, dsnStr) - if err != nil { - return nil, err - } - - log.Info("Start mysql sink") - - db.SetMaxIdleConns(params.workerCount) - db.SetMaxOpenConns(params.workerCount) - - metricConflictDetectDurationHis := conflictDetectDurationHis.WithLabelValues( - params.captureAddr, params.changefeedID) - metricBucketSizeCounters := make([]prometheus.Counter, params.workerCount) - for i := 0; i < params.workerCount; i++ { - metricBucketSizeCounters[i] = bucketSizeCounter.WithLabelValues( - params.captureAddr, params.changefeedID, strconv.Itoa(i)) - } - ctx, cancel := context.WithCancel(ctx) - - sink := &mysqlSink{ - db: db, - params: params, - filter: filter, - txnCache: common.NewUnresolvedTxnCache(), - statistics: NewStatistics(ctx, "mysql", opts), - metricConflictDetectDurationHis: metricConflictDetectDurationHis, - metricBucketSizeCounters: metricBucketSizeCounters, - errCh: make(chan error, 1), - forceReplicate: replicaConfig.ForceReplicate, - cancel: cancel, - } - - if val, ok := opts[mark.OptCyclicConfig]; ok { - cfg := new(config.CyclicConfig) - err := cfg.Unmarshal([]byte(val)) - if err != nil { - return nil, cerror.WrapError(cerror.ErrMySQLInvalidConfig, err) - } - sink.cyclic = cyclic.NewCyclic(cfg) - - err = sink.adjustSQLMode(ctx) - if err != nil { - return nil, errors.Trace(err) - } - } - - sink.execWaitNotifier = new(notify.Notifier) - sink.resolvedNotifier = new(notify.Notifier) - - err = sink.createSinkWorkers(ctx) - - if err != nil { - return nil, err - } - - receiver, err := sink.resolvedNotifier.NewReceiver(50 * time.Millisecond) - if err != nil { - return nil, err - } - go sink.flushRowChangedEvents(ctx, receiver) - - return sink, nil -} - -func (s *mysqlSink) EmitRowChangedEvents(ctx context.Context, rows ...*model.RowChangedEvent) error { - count := s.txnCache.Append(s.filter, rows...) - s.statistics.AddRowsCount(count) - return nil -} - -// FlushRowChangedEvents will flush all received events, we don't allow mysql -// sink to receive events before resolving -func (s *mysqlSink) FlushRowChangedEvents(ctx context.Context, tableID model.TableID, resolvedTs uint64) (uint64, error) { - v, ok := s.tableMaxResolvedTs.Load(tableID) - if !ok || v.(uint64) < resolvedTs { - s.tableMaxResolvedTs.Store(tableID, resolvedTs) - } - s.resolvedNotifier.Notify() - - // check and throw error - select { - case err := <-s.errCh: - return 0, err - default: - } - - checkpointTs := s.getTableCheckpointTs(tableID) - s.statistics.PrintStatus(ctx) - return checkpointTs, nil -} - -func (s *mysqlSink) flushRowChangedEvents(ctx context.Context, receiver *notify.Receiver) { - defer func() { - for _, worker := range s.workers { - worker.closedCh <- struct{}{} - } - }() - for { - select { - case <-ctx.Done(): - return - case <-receiver.C: - } - flushedResolvedTsMap, resolvedTxnsMap := s.txnCache.Resolved(&s.tableMaxResolvedTs) - if len(resolvedTxnsMap) == 0 { - s.tableMaxResolvedTs.Range(func(key, value interface{}) bool { - s.tableCheckpointTs.Store(key, value) - return true - }) - continue - } - - if s.cyclic != nil { - // Filter rows if it is origin from downstream. - skippedRowCount := cyclic.FilterAndReduceTxns( - resolvedTxnsMap, s.cyclic.FilterReplicaID(), s.cyclic.ReplicaID()) - s.statistics.SubRowsCount(skippedRowCount) - } - - s.dispatchAndExecTxns(ctx, resolvedTxnsMap) - for tableID, resolvedTs := range flushedResolvedTsMap { - s.tableCheckpointTs.Store(tableID, resolvedTs) - } - } -} - -func (s *mysqlSink) EmitCheckpointTs(ctx context.Context, ts uint64) error { - // do nothing - return nil -} - -func (s *mysqlSink) EmitDDLEvent(ctx context.Context, ddl *model.DDLEvent) error { - if s.filter.ShouldIgnoreDDLEvent(ddl.StartTs, ddl.Type, ddl.TableInfo.Schema, ddl.TableInfo.Table) { - log.Info( - "DDL event ignored", - zap.String("query", ddl.Query), - zap.Uint64("startTs", ddl.StartTs), - zap.Uint64("commitTs", ddl.CommitTs), - ) - return cerror.ErrDDLEventIgnored.GenWithStackByArgs() - } - s.statistics.AddDDLCount() - err := s.execDDLWithMaxRetries(ctx, ddl) - return errors.Trace(err) -} - -func (s *mysqlSink) execDDLWithMaxRetries(ctx context.Context, ddl *model.DDLEvent) error { - return retry.Do(ctx, func() error { - err := s.execDDL(ctx, ddl) - if errorutil.IsIgnorableMySQLDDLError(err) { - log.Info("execute DDL failed, but error can be ignored", zap.String("query", ddl.Query), zap.Error(err)) - return nil - } - if err != nil { - log.Warn("execute DDL with error, retry later", zap.String("query", ddl.Query), zap.Error(err)) - } - return err - }, retry.WithBackoffBaseDelay(backoffBaseDelayInMs), retry.WithBackoffMaxDelay(backoffMaxDelayInMs), retry.WithMaxTries(defaultDDLMaxRetryTime), retry.WithIsRetryableErr(cerror.IsRetryableError)) -} - -func (s *mysqlSink) execDDL(ctx context.Context, ddl *model.DDLEvent) error { - shouldSwitchDB := needSwitchDB(ddl) - - failpoint.Inject("MySQLSinkExecDDLDelay", func() { - select { - case <-ctx.Done(): - failpoint.Return(ctx.Err()) - case <-time.After(time.Hour): - } - failpoint.Return(nil) - }) - err := s.statistics.RecordDDLExecution(func() error { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return err - } - - if shouldSwitchDB { - _, err = tx.ExecContext(ctx, "USE "+quotes.QuoteName(ddl.TableInfo.Schema)+";") - if err != nil { - if rbErr := tx.Rollback(); rbErr != nil { - log.Error("Failed to rollback", zap.Error(err)) - } - return err - } - } - - if _, err = tx.ExecContext(ctx, ddl.Query); err != nil { - if rbErr := tx.Rollback(); rbErr != nil { - log.Error("Failed to rollback", zap.String("sql", ddl.Query), zap.Error(err)) - } - return err - } - - return tx.Commit() - }) - if err != nil { - return cerror.WrapError(cerror.ErrMySQLTxnError, err) - } - - log.Info("Exec DDL succeeded", zap.String("sql", ddl.Query)) - return nil -} - -func needSwitchDB(ddl *model.DDLEvent) bool { - if len(ddl.TableInfo.Schema) == 0 { - return false - } - if ddl.Type == timodel.ActionCreateSchema || ddl.Type == timodel.ActionDropSchema { - return false - } - return true -} - -// adjustSQLMode adjust sql mode according to sink config. -func (s *mysqlSink) adjustSQLMode(ctx context.Context) error { - // Must relax sql mode to support cyclic replication, as downstream may have - // extra columns (not null and no default value). - if s.cyclic == nil || !s.cyclic.Enabled() { - return nil - } - var oldMode, newMode string - row := s.db.QueryRowContext(ctx, "SELECT @@SESSION.sql_mode;") - err := row.Scan(&oldMode) - if err != nil { - return cerror.WrapError(cerror.ErrMySQLQueryError, err) - } - - newMode = cyclic.RelaxSQLMode(oldMode) - _, err = s.db.ExecContext(ctx, fmt.Sprintf("SET sql_mode = '%s';", newMode)) - if err != nil { - return cerror.WrapError(cerror.ErrMySQLQueryError, err) - } - return nil -} - -func (s *mysqlSink) createSinkWorkers(ctx context.Context) error { - s.workers = make([]*mysqlSinkWorker, s.params.workerCount) - for i := range s.workers { - receiver, err := s.execWaitNotifier.NewReceiver(defaultFlushInterval) - if err != nil { - return err - } - worker := newMySQLSinkWorker( - s.params.maxTxnRow, i, s.metricBucketSizeCounters[i], receiver, s.execDMLs) - s.workers[i] = worker - go func() { - err := worker.run(ctx) - if err != nil && errors.Cause(err) != context.Canceled { - select { - case s.errCh <- err: - default: - log.Info("mysql sink receives redundant error", zap.Error(err)) - } - } - worker.cleanup() - }() - } - return nil -} - -func (s *mysqlSink) notifyAndWaitExec(ctx context.Context) { - s.broadcastFinishTxn() - s.execWaitNotifier.Notify() - done := make(chan struct{}) - go func() { - s.flushSyncWg.Wait() - close(done) - }() - // This is a hack code to avoid io wait in some routine blocks others to exit. - // As the network io wait is blocked in kernel code, the goroutine is in a - // D-state that we could not even stop it by cancel the context. So if this - // scenario happens, the blocked goroutine will be leak. - select { - case <-ctx.Done(): - case <-done: - } -} - -func (s *mysqlSink) broadcastFinishTxn() { - // Note all data txn is sent via channel, the control txn must come after all - // data txns in each worker. So after worker receives the control txn, it can - // flush txns immediately and call wait group done once. - for _, worker := range s.workers { - worker.appendFinishTxn(&s.flushSyncWg) - } -} - -func (s *mysqlSink) dispatchAndExecTxns(ctx context.Context, txnsGroup map[model.TableID][]*model.SingleTableTxn) { - nWorkers := s.params.workerCount - causality := newCausality() - rowsChIdx := 0 - - sendFn := func(txn *model.SingleTableTxn, keys [][]byte, idx int) { - causality.add(keys, idx) - s.workers[idx].appendTxn(ctx, txn) - } - resolveConflict := func(txn *model.SingleTableTxn) { - keys := genTxnKeys(txn) - if conflict, idx := causality.detectConflict(keys); conflict { - if idx >= 0 { - sendFn(txn, keys, idx) - return - } - s.notifyAndWaitExec(ctx) - causality.reset() - } - sendFn(txn, keys, rowsChIdx) - rowsChIdx++ - rowsChIdx = rowsChIdx % nWorkers - } - h := newTxnsHeap(txnsGroup) - h.iter(func(txn *model.SingleTableTxn) { - startTime := time.Now() - resolveConflict(txn) - s.metricConflictDetectDurationHis.Observe(time.Since(startTime).Seconds()) - }) - s.notifyAndWaitExec(ctx) -} - -func (s *mysqlSink) Close(ctx context.Context) error { - s.execWaitNotifier.Close() - s.resolvedNotifier.Close() - err := s.db.Close() - s.cancel() - return cerror.WrapError(cerror.ErrMySQLConnectionError, err) -} - -func (s *mysqlSink) Barrier(ctx context.Context, tableID model.TableID) error { - warnDuration := 3 * time.Minute - ticker := time.NewTicker(warnDuration) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return errors.Trace(ctx.Err()) - case <-ticker.C: - maxResolvedTs, ok := s.tableMaxResolvedTs.Load(tableID) - log.Warn("Barrier doesn't return in time, may be stuck", - zap.Int64("tableID", tableID), - zap.Bool("has resolvedTs", ok), - zap.Any("resolvedTs", maxResolvedTs), - zap.Uint64("checkpointTs", s.getTableCheckpointTs(tableID))) - default: - v, ok := s.tableMaxResolvedTs.Load(tableID) - if !ok { - log.Info("No table resolvedTs is found", zap.Int64("table-id", tableID)) - return nil - } - maxResolvedTs := v.(uint64) - if s.getTableCheckpointTs(tableID) >= maxResolvedTs { - return nil - } - checkpointTs, err := s.FlushRowChangedEvents(ctx, tableID, maxResolvedTs) - if err != nil { - return err - } - if checkpointTs >= maxResolvedTs { - return nil - } - // short sleep to avoid cpu spin - time.Sleep(time.Second) - } - } -} - -func (s *mysqlSink) getTableCheckpointTs(tableID model.TableID) uint64 { - v, ok := s.tableCheckpointTs.Load(tableID) - if ok { - return v.(uint64) - } - return uint64(0) -} - -func logDMLTxnErr(err error) error { - if isRetryableDMLError(err) { - log.Warn("execute DMLs with error, retry later", zap.Error(err)) - } - return err -} - -func isRetryableDMLError(err error) bool { - if !cerror.IsRetryableError(err) { - return false - } - - errCode, ok := getSQLErrCode(err) - if !ok { - return true - } - - switch errCode { - case mysql.ErrNoSuchTable, mysql.ErrBadDB: - return false - } - return true -} - -func (s *mysqlSink) execDMLWithMaxRetries(ctx context.Context, dmls *preparedDMLs, bucket int) error { - if len(dmls.sqls) != len(dmls.values) { - log.Panic("unexpected number of sqls and values", - zap.Strings("sqls", dmls.sqls), - zap.Any("values", dmls.values)) - } - - return retry.Do(ctx, func() error { - failpoint.Inject("MySQLSinkTxnRandomError", func() { - failpoint.Return(logDMLTxnErr(errors.Trace(dmysql.ErrInvalidConn))) - }) - failpoint.Inject("MySQLSinkHangLongTime", func() { - time.Sleep(time.Hour) - }) - - err := s.statistics.RecordBatchExecution(func() (int, error) { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return 0, logDMLTxnErr(cerror.WrapError(cerror.ErrMySQLTxnError, err)) - } - - for i, query := range dmls.sqls { - args := dmls.values[i] - log.Debug("exec row", zap.String("sql", query), zap.Any("args", args)) - if _, err := tx.ExecContext(ctx, query, args...); err != nil { - if rbErr := tx.Rollback(); rbErr != nil { - log.Warn("failed to rollback txn", zap.Error(err)) - } - return 0, logDMLTxnErr(cerror.WrapError(cerror.ErrMySQLTxnError, err)) - } - } - - if len(dmls.markSQL) != 0 { - log.Debug("exec row", zap.String("sql", dmls.markSQL)) - if _, err := tx.ExecContext(ctx, dmls.markSQL); err != nil { - if rbErr := tx.Rollback(); rbErr != nil { - log.Warn("failed to rollback txn", zap.Error(err)) - } - return 0, logDMLTxnErr(cerror.WrapError(cerror.ErrMySQLTxnError, err)) - } - } - - if err = tx.Commit(); err != nil { - return 0, logDMLTxnErr(cerror.WrapError(cerror.ErrMySQLTxnError, err)) - } - return dmls.rowCount, nil - }) - if err != nil { - return errors.Trace(err) - } - log.Debug("Exec Rows succeeded", - zap.String("changefeed", s.params.changefeedID), - zap.Int("num of Rows", dmls.rowCount), - zap.Int("bucket", bucket)) - return nil - }, retry.WithBackoffBaseDelay(backoffBaseDelayInMs), retry.WithBackoffMaxDelay(backoffMaxDelayInMs), retry.WithMaxTries(defaultDMLMaxRetryTime), retry.WithIsRetryableErr(isRetryableDMLError)) -} - -type preparedDMLs struct { - sqls []string - values [][]interface{} - markSQL string - rowCount int -} - -// prepareDMLs converts model.RowChangedEvent list to query string list and args list -func (s *mysqlSink) prepareDMLs(rows []*model.RowChangedEvent, replicaID uint64, bucket int) *preparedDMLs { - sqls := make([]string, 0, len(rows)) - values := make([][]interface{}, 0, len(rows)) - replaces := make(map[string][][]interface{}) - rowCount := 0 - translateToInsert := s.params.enableOldValue && !s.params.safeMode - - // flush cached batch replace or insert, to keep the sequence of DMLs - flushCacheDMLs := func() { - if s.params.batchReplaceEnabled && len(replaces) > 0 { - replaceSqls, replaceValues := reduceReplace(replaces, s.params.batchReplaceSize) - sqls = append(sqls, replaceSqls...) - values = append(values, replaceValues...) - replaces = make(map[string][][]interface{}) - } - } - - for _, row := range rows { - var query string - var args []interface{} - quoteTable := quotes.QuoteSchema(row.Table.Schema, row.Table.Table) - - // If the old value is enabled, is not in safe mode and is an update event, then translate to UPDATE. - // NOTICE: Only update events with the old value feature enabled will have both columns and preColumns. - if translateToInsert && len(row.PreColumns) != 0 && len(row.Columns) != 0 { - flushCacheDMLs() - query, args = prepareUpdate(quoteTable, row.PreColumns, row.Columns, s.forceReplicate) - if query != "" { - sqls = append(sqls, query) - values = append(values, args) - rowCount++ - } - continue - } - - // Case for update event or delete event. - // For update event: - // If old value is disabled or in safe mode, update will be translated to DELETE + REPLACE SQL. - // So we will prepare a DELETE SQL here. - // For delete event: - // It will be translated directly into a DELETE SQL. - if len(row.PreColumns) != 0 { - flushCacheDMLs() - query, args = prepareDelete(quoteTable, row.PreColumns, s.forceReplicate) - if query != "" { - sqls = append(sqls, query) - values = append(values, args) - rowCount++ - } - } - - // Case for update event or insert event. - // For update event: - // If old value is disabled or in safe mode, update will be translated to DELETE + REPLACE SQL. - // So we will prepare a REPLACE SQL here. - // For insert event: - // It will be translated directly into a - // INSERT(old value is enabled and not in safe mode) - // or REPLACE(old value is disabled or in safe mode) SQL. - if len(row.Columns) != 0 { - if s.params.batchReplaceEnabled { - query, args = prepareReplace(quoteTable, row.Columns, false /* appendPlaceHolder */, translateToInsert) - if query != "" { - if _, ok := replaces[query]; !ok { - replaces[query] = make([][]interface{}, 0) - } - replaces[query] = append(replaces[query], args) - rowCount++ - } - } else { - query, args = prepareReplace(quoteTable, row.Columns, true /* appendPlaceHolder */, translateToInsert) - sqls = append(sqls, query) - values = append(values, args) - if query != "" { - sqls = append(sqls, query) - values = append(values, args) - rowCount++ - } - } - } - } - flushCacheDMLs() - - dmls := &preparedDMLs{ - sqls: sqls, - values: values, - } - if s.cyclic != nil && len(rows) > 0 { - // Write mark table with the current replica ID. - row := rows[0] - updateMark := s.cyclic.UdpateSourceTableCyclicMark( - row.Table.Schema, row.Table.Table, uint64(bucket), replicaID, row.StartTs) - dmls.markSQL = updateMark - // rowCount is used in statistics, and for simplicity, - // we do not count mark table rows in rowCount. - } - dmls.rowCount = rowCount - return dmls -} - -func (s *mysqlSink) execDMLs(ctx context.Context, rows []*model.RowChangedEvent, replicaID uint64, bucket int) error { - failpoint.Inject("SinkFlushDMLPanic", func() { - time.Sleep(time.Second) - log.Fatal("SinkFlushDMLPanic") - }) - failpoint.Inject("MySQLSinkExecDMLError", func() { - // Add a delay to ensure the sink worker with `MySQLSinkHangLongTime` - // failpoint injected is executed first. - time.Sleep(time.Second * 2) - failpoint.Return(errors.Trace(dmysql.ErrInvalidConn)) - }) - dmls := s.prepareDMLs(rows, replicaID, bucket) - log.Debug("prepare DMLs", zap.Any("rows", rows), zap.Strings("sqls", dmls.sqls), zap.Any("values", dmls.values)) - if err := s.execDMLWithMaxRetries(ctx, dmls, bucket); err != nil { - log.Error("execute DMLs failed", zap.String("err", err.Error())) - return errors.Trace(err) - } - return nil -} - -func prepareReplace( - quoteTable string, - cols []*model.Column, - appendPlaceHolder bool, - translateToInsert bool, -) (string, []interface{}) { - var builder strings.Builder - columnNames := make([]string, 0, len(cols)) - args := make([]interface{}, 0, len(cols)) - for _, col := range cols { - if col == nil || col.Flag.IsGeneratedColumn() { - continue - } - columnNames = append(columnNames, col.Name) - args = append(args, col.Value) - } - if len(args) == 0 { - return "", nil - } - - colList := "(" + buildColumnList(columnNames) + ")" - if translateToInsert { - builder.WriteString("INSERT INTO " + quoteTable + colList + " VALUES ") - } else { - builder.WriteString("REPLACE INTO " + quoteTable + colList + " VALUES ") - } - if appendPlaceHolder { - builder.WriteString("(" + model.HolderString(len(columnNames)) + ");") - } - - return builder.String(), args -} - -// reduceReplace groups SQLs with the same replace statement format, as following -// sql: `REPLACE INTO `test`.`t` (`a`,`b`) VALUES (?,?,?,?,?,?)` -// args: (1,"",2,"2",3,"") -func reduceReplace(replaces map[string][][]interface{}, batchSize int) ([]string, [][]interface{}) { - nextHolderString := func(query string, valueNum int, last bool) string { - query += "(" + model.HolderString(valueNum) + ")" - if !last { - query += "," - } - return query - } - sqls := make([]string, 0) - args := make([][]interface{}, 0) - for replace, vals := range replaces { - query := replace - cacheCount := 0 - cacheArgs := make([]interface{}, 0) - last := false - for i, val := range vals { - cacheCount++ - if i == len(vals)-1 || cacheCount >= batchSize { - last = true - } - query = nextHolderString(query, len(val), last) - cacheArgs = append(cacheArgs, val...) - if last { - sqls = append(sqls, query) - args = append(args, cacheArgs) - query = replace - cacheCount = 0 - cacheArgs = make([]interface{}, 0, len(cacheArgs)) - last = false - } - } - } - return sqls, args -} - -func prepareUpdate(quoteTable string, preCols, cols []*model.Column, forceReplicate bool) (string, []interface{}) { - var builder strings.Builder - builder.WriteString("UPDATE " + quoteTable + " SET ") - - columnNames := make([]string, 0, len(cols)) - args := make([]interface{}, 0, len(cols)+len(preCols)) - for _, col := range cols { - if col == nil || col.Flag.IsGeneratedColumn() { - continue - } - columnNames = append(columnNames, col.Name) - args = append(args, col.Value) - } - if len(args) == 0 { - return "", nil - } - for i, column := range columnNames { - if i == len(columnNames)-1 { - builder.WriteString("`" + quotes.EscapeName(column) + "`=?") - } else { - builder.WriteString("`" + quotes.EscapeName(column) + "`=?,") - } - } - - builder.WriteString(" WHERE ") - colNames, wargs := whereSlice(preCols, forceReplicate) - if len(wargs) == 0 { - return "", nil - } - for i := 0; i < len(colNames); i++ { - if i > 0 { - builder.WriteString(" AND ") - } - if wargs[i] == nil { - builder.WriteString(quotes.QuoteName(colNames[i]) + " IS NULL") - } else { - builder.WriteString(quotes.QuoteName(colNames[i]) + "=?") - args = append(args, wargs[i]) - } - } - builder.WriteString(" LIMIT 1;") - sql := builder.String() - return sql, args -} - -func prepareDelete(quoteTable string, cols []*model.Column, forceReplicate bool) (string, []interface{}) { - var builder strings.Builder - builder.WriteString("DELETE FROM " + quoteTable + " WHERE ") - - colNames, wargs := whereSlice(cols, forceReplicate) - if len(wargs) == 0 { - return "", nil - } - args := make([]interface{}, 0, len(wargs)) - for i := 0; i < len(colNames); i++ { - if i > 0 { - builder.WriteString(" AND ") - } - if wargs[i] == nil { - builder.WriteString(quotes.QuoteName(colNames[i]) + " IS NULL") - } else { - builder.WriteString(quotes.QuoteName(colNames[i]) + " = ?") - args = append(args, wargs[i]) - } - } - builder.WriteString(" LIMIT 1;") - sql := builder.String() - return sql, args -} - -func whereSlice(cols []*model.Column, forceReplicate bool) (colNames []string, args []interface{}) { - // Try to use unique key values when available - for _, col := range cols { - if col == nil || !col.Flag.IsHandleKey() { - continue - } - colNames = append(colNames, col.Name) - args = append(args, col.Value) - } - // if no explicit row id but force replicate, use all key-values in where condition - if len(colNames) == 0 && forceReplicate { - colNames = make([]string, 0, len(cols)) - args = make([]interface{}, 0, len(cols)) - for _, col := range cols { - colNames = append(colNames, col.Name) - args = append(args, col.Value) - } - } - return -} - -func getSQLErrCode(err error) (errors.ErrCode, bool) { - mysqlErr, ok := errors.Cause(err).(*dmysql.MySQLError) - if !ok { - return -1, false - } - - return errors.ErrCode(mysqlErr.Number), true -} - -func buildColumnList(names []string) string { - var b strings.Builder - for i, name := range names { - if i > 0 { - b.WriteString(",") - } - b.WriteString(quotes.QuoteName(name)) - - } - - return b.String() -} - -// GetDBConnImpl is the implement holder to get db connection. Export it for tests -var GetDBConnImpl = getDBConn - -func getDBConn(ctx context.Context, dsnStr string) (*sql.DB, error) { - db, err := sql.Open("mysql", dsnStr) - if err != nil { - return nil, cerror.ErrMySQLConnectionError.Wrap(err).GenWithStack("fail to open MySQL connection") - } - err = db.PingContext(ctx) - if err != nil { - // close db to recycle resources - if closeErr := db.Close(); closeErr != nil { - log.Warn("close db failed", zap.Error(err)) - } - return nil, cerror.ErrMySQLConnectionError.Wrap(err).GenWithStack("fail to open MySQL connection") - } - return db, nil -} diff --git a/cdc/cdc/sink/mysql_params.go b/cdc/cdc/sink/mysql_params.go deleted file mode 100644 index 62fe78f1..00000000 --- a/cdc/cdc/sink/mysql_params.go +++ /dev/null @@ -1,269 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package sink - -import ( - "context" - "database/sql" - "fmt" - "net/url" - "strconv" - "strings" - "time" - - dmysql "github.com/go-sql-driver/mysql" - "github.com/pingcap/errors" - "github.com/pingcap/log" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/security" - "github.com/tikv/migration/cdc/pkg/util" - "go.uber.org/zap" -) - -const ( - // expose these two variables for redo log applier - DefaultWorkerCount = 16 - DefaultMaxTxnRow = 256 - - defaultDMLMaxRetryTime = 8 - defaultDDLMaxRetryTime = 20 - defaultTiDBTxnMode = "optimistic" - defaultFlushInterval = time.Millisecond * 50 - defaultBatchReplaceEnabled = true - defaultBatchReplaceSize = 20 - defaultReadTimeout = "2m" - defaultWriteTimeout = "2m" - defaultDialTimeout = "2m" - defaultSafeMode = true -) - -var defaultParams = &sinkParams{ - workerCount: DefaultWorkerCount, - maxTxnRow: DefaultMaxTxnRow, - tidbTxnMode: defaultTiDBTxnMode, - batchReplaceEnabled: defaultBatchReplaceEnabled, - batchReplaceSize: defaultBatchReplaceSize, - readTimeout: defaultReadTimeout, - writeTimeout: defaultWriteTimeout, - dialTimeout: defaultDialTimeout, - safeMode: defaultSafeMode, -} - -var validSchemes = map[string]bool{ - "mysql": true, - "mysql+ssl": true, - "tidb": true, - "tidb+ssl": true, -} - -type sinkParams struct { - workerCount int - maxTxnRow int - tidbTxnMode string - changefeedID string - captureAddr string - batchReplaceEnabled bool - batchReplaceSize int - readTimeout string - writeTimeout string - dialTimeout string - enableOldValue bool - safeMode bool - timezone string - tls string -} - -func (s *sinkParams) Clone() *sinkParams { - clone := *s - return &clone -} - -func parseSinkURIToParams(ctx context.Context, sinkURI *url.URL, opts map[string]string) (*sinkParams, error) { - params := defaultParams.Clone() - - if cid, ok := opts[OptChangefeedID]; ok { - params.changefeedID = cid - } - if caddr, ok := opts[OptCaptureAddr]; ok { - params.captureAddr = caddr - } - - if sinkURI == nil { - return nil, cerror.ErrMySQLConnectionError.GenWithStack("fail to open MySQL sink, empty URL") - } - scheme := strings.ToLower(sinkURI.Scheme) - if _, ok := validSchemes[scheme]; !ok { - return nil, cerror.ErrMySQLConnectionError.GenWithStack("can't create mysql sink with unsupported scheme: %s", scheme) - } - s := sinkURI.Query().Get("worker-count") - if s != "" { - c, err := strconv.Atoi(s) - if err != nil { - return nil, cerror.WrapError(cerror.ErrMySQLInvalidConfig, err) - } - if c > 0 { - params.workerCount = c - } - } - s = sinkURI.Query().Get("max-txn-row") - if s != "" { - c, err := strconv.Atoi(s) - if err != nil { - return nil, cerror.WrapError(cerror.ErrMySQLInvalidConfig, err) - } - params.maxTxnRow = c - } - s = sinkURI.Query().Get("tidb-txn-mode") - if s != "" { - if s == "pessimistic" || s == "optimistic" { - params.tidbTxnMode = s - } else { - log.Warn("invalid tidb-txn-mode, should be pessimistic or optimistic, use optimistic as default") - } - } - if sinkURI.Query().Get("ssl-ca") != "" { - credential := security.Credential{ - CAPath: sinkURI.Query().Get("ssl-ca"), - CertPath: sinkURI.Query().Get("ssl-cert"), - KeyPath: sinkURI.Query().Get("ssl-key"), - } - tlsCfg, err := credential.ToTLSConfig() - if err != nil { - return nil, errors.Trace(err) - } - name := "cdc_mysql_tls" + params.changefeedID - err = dmysql.RegisterTLSConfig(name, tlsCfg) - if err != nil { - return nil, cerror.ErrMySQLConnectionError.Wrap(err).GenWithStack("fail to open MySQL connection") - } - params.tls = "?tls=" + name - } - - s = sinkURI.Query().Get("batch-replace-enable") - if s != "" { - enable, err := strconv.ParseBool(s) - if err != nil { - return nil, cerror.WrapError(cerror.ErrMySQLInvalidConfig, err) - } - params.batchReplaceEnabled = enable - } - if params.batchReplaceEnabled && sinkURI.Query().Get("batch-replace-size") != "" { - size, err := strconv.Atoi(sinkURI.Query().Get("batch-replace-size")) - if err != nil { - return nil, cerror.WrapError(cerror.ErrMySQLInvalidConfig, err) - } - params.batchReplaceSize = size - } - - // TODO: force safe mode in startup phase - s = sinkURI.Query().Get("safe-mode") - if s != "" { - safeModeEnabled, err := strconv.ParseBool(s) - if err != nil { - return nil, cerror.WrapError(cerror.ErrMySQLInvalidConfig, err) - } - params.safeMode = safeModeEnabled - } - - if _, ok := sinkURI.Query()["time-zone"]; ok { - s = sinkURI.Query().Get("time-zone") - if s == "" { - params.timezone = "" - } else { - params.timezone = fmt.Sprintf(`"%s"`, s) - } - } else { - tz := util.TimezoneFromCtx(ctx) - params.timezone = fmt.Sprintf(`"%s"`, tz.String()) - } - - // read, write, and dial timeout for each individual connection, equals to - // readTimeout, writeTimeout, timeout in go mysql driver respectively. - // ref: https://github.com/go-sql-driver/mysql#connection-pool-and-timeouts - // To keep the same style with other sink parameters, we use dash as word separator. - s = sinkURI.Query().Get("read-timeout") - if s != "" { - params.readTimeout = s - } - s = sinkURI.Query().Get("write-timeout") - if s != "" { - params.writeTimeout = s - } - s = sinkURI.Query().Get("timeout") - if s != "" { - params.dialTimeout = s - } - - return params, nil -} - -func generateDSNByParams( - ctx context.Context, - dsnCfg *dmysql.Config, - params *sinkParams, - testDB *sql.DB, -) (string, error) { - if dsnCfg.Params == nil { - dsnCfg.Params = make(map[string]string, 1) - } - dsnCfg.DBName = "" - dsnCfg.InterpolateParams = true - dsnCfg.MultiStatements = true - // if timezone is empty string, we don't pass this variable in dsn - if params.timezone != "" { - dsnCfg.Params["time_zone"] = params.timezone - } - dsnCfg.Params["readTimeout"] = params.readTimeout - dsnCfg.Params["writeTimeout"] = params.writeTimeout - dsnCfg.Params["timeout"] = params.dialTimeout - - autoRandom, err := checkTiDBVariable(ctx, testDB, "allow_auto_random_explicit_insert", "1") - if err != nil { - return "", err - } - if autoRandom != "" { - dsnCfg.Params["allow_auto_random_explicit_insert"] = autoRandom - } - - txnMode, err := checkTiDBVariable(ctx, testDB, "tidb_txn_mode", params.tidbTxnMode) - if err != nil { - return "", err - } - if txnMode != "" { - dsnCfg.Params["tidb_txn_mode"] = txnMode - } - - dsnClone := dsnCfg.Clone() - dsnClone.Passwd = "******" - log.Info("sink uri is configured", zap.String("format dsn", dsnClone.FormatDSN())) - - return dsnCfg.FormatDSN(), nil -} - -func checkTiDBVariable(ctx context.Context, db *sql.DB, variableName, defaultValue string) (string, error) { - var name string - var value string - querySQL := fmt.Sprintf("show session variables like '%s';", variableName) - err := db.QueryRowContext(ctx, querySQL).Scan(&name, &value) - if err != nil && err != sql.ErrNoRows { - errMsg := "fail to query session variable " + variableName - return "", cerror.ErrMySQLQueryError.Wrap(err).GenWithStack(errMsg) - } - // session variable works, use given default value - if err == nil { - return defaultValue, nil - } - // session variable not exists, return "" to ignore it - return "", nil -} diff --git a/cdc/cdc/sink/mysql_params_test.go b/cdc/cdc/sink/mysql_params_test.go deleted file mode 100644 index 5b9bba53..00000000 --- a/cdc/cdc/sink/mysql_params_test.go +++ /dev/null @@ -1,228 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package sink - -import ( - "context" - "database/sql" - "net/url" - "strings" - "testing" - - "github.com/DATA-DOG/go-sqlmock" - dmysql "github.com/go-sql-driver/mysql" - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/pkg/util/testleak" -) - -func TestSinkParamsClone(t *testing.T) { - defer testleak.AfterTestT(t)() - param1 := defaultParams.Clone() - param2 := param1.Clone() - param2.changefeedID = "123" - param2.batchReplaceEnabled = false - param2.maxTxnRow = 1 - require.Equal(t, &sinkParams{ - workerCount: DefaultWorkerCount, - maxTxnRow: DefaultMaxTxnRow, - tidbTxnMode: defaultTiDBTxnMode, - batchReplaceEnabled: defaultBatchReplaceEnabled, - batchReplaceSize: defaultBatchReplaceSize, - readTimeout: defaultReadTimeout, - writeTimeout: defaultWriteTimeout, - dialTimeout: defaultDialTimeout, - safeMode: defaultSafeMode, - }, param1) - require.Equal(t, &sinkParams{ - changefeedID: "123", - workerCount: DefaultWorkerCount, - maxTxnRow: 1, - tidbTxnMode: defaultTiDBTxnMode, - batchReplaceEnabled: false, - batchReplaceSize: defaultBatchReplaceSize, - readTimeout: defaultReadTimeout, - writeTimeout: defaultWriteTimeout, - dialTimeout: defaultDialTimeout, - safeMode: defaultSafeMode, - }, param2) -} - -func TestGenerateDSNByParams(t *testing.T) { - defer testleak.AfterTestT(t)() - - testDefaultParams := func() { - db, err := mockTestDB() - require.Nil(t, err) - defer db.Close() - - dsn, err := dmysql.ParseDSN("root:123456@tcp(127.0.0.1:4000)/") - require.Nil(t, err) - params := defaultParams.Clone() - dsnStr, err := generateDSNByParams(context.TODO(), dsn, params, db) - require.Nil(t, err) - expectedParams := []string{ - "tidb_txn_mode=optimistic", - "readTimeout=2m", - "writeTimeout=2m", - "allow_auto_random_explicit_insert=1", - } - for _, param := range expectedParams { - require.True(t, strings.Contains(dsnStr, param)) - } - require.False(t, strings.Contains(dsnStr, "time_zone")) - } - - testTimezoneParam := func() { - db, err := mockTestDB() - require.Nil(t, err) - defer db.Close() - - dsn, err := dmysql.ParseDSN("root:123456@tcp(127.0.0.1:4000)/") - require.Nil(t, err) - params := defaultParams.Clone() - params.timezone = `"UTC"` - dsnStr, err := generateDSNByParams(context.TODO(), dsn, params, db) - require.Nil(t, err) - require.True(t, strings.Contains(dsnStr, "time_zone=%22UTC%22")) - } - - testTimeoutParams := func() { - db, err := mockTestDB() - require.Nil(t, err) - defer db.Close() - - dsn, err := dmysql.ParseDSN("root:123456@tcp(127.0.0.1:4000)/") - require.Nil(t, err) - uri, err := url.Parse("mysql://127.0.0.1:3306/?read-timeout=4m&write-timeout=5m&timeout=3m") - require.Nil(t, err) - params, err := parseSinkURIToParams(context.TODO(), uri, map[string]string{}) - require.Nil(t, err) - dsnStr, err := generateDSNByParams(context.TODO(), dsn, params, db) - require.Nil(t, err) - expectedParams := []string{ - "readTimeout=4m", - "writeTimeout=5m", - "timeout=3m", - } - for _, param := range expectedParams { - require.True(t, strings.Contains(dsnStr, param)) - } - } - - testDefaultParams() - testTimezoneParam() - testTimeoutParams() -} - -func TestParseSinkURIToParams(t *testing.T) { - defer testleak.AfterTestT(t)() - expected := defaultParams.Clone() - expected.workerCount = 64 - expected.maxTxnRow = 20 - expected.batchReplaceEnabled = true - expected.batchReplaceSize = 50 - expected.safeMode = true - expected.timezone = `"UTC"` - expected.changefeedID = "cf-id" - expected.captureAddr = "127.0.0.1:8300" - expected.tidbTxnMode = "pessimistic" - uriStr := "mysql://127.0.0.1:3306/?worker-count=64&max-txn-row=20" + - "&batch-replace-enable=true&batch-replace-size=50&safe-mode=true" + - "&tidb-txn-mode=pessimistic" - opts := map[string]string{ - OptChangefeedID: expected.changefeedID, - OptCaptureAddr: expected.captureAddr, - } - uri, err := url.Parse(uriStr) - require.Nil(t, err) - params, err := parseSinkURIToParams(context.TODO(), uri, opts) - require.Nil(t, err) - require.Equal(t, expected, params) -} - -func TestParseSinkURITimezone(t *testing.T) { - defer testleak.AfterTestT(t)() - uris := []string{ - "mysql://127.0.0.1:3306/?time-zone=Asia/Shanghai&worker-count=32", - "mysql://127.0.0.1:3306/?time-zone=&worker-count=32", - "mysql://127.0.0.1:3306/?worker-count=32", - } - expected := []string{ - "\"Asia/Shanghai\"", - "", - "\"UTC\"", - } - ctx := context.TODO() - opts := map[string]string{} - for i, uriStr := range uris { - uri, err := url.Parse(uriStr) - require.Nil(t, err) - params, err := parseSinkURIToParams(ctx, uri, opts) - require.Nil(t, err) - require.Equal(t, expected[i], params.timezone) - } -} - -func TestParseSinkURIBadQueryString(t *testing.T) { - defer testleak.AfterTestT(t)() - uris := []string{ - "", - "postgre://127.0.0.1:3306", - "mysql://127.0.0.1:3306/?worker-count=not-number", - "mysql://127.0.0.1:3306/?max-txn-row=not-number", - "mysql://127.0.0.1:3306/?ssl-ca=only-ca-exists", - "mysql://127.0.0.1:3306/?batch-replace-enable=not-bool", - "mysql://127.0.0.1:3306/?batch-replace-enable=true&batch-replace-size=not-number", - "mysql://127.0.0.1:3306/?safe-mode=not-bool", - } - ctx := context.TODO() - opts := map[string]string{OptChangefeedID: "changefeed-01"} - var uri *url.URL - var err error - for _, uriStr := range uris { - if uriStr != "" { - uri, err = url.Parse(uriStr) - require.Nil(t, err) - } else { - uri = nil - } - _, err = parseSinkURIToParams(ctx, uri, opts) - require.NotNil(t, err) - } -} - -func TestCheckTiDBVariable(t *testing.T) { - defer testleak.AfterTestT(t)() - db, mock, err := sqlmock.New() - require.Nil(t, err) - defer db.Close() //nolint:errcheck - columns := []string{"Variable_name", "Value"} - - mock.ExpectQuery("show session variables like 'allow_auto_random_explicit_insert';").WillReturnRows( - sqlmock.NewRows(columns).AddRow("allow_auto_random_explicit_insert", "0"), - ) - val, err := checkTiDBVariable(context.TODO(), db, "allow_auto_random_explicit_insert", "1") - require.Nil(t, err) - require.Equal(t, "1", val) - - mock.ExpectQuery("show session variables like 'no_exist_variable';").WillReturnError(sql.ErrNoRows) - val, err = checkTiDBVariable(context.TODO(), db, "no_exist_variable", "0") - require.Nil(t, err) - require.Equal(t, "", val) - - mock.ExpectQuery("show session variables like 'version';").WillReturnError(sql.ErrConnDone) - _, err = checkTiDBVariable(context.TODO(), db, "version", "5.7.25-TiDB-v4.0.0") - require.NotNil(t, err) - require.Regexp(t, ".*"+sql.ErrConnDone.Error(), err.Error()) -} diff --git a/cdc/cdc/sink/mysql_syncpoint_store.go b/cdc/cdc/sink/mysql_syncpoint_store.go deleted file mode 100644 index 8524087f..00000000 --- a/cdc/cdc/sink/mysql_syncpoint_store.go +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package sink - -import ( - "context" - "database/sql" - "fmt" - "net/url" - "strings" - - dmysql "github.com/go-sql-driver/mysql" - "github.com/pingcap/errors" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/pkg/cyclic/mark" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/security" - "github.com/tikv/migration/cdc/pkg/util" - "go.uber.org/zap" -) - -// SyncpointTableName is the name of table where all syncpoint maps sit -const syncpointTableName string = "syncpoint_v1" - -type mysqlSyncpointStore struct { - db *sql.DB -} - -// newSyncpointStore create a sink to record the syncpoint map in downstream DB for every changefeed -func newMySQLSyncpointStore(ctx context.Context, id string, sinkURI *url.URL) (SyncpointStore, error) { - var syncDB *sql.DB - - // todo If is neither mysql nor tidb, such as kafka, just ignore this feature. - scheme := strings.ToLower(sinkURI.Scheme) - if scheme != "mysql" && scheme != "tidb" && scheme != "mysql+ssl" && scheme != "tidb+ssl" { - return nil, errors.New("can create mysql sink with unsupported scheme") - } - params := defaultParams.Clone() - s := sinkURI.Query().Get("tidb-txn-mode") - if s != "" { - if s == "pessimistic" || s == "optimistic" { - params.tidbTxnMode = s - } else { - log.Warn("invalid tidb-txn-mode, should be pessimistic or optimistic, use optimistic as default") - } - } - var tlsParam string - if sinkURI.Query().Get("ssl-ca") != "" { - credential := security.Credential{ - CAPath: sinkURI.Query().Get("ssl-ca"), - CertPath: sinkURI.Query().Get("ssl-cert"), - KeyPath: sinkURI.Query().Get("ssl-key"), - } - tlsCfg, err := credential.ToTLSConfig() - if err != nil { - return nil, cerror.ErrMySQLConnectionError.Wrap(err).GenWithStack("fail to open MySQL connection") - } - name := "cdc_mysql_tls" + "syncpoint" + id - err = dmysql.RegisterTLSConfig(name, tlsCfg) - if err != nil { - return nil, cerror.ErrMySQLConnectionError.Wrap(err).GenWithStack("fail to open MySQL connection") - } - tlsParam = "?tls=" + name - } - if _, ok := sinkURI.Query()["time-zone"]; ok { - s = sinkURI.Query().Get("time-zone") - if s == "" { - params.timezone = "" - } else { - params.timezone = fmt.Sprintf(`"%s"`, s) - } - } else { - tz := util.TimezoneFromCtx(ctx) - params.timezone = fmt.Sprintf(`"%s"`, tz.String()) - } - - // dsn format of the driver: - // [username[:password]@][protocol[(address)]]/dbname[?param1=value1&...¶mN=valueN] - username := sinkURI.User.Username() - password, _ := sinkURI.User.Password() - port := sinkURI.Port() - if username == "" { - username = "root" - } - if port == "" { - port = "4000" - } - - dsnStr := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s", username, password, sinkURI.Hostname(), port, tlsParam) - dsn, err := dmysql.ParseDSN(dsnStr) - if err != nil { - return nil, errors.Trace(err) - } - - // create test db used for parameter detection - if dsn.Params == nil { - dsn.Params = make(map[string]string, 1) - } - testDB, err := sql.Open("mysql", dsn.FormatDSN()) - if err != nil { - return nil, cerror.ErrMySQLConnectionError.Wrap(err).GenWithStack("fail to open MySQL connection when configuring sink") - } - defer testDB.Close() - dsnStr, err = generateDSNByParams(ctx, dsn, params, testDB) - if err != nil { - return nil, errors.Trace(err) - } - syncDB, err = sql.Open("mysql", dsnStr) - if err != nil { - return nil, cerror.ErrMySQLConnectionError.Wrap(err).GenWithStack("fail to open MySQL connection") - } - err = syncDB.PingContext(ctx) - if err != nil { - return nil, cerror.ErrMySQLConnectionError.Wrap(err).GenWithStack("fail to open MySQL connection") - } - - log.Info("Start mysql syncpoint sink") - syncpointStore := &mysqlSyncpointStore{ - db: syncDB, - } - - return syncpointStore, nil -} - -func (s *mysqlSyncpointStore) CreateSynctable(ctx context.Context) error { - database := mark.SchemaName - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - log.Error("create sync table: begin Tx fail", zap.Error(err)) - return cerror.WrapError(cerror.ErrMySQLTxnError, err) - } - _, err = tx.Exec("CREATE DATABASE IF NOT EXISTS " + database) - if err != nil { - err2 := tx.Rollback() - if err2 != nil { - log.Error("failed to create syncpoint table", zap.Error(cerror.WrapError(cerror.ErrMySQLTxnError, err2))) - } - return cerror.WrapError(cerror.ErrMySQLTxnError, err) - } - _, err = tx.Exec("USE " + database) - if err != nil { - err2 := tx.Rollback() - if err2 != nil { - log.Error("failed to create syncpoint table", zap.Error(cerror.WrapError(cerror.ErrMySQLTxnError, err2))) - } - return cerror.WrapError(cerror.ErrMySQLTxnError, err) - } - _, err = tx.Exec("CREATE TABLE IF NOT EXISTS " + syncpointTableName + " (cf varchar(255),primary_ts varchar(18),secondary_ts varchar(18),PRIMARY KEY ( `cf`, `primary_ts` ) )") - if err != nil { - err2 := tx.Rollback() - if err2 != nil { - log.Error("failed to create syncpoint table", zap.Error(cerror.WrapError(cerror.ErrMySQLTxnError, err2))) - } - return cerror.WrapError(cerror.ErrMySQLTxnError, err) - } - err = tx.Commit() - return cerror.WrapError(cerror.ErrMySQLTxnError, err) -} - -func (s *mysqlSyncpointStore) SinkSyncpoint(ctx context.Context, id string, checkpointTs uint64) error { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - log.Error("sync table: begin Tx fail", zap.Error(err)) - return cerror.WrapError(cerror.ErrMySQLTxnError, err) - } - row := tx.QueryRow("select @@tidb_current_ts") - var secondaryTs string - err = row.Scan(&secondaryTs) - if err != nil { - log.Info("sync table: get tidb_current_ts err") - err2 := tx.Rollback() - if err2 != nil { - log.Error("failed to write syncpoint table", zap.Error(cerror.WrapError(cerror.ErrMySQLTxnError, err2))) - } - return cerror.WrapError(cerror.ErrMySQLTxnError, err) - } - _, err = tx.Exec("insert ignore into "+mark.SchemaName+"."+syncpointTableName+"(cf, primary_ts, secondary_ts) VALUES (?,?,?)", id, checkpointTs, secondaryTs) - if err != nil { - err2 := tx.Rollback() - if err2 != nil { - log.Error("failed to write syncpoint table", zap.Error(cerror.WrapError(cerror.ErrMySQLTxnError, err2))) - } - return cerror.WrapError(cerror.ErrMySQLTxnError, err) - } - err = tx.Commit() - return cerror.WrapError(cerror.ErrMySQLTxnError, err) -} - -func (s *mysqlSyncpointStore) Close() error { - err := s.db.Close() - return cerror.WrapError(cerror.ErrMySQLConnectionError, err) -} diff --git a/cdc/cdc/sink/mysql_test.go b/cdc/cdc/sink/mysql_test.go deleted file mode 100644 index 724f922c..00000000 --- a/cdc/cdc/sink/mysql_test.go +++ /dev/null @@ -1,1185 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package sink - -import ( - "context" - "database/sql" - "database/sql/driver" - "fmt" - "net" - "net/url" - "sort" - "sync" - "testing" - "time" - - "github.com/DATA-DOG/go-sqlmock" - dmysql "github.com/go-sql-driver/mysql" - "github.com/pingcap/errors" - "github.com/pingcap/tidb/infoschema" - timodel "github.com/pingcap/tidb/parser/model" - "github.com/pingcap/tidb/parser/mysql" - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/sink/common" - "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/cyclic/mark" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/filter" - "github.com/tikv/migration/cdc/pkg/retry" -) - -func newMySQLSink4Test(ctx context.Context, t *testing.T) *mysqlSink { - f, err := filter.NewFilter(config.GetDefaultReplicaConfig()) - require.Nil(t, err) - params := defaultParams.Clone() - params.batchReplaceEnabled = false - return &mysqlSink{ - txnCache: common.NewUnresolvedTxnCache(), - filter: f, - statistics: NewStatistics(ctx, "test", make(map[string]string)), - params: params, - } -} - -func TestPrepareDML(t *testing.T) { - testCases := []struct { - input []*model.RowChangedEvent - expected *preparedDMLs - }{{ - input: []*model.RowChangedEvent{}, - expected: &preparedDMLs{sqls: []string{}, values: [][]interface{}{}}, - }, { - input: []*model.RowChangedEvent{ - { - StartTs: 418658114257813514, - CommitTs: 418658114257813515, - Table: &model.TableName{Schema: "common_1", Table: "uk_without_pk"}, - PreColumns: []*model.Column{nil, { - Name: "a1", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.MultipleKeyFlag | model.HandleKeyFlag, - Value: 1, - }, { - Name: "a3", - Type: mysql.TypeLong, - Flag: model.BinaryFlag | model.MultipleKeyFlag | model.HandleKeyFlag, - Value: 1, - }}, - IndexColumns: [][]int{{1, 2}}, - }, - }, - expected: &preparedDMLs{ - sqls: []string{"DELETE FROM `common_1`.`uk_without_pk` WHERE `a1` = ? AND `a3` = ? LIMIT 1;"}, - values: [][]interface{}{{1, 1}}, - rowCount: 1, - }, - }} - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ms := newMySQLSink4Test(ctx, t) - for i, tc := range testCases { - dmls := ms.prepareDMLs(tc.input, 0, 0) - require.Equal(t, tc.expected, dmls, tc.expected, fmt.Sprintf("%d", i)) - } -} - -func TestPrepareUpdate(t *testing.T) { - testCases := []struct { - quoteTable string - preCols []*model.Column - cols []*model.Column - expectedSQL string - expectedArgs []interface{} - }{ - { - quoteTable: "`test`.`t1`", - preCols: []*model.Column{}, - cols: []*model.Column{}, - expectedSQL: "", - expectedArgs: nil, - }, - { - quoteTable: "`test`.`t1`", - preCols: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, Value: 1}, - {Name: "b", Type: mysql.TypeVarchar, Flag: 0, Value: "test"}, - }, - cols: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, Value: 1}, - {Name: "b", Type: mysql.TypeVarchar, Flag: 0, Value: "test2"}, - }, - expectedSQL: "UPDATE `test`.`t1` SET `a`=?,`b`=? WHERE `a`=? LIMIT 1;", - expectedArgs: []interface{}{1, "test2", 1}, - }, - { - quoteTable: "`test`.`t1`", - preCols: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.MultipleKeyFlag | model.HandleKeyFlag, Value: 1}, - {Name: "b", Type: mysql.TypeVarString, Flag: model.MultipleKeyFlag | model.HandleKeyFlag, Value: "test"}, - {Name: "c", Type: mysql.TypeLong, Flag: model.GeneratedColumnFlag, Value: 100}, - }, - cols: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.MultipleKeyFlag | model.HandleKeyFlag, Value: 2}, - {Name: "b", Type: mysql.TypeVarString, Flag: model.MultipleKeyFlag | model.HandleKeyFlag, Value: "test2"}, - {Name: "c", Type: mysql.TypeLong, Flag: model.GeneratedColumnFlag, Value: 100}, - }, - expectedSQL: "UPDATE `test`.`t1` SET `a`=?,`b`=? WHERE `a`=? AND `b`=? LIMIT 1;", - expectedArgs: []interface{}{2, "test2", 1, "test"}, - }, - } - for _, tc := range testCases { - query, args := prepareUpdate(tc.quoteTable, tc.preCols, tc.cols, false) - require.Equal(t, tc.expectedSQL, query) - require.Equal(t, tc.expectedArgs, args) - } -} - -func TestPrepareDelete(t *testing.T) { - testCases := []struct { - quoteTable string - preCols []*model.Column - expectedSQL string - expectedArgs []interface{} - }{ - { - quoteTable: "`test`.`t1`", - preCols: []*model.Column{}, - expectedSQL: "", - expectedArgs: nil, - }, - { - quoteTable: "`test`.`t1`", - preCols: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, Value: 1}, - {Name: "b", Type: mysql.TypeVarchar, Flag: 0, Value: "test"}, - }, - expectedSQL: "DELETE FROM `test`.`t1` WHERE `a` = ? LIMIT 1;", - expectedArgs: []interface{}{1}, - }, - { - quoteTable: "`test`.`t1`", - preCols: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.MultipleKeyFlag | model.HandleKeyFlag, Value: 1}, - {Name: "b", Type: mysql.TypeVarString, Flag: model.MultipleKeyFlag | model.HandleKeyFlag, Value: "test"}, - {Name: "c", Type: mysql.TypeLong, Flag: model.GeneratedColumnFlag, Value: 100}, - }, - expectedSQL: "DELETE FROM `test`.`t1` WHERE `a` = ? AND `b` = ? LIMIT 1;", - expectedArgs: []interface{}{1, "test"}, - }, - } - for _, tc := range testCases { - query, args := prepareDelete(tc.quoteTable, tc.preCols, false) - require.Equal(t, tc.expectedSQL, query) - require.Equal(t, tc.expectedArgs, args) - } -} - -func TestWhereSlice(t *testing.T) { - testCases := []struct { - cols []*model.Column - forceReplicate bool - expectedColNames []string - expectedArgs []interface{} - }{ - { - cols: []*model.Column{}, - forceReplicate: false, - expectedColNames: nil, - expectedArgs: nil, - }, - { - cols: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, Value: 1}, - {Name: "b", Type: mysql.TypeVarchar, Flag: 0, Value: "test"}, - }, - forceReplicate: false, - expectedColNames: []string{"a"}, - expectedArgs: []interface{}{1}, - }, - { - cols: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.MultipleKeyFlag | model.HandleKeyFlag, Value: 1}, - {Name: "b", Type: mysql.TypeVarString, Flag: model.MultipleKeyFlag | model.HandleKeyFlag, Value: "test"}, - {Name: "c", Type: mysql.TypeLong, Flag: model.GeneratedColumnFlag, Value: 100}, - }, - forceReplicate: false, - expectedColNames: []string{"a", "b"}, - expectedArgs: []interface{}{1, "test"}, - }, - { - cols: []*model.Column{}, - forceReplicate: true, - expectedColNames: []string{}, - expectedArgs: []interface{}{}, - }, - { - cols: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, Value: 1}, - {Name: "b", Type: mysql.TypeVarchar, Flag: 0, Value: "test"}, - }, - forceReplicate: true, - expectedColNames: []string{"a"}, - expectedArgs: []interface{}{1}, - }, - { - cols: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.MultipleKeyFlag | model.HandleKeyFlag, Value: 1}, - {Name: "b", Type: mysql.TypeVarString, Flag: model.MultipleKeyFlag | model.HandleKeyFlag, Value: "test"}, - {Name: "c", Type: mysql.TypeLong, Flag: model.GeneratedColumnFlag, Value: 100}, - }, - forceReplicate: true, - expectedColNames: []string{"a", "b"}, - expectedArgs: []interface{}{1, "test"}, - }, - { - cols: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.UniqueKeyFlag, Value: 1}, - {Name: "b", Type: mysql.TypeVarchar, Flag: 0, Value: "test"}, - }, - forceReplicate: true, - expectedColNames: []string{"a", "b"}, - expectedArgs: []interface{}{1, "test"}, - }, - { - cols: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.MultipleKeyFlag, Value: 1}, - {Name: "b", Type: mysql.TypeVarString, Flag: model.MultipleKeyFlag, Value: "test"}, - {Name: "c", Type: mysql.TypeLong, Flag: model.GeneratedColumnFlag, Value: 100}, - }, - forceReplicate: true, - expectedColNames: []string{"a", "b", "c"}, - expectedArgs: []interface{}{1, "test", 100}, - }, - } - for _, tc := range testCases { - colNames, args := whereSlice(tc.cols, tc.forceReplicate) - require.Equal(t, tc.expectedColNames, colNames) - require.Equal(t, tc.expectedArgs, args) - } -} - -func TestMapReplace(t *testing.T) { - testCases := []struct { - quoteTable string - cols []*model.Column - expectedQuery string - expectedArgs []interface{} - }{ - { - quoteTable: "`test`.`t1`", - cols: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Value: 1}, - {Name: "b", Type: mysql.TypeVarchar, Value: "varchar"}, - {Name: "c", Type: mysql.TypeLong, Value: 1, Flag: model.GeneratedColumnFlag}, - {Name: "d", Type: mysql.TypeTiny, Value: uint8(255)}, - }, - expectedQuery: "REPLACE INTO `test`.`t1`(`a`,`b`,`d`) VALUES ", - expectedArgs: []interface{}{1, "varchar", uint8(255)}, - }, - { - quoteTable: "`test`.`t1`", - cols: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Value: 1}, - {Name: "b", Type: mysql.TypeVarchar, Value: "varchar"}, - {Name: "c", Type: mysql.TypeLong, Value: 1}, - {Name: "d", Type: mysql.TypeTiny, Value: uint8(255)}, - }, - expectedQuery: "REPLACE INTO `test`.`t1`(`a`,`b`,`c`,`d`) VALUES ", - expectedArgs: []interface{}{1, "varchar", 1, uint8(255)}, - }, - } - for _, tc := range testCases { - // multiple times to verify the stability of column sequence in query string - for i := 0; i < 10; i++ { - query, args := prepareReplace(tc.quoteTable, tc.cols, false, false) - require.Equal(t, tc.expectedQuery, query) - require.Equal(t, tc.expectedArgs, args) - } - } -} - -type sqlArgs [][]interface{} - -func (a sqlArgs) Len() int { return len(a) } -func (a sqlArgs) Less(i, j int) bool { return fmt.Sprintf("%s", a[i]) < fmt.Sprintf("%s", a[j]) } -func (a sqlArgs) Swap(i, j int) { a[i], a[j] = a[j], a[i] } - -func TestReduceReplace(t *testing.T) { - testCases := []struct { - replaces map[string][][]interface{} - batchSize int - sort bool - expectSQLs []string - expectArgs [][]interface{} - }{ - { - replaces: map[string][][]interface{}{ - "REPLACE INTO `test`.`t1`(`a`,`b`) VALUES ": { - []interface{}{1, "1"}, - []interface{}{2, "2"}, - []interface{}{3, "3"}, - }, - }, - batchSize: 1, - sort: false, - expectSQLs: []string{ - "REPLACE INTO `test`.`t1`(`a`,`b`) VALUES (?,?)", - "REPLACE INTO `test`.`t1`(`a`,`b`) VALUES (?,?)", - "REPLACE INTO `test`.`t1`(`a`,`b`) VALUES (?,?)", - }, - expectArgs: [][]interface{}{ - {1, "1"}, - {2, "2"}, - {3, "3"}, - }, - }, - { - replaces: map[string][][]interface{}{ - "REPLACE INTO `test`.`t1`(`a`,`b`) VALUES ": { - []interface{}{1, "1"}, - []interface{}{2, "2"}, - []interface{}{3, "3"}, - []interface{}{4, "3"}, - []interface{}{5, "5"}, - }, - }, - batchSize: 3, - sort: false, - expectSQLs: []string{ - "REPLACE INTO `test`.`t1`(`a`,`b`) VALUES (?,?),(?,?),(?,?)", - "REPLACE INTO `test`.`t1`(`a`,`b`) VALUES (?,?),(?,?)", - }, - expectArgs: [][]interface{}{ - {1, "1", 2, "2", 3, "3"}, - {4, "3", 5, "5"}, - }, - }, - { - replaces: map[string][][]interface{}{ - "REPLACE INTO `test`.`t1`(`a`,`b`) VALUES ": { - []interface{}{1, "1"}, - []interface{}{2, "2"}, - []interface{}{3, "3"}, - []interface{}{4, "3"}, - []interface{}{5, "5"}, - }, - }, - batchSize: 10, - sort: false, - expectSQLs: []string{ - "REPLACE INTO `test`.`t1`(`a`,`b`) VALUES (?,?),(?,?),(?,?),(?,?),(?,?)", - }, - expectArgs: [][]interface{}{ - {1, "1", 2, "2", 3, "3", 4, "3", 5, "5"}, - }, - }, - { - replaces: map[string][][]interface{}{ - "REPLACE INTO `test`.`t1`(`a`,`b`) VALUES ": { - []interface{}{1, "1"}, - []interface{}{2, "2"}, - []interface{}{3, "3"}, - []interface{}{4, "3"}, - []interface{}{5, "5"}, - []interface{}{6, "6"}, - }, - "REPLACE INTO `test`.`t2`(`a`,`b`) VALUES ": { - []interface{}{7, ""}, - []interface{}{8, ""}, - []interface{}{9, ""}, - }, - }, - batchSize: 3, - sort: true, - expectSQLs: []string{ - "REPLACE INTO `test`.`t1`(`a`,`b`) VALUES (?,?),(?,?),(?,?)", - "REPLACE INTO `test`.`t1`(`a`,`b`) VALUES (?,?),(?,?),(?,?)", - "REPLACE INTO `test`.`t2`(`a`,`b`) VALUES (?,?),(?,?),(?,?)", - }, - expectArgs: [][]interface{}{ - {1, "1", 2, "2", 3, "3"}, - {4, "3", 5, "5", 6, "6"}, - {7, "", 8, "", 9, ""}, - }, - }, - } - for _, tc := range testCases { - sqls, args := reduceReplace(tc.replaces, tc.batchSize) - if tc.sort { - sort.Strings(sqls) - sort.Sort(sqlArgs(args)) - } - require.Equal(t, tc.expectSQLs, sqls) - require.Equal(t, tc.expectArgs, args) - } -} - -func mockTestDB() (*sql.DB, error) { - // mock for test db, which is used querying TiDB session variable - db, mock, err := sqlmock.New() - if err != nil { - return nil, err - } - columns := []string{"Variable_name", "Value"} - mock.ExpectQuery("show session variables like 'allow_auto_random_explicit_insert';").WillReturnRows( - sqlmock.NewRows(columns).AddRow("allow_auto_random_explicit_insert", "0"), - ) - mock.ExpectQuery("show session variables like 'tidb_txn_mode';").WillReturnRows( - sqlmock.NewRows(columns).AddRow("tidb_txn_mode", "pessimistic"), - ) - mock.ExpectClose() - return db, nil -} - -func TestAdjustSQLMode(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - dbIndex := 0 - mockGetDBConn := func(ctx context.Context, dsnStr string) (*sql.DB, error) { - defer func() { - dbIndex++ - }() - if dbIndex == 0 { - // test db - db, err := mockTestDB() - require.Nil(t, err) - return db, nil - } - // normal db - db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) - require.Nil(t, err) - mock.ExpectQuery("SELECT @@SESSION.sql_mode;"). - WillReturnRows(sqlmock.NewRows([]string{"@@SESSION.sql_mode"}). - AddRow("ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE")) - mock.ExpectExec("SET sql_mode = 'ONLY_FULL_GROUP_BY,NO_ZERO_IN_DATE,NO_ZERO_DATE';"). - WillReturnResult(sqlmock.NewResult(0, 0)) - mock.ExpectClose() - return db, nil - } - backupGetDBConn := GetDBConnImpl - GetDBConnImpl = mockGetDBConn - defer func() { - GetDBConnImpl = backupGetDBConn - }() - - changefeed := "test-changefeed" - sinkURI, err := url.Parse("mysql://127.0.0.1:4000/?time-zone=UTC&worker-count=4") - require.Nil(t, err) - require.Nil(t, err) - rc := config.GetDefaultReplicaConfig() - rc.Cyclic = &config.CyclicConfig{ - Enable: true, - ReplicaID: 1, - FilterReplicaID: []uint64{2}, - } - f, err := filter.NewFilter(rc) - require.Nil(t, err) - cyclicConfig, err := rc.Cyclic.Marshal() - require.Nil(t, err) - opts := map[string]string{ - mark.OptCyclicConfig: cyclicConfig, - } - sink, err := newMySQLSink(ctx, changefeed, sinkURI, f, rc, opts) - require.Nil(t, err) - - err = sink.Close(ctx) - require.Nil(t, err) -} - -type mockUnavailableMySQL struct { - listener net.Listener - quit chan interface{} - wg sync.WaitGroup -} - -func newMockUnavailableMySQL(addr string, t *testing.T) *mockUnavailableMySQL { - s := &mockUnavailableMySQL{ - quit: make(chan interface{}), - } - l, err := net.Listen("tcp", addr) - require.Nil(t, err) - s.listener = l - s.wg.Add(1) - go s.serve(t) - return s -} - -func (s *mockUnavailableMySQL) serve(t *testing.T) { - defer s.wg.Done() - - for { - _, err := s.listener.Accept() - if err != nil { - select { - case <-s.quit: - return - default: - require.Error(t, err) - } - } else { - s.wg.Add(1) - go func() { - // don't read from TCP connection, to simulate database service unavailable - <-s.quit - s.wg.Done() - }() - } - } -} - -func (s *mockUnavailableMySQL) Stop() { - close(s.quit) - s.listener.Close() - s.wg.Wait() -} - -func TestNewMySQLTimeout(t *testing.T) { - addr := "127.0.0.1:33333" - mockMySQL := newMockUnavailableMySQL(addr, t) - defer mockMySQL.Stop() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - changefeed := "test-changefeed" - sinkURI, err := url.Parse(fmt.Sprintf("mysql://%s/?read-timeout=2s&timeout=2s", addr)) - require.Nil(t, err) - rc := config.GetDefaultReplicaConfig() - f, err := filter.NewFilter(rc) - require.Nil(t, err) - _, err = newMySQLSink(ctx, changefeed, sinkURI, f, rc, map[string]string{}) - require.Equal(t, driver.ErrBadConn, errors.Cause(err)) -} - -func TestNewMySQLSinkExecDML(t *testing.T) { - dbIndex := 0 - mockGetDBConn := func(ctx context.Context, dsnStr string) (*sql.DB, error) { - defer func() { - dbIndex++ - }() - if dbIndex == 0 { - // test db - db, err := mockTestDB() - require.Nil(t, err) - return db, nil - } - // normal db - db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) - require.Nil(t, err) - mock.ExpectBegin() - mock.ExpectExec("REPLACE INTO `s1`.`t1`(`a`,`b`) VALUES (?,?),(?,?)"). - WithArgs(1, "test", 2, "test"). - WillReturnResult(sqlmock.NewResult(2, 2)) - mock.ExpectCommit() - mock.ExpectBegin() - mock.ExpectExec("REPLACE INTO `s1`.`t2`(`a`,`b`) VALUES (?,?),(?,?)"). - WithArgs(1, "test", 2, "test"). - WillReturnResult(sqlmock.NewResult(2, 2)) - mock.ExpectCommit() - mock.ExpectClose() - return db, nil - } - backupGetDBConn := GetDBConnImpl - GetDBConnImpl = mockGetDBConn - defer func() { - GetDBConnImpl = backupGetDBConn - }() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - changefeed := "test-changefeed" - sinkURI, err := url.Parse("mysql://127.0.0.1:4000/?time-zone=UTC&worker-count=4") - require.Nil(t, err) - rc := config.GetDefaultReplicaConfig() - f, err := filter.NewFilter(rc) - require.Nil(t, err) - sink, err := newMySQLSink(ctx, changefeed, sinkURI, f, rc, map[string]string{}) - require.Nil(t, err) - - rows := []*model.RowChangedEvent{ - { - StartTs: 1, - CommitTs: 2, - Table: &model.TableName{Schema: "s1", Table: "t1", TableID: 1}, - Columns: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, Value: 1}, - {Name: "b", Type: mysql.TypeVarchar, Flag: 0, Value: "test"}, - }, - }, - { - StartTs: 1, - CommitTs: 2, - Table: &model.TableName{Schema: "s1", Table: "t1", TableID: 1}, - Columns: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, Value: 2}, - {Name: "b", Type: mysql.TypeVarchar, Flag: 0, Value: "test"}, - }, - }, - { - StartTs: 5, - CommitTs: 6, - Table: &model.TableName{Schema: "s1", Table: "t1", TableID: 1}, - Columns: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, Value: 3}, - {Name: "b", Type: mysql.TypeVarchar, Flag: 0, Value: "test"}, - }, - }, - { - StartTs: 3, - CommitTs: 4, - Table: &model.TableName{Schema: "s1", Table: "t2", TableID: 2}, - Columns: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, Value: 1}, - {Name: "b", Type: mysql.TypeVarchar, Flag: 0, Value: "test"}, - }, - }, - { - StartTs: 3, - CommitTs: 4, - Table: &model.TableName{Schema: "s1", Table: "t2", TableID: 2}, - Columns: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, Value: 2}, - {Name: "b", Type: mysql.TypeVarchar, Flag: 0, Value: "test"}, - }, - }, - } - - err = sink.EmitRowChangedEvents(ctx, rows...) - require.Nil(t, err) - - // retry to make sure event is flushed - err = retry.Do(context.Background(), func() error { - ts, err := sink.FlushRowChangedEvents(ctx, 1, uint64(2)) - require.Nil(t, err) - if ts < uint64(2) { - return errors.Errorf("checkpoint ts %d less than resolved ts %d", ts, 2) - } - return nil - }, retry.WithBackoffBaseDelay(20), retry.WithMaxTries(10), retry.WithIsRetryableErr(cerror.IsRetryableError)) - - require.Nil(t, err) - - err = retry.Do(context.Background(), func() error { - ts, err := sink.FlushRowChangedEvents(ctx, 2, uint64(4)) - require.Nil(t, err) - if ts < uint64(4) { - return errors.Errorf("checkpoint ts %d less than resolved ts %d", ts, 4) - } - return nil - }, retry.WithBackoffBaseDelay(20), retry.WithMaxTries(10), retry.WithIsRetryableErr(cerror.IsRetryableError)) - require.Nil(t, err) - - err = sink.Barrier(ctx, 2) - require.Nil(t, err) - - err = sink.Close(ctx) - require.Nil(t, err) -} - -func TestExecDMLRollbackErrDatabaseNotExists(t *testing.T) { - rows := []*model.RowChangedEvent{ - { - Table: &model.TableName{Schema: "s1", Table: "t1", TableID: 1}, - Columns: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, Value: 1}, - }, - }, - { - Table: &model.TableName{Schema: "s1", Table: "t1", TableID: 1}, - Columns: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, Value: 2}, - }, - }, - } - - errDatabaseNotExists := &dmysql.MySQLError{ - Number: uint16(infoschema.ErrDatabaseNotExists.Code()), - } - - dbIndex := 0 - mockGetDBConnErrDatabaseNotExists := func(ctx context.Context, dsnStr string) (*sql.DB, error) { - defer func() { - dbIndex++ - }() - if dbIndex == 0 { - // test db - db, err := mockTestDB() - require.Nil(t, err) - return db, nil - } - // normal db - db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) - require.Nil(t, err) - mock.ExpectBegin() - mock.ExpectExec("REPLACE INTO `s1`.`t1`(`a`) VALUES (?),(?)"). - WithArgs(1, 2). - WillReturnError(errDatabaseNotExists) - mock.ExpectRollback() - mock.ExpectClose() - return db, nil - } - backupGetDBConn := GetDBConnImpl - GetDBConnImpl = mockGetDBConnErrDatabaseNotExists - defer func() { - GetDBConnImpl = backupGetDBConn - }() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - changefeed := "test-changefeed" - sinkURI, err := url.Parse("mysql://127.0.0.1:4000/?time-zone=UTC&worker-count=1") - require.Nil(t, err) - rc := config.GetDefaultReplicaConfig() - f, err := filter.NewFilter(rc) - require.Nil(t, err) - sink, err := newMySQLSink(ctx, changefeed, sinkURI, f, rc, map[string]string{}) - require.Nil(t, err) - - err = sink.(*mysqlSink).execDMLs(ctx, rows, 1 /* replicaID */, 1 /* bucket */) - require.Equal(t, errDatabaseNotExists, errors.Cause(err)) - - err = sink.Close(ctx) - require.Nil(t, err) -} - -func TestExecDMLRollbackErrTableNotExists(t *testing.T) { - rows := []*model.RowChangedEvent{ - { - Table: &model.TableName{Schema: "s1", Table: "t1", TableID: 1}, - Columns: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, Value: 1}, - }, - }, - { - Table: &model.TableName{Schema: "s1", Table: "t1", TableID: 1}, - Columns: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, Value: 2}, - }, - }, - } - - errTableNotExists := &dmysql.MySQLError{ - Number: uint16(infoschema.ErrTableNotExists.Code()), - } - - dbIndex := 0 - mockGetDBConnErrDatabaseNotExists := func(ctx context.Context, dsnStr string) (*sql.DB, error) { - defer func() { - dbIndex++ - }() - if dbIndex == 0 { - // test db - db, err := mockTestDB() - require.Nil(t, err) - return db, nil - } - // normal db - db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) - require.Nil(t, err) - mock.ExpectBegin() - mock.ExpectExec("REPLACE INTO `s1`.`t1`(`a`) VALUES (?),(?)"). - WithArgs(1, 2). - WillReturnError(errTableNotExists) - mock.ExpectRollback() - mock.ExpectClose() - return db, nil - } - backupGetDBConn := GetDBConnImpl - GetDBConnImpl = mockGetDBConnErrDatabaseNotExists - defer func() { - GetDBConnImpl = backupGetDBConn - }() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - changefeed := "test-changefeed" - sinkURI, err := url.Parse("mysql://127.0.0.1:4000/?time-zone=UTC&worker-count=1") - require.Nil(t, err) - rc := config.GetDefaultReplicaConfig() - f, err := filter.NewFilter(rc) - require.Nil(t, err) - sink, err := newMySQLSink(ctx, changefeed, sinkURI, f, rc, map[string]string{}) - require.Nil(t, err) - - err = sink.(*mysqlSink).execDMLs(ctx, rows, 1 /* replicaID */, 1 /* bucket */) - require.Equal(t, errTableNotExists, errors.Cause(err)) - - err = sink.Close(ctx) - require.Nil(t, err) -} - -func TestExecDMLRollbackErrRetryable(t *testing.T) { - rows := []*model.RowChangedEvent{ - { - Table: &model.TableName{Schema: "s1", Table: "t1", TableID: 1}, - Columns: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, Value: 1}, - }, - }, - { - Table: &model.TableName{Schema: "s1", Table: "t1", TableID: 1}, - Columns: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, Value: 2}, - }, - }, - } - - errLockDeadlock := &dmysql.MySQLError{ - Number: mysql.ErrLockDeadlock, - } - - dbIndex := 0 - mockGetDBConnErrDatabaseNotExists := func(ctx context.Context, dsnStr string) (*sql.DB, error) { - defer func() { - dbIndex++ - }() - if dbIndex == 0 { - // test db - db, err := mockTestDB() - require.Nil(t, err) - return db, nil - } - // normal db - db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) - require.Nil(t, err) - for i := 0; i < defaultDMLMaxRetryTime; i++ { - mock.ExpectBegin() - mock.ExpectExec("REPLACE INTO `s1`.`t1`(`a`) VALUES (?),(?)"). - WithArgs(1, 2). - WillReturnError(errLockDeadlock) - mock.ExpectRollback() - } - mock.ExpectClose() - return db, nil - } - backupGetDBConn := GetDBConnImpl - GetDBConnImpl = mockGetDBConnErrDatabaseNotExists - defer func() { - GetDBConnImpl = backupGetDBConn - }() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - changefeed := "test-changefeed" - sinkURI, err := url.Parse("mysql://127.0.0.1:4000/?time-zone=UTC&worker-count=1") - require.Nil(t, err) - rc := config.GetDefaultReplicaConfig() - f, err := filter.NewFilter(rc) - require.Nil(t, err) - sink, err := newMySQLSink(ctx, changefeed, sinkURI, f, rc, map[string]string{}) - require.Nil(t, err) - - err = sink.(*mysqlSink).execDMLs(ctx, rows, 1 /* replicaID */, 1 /* bucket */) - require.Equal(t, errLockDeadlock, errors.Cause(err)) - - err = sink.Close(ctx) - require.Nil(t, err) -} - -func TestNewMySQLSinkExecDDL(t *testing.T) { - dbIndex := 0 - mockGetDBConn := func(ctx context.Context, dsnStr string) (*sql.DB, error) { - defer func() { - dbIndex++ - }() - if dbIndex == 0 { - // test db - db, err := mockTestDB() - require.Nil(t, err) - return db, nil - } - // normal db - db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) - require.Nil(t, err) - mock.ExpectBegin() - mock.ExpectExec("USE `test`;").WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectExec("ALTER TABLE test.t1 ADD COLUMN a int").WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - mock.ExpectBegin() - mock.ExpectExec("USE `test`;").WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectExec("ALTER TABLE test.t1 ADD COLUMN a int"). - WillReturnError(&dmysql.MySQLError{ - Number: uint16(infoschema.ErrColumnExists.Code()), - }) - mock.ExpectRollback() - mock.ExpectClose() - return db, nil - } - backupGetDBConn := GetDBConnImpl - GetDBConnImpl = mockGetDBConn - defer func() { - GetDBConnImpl = backupGetDBConn - }() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - changefeed := "test-changefeed" - sinkURI, err := url.Parse("mysql://127.0.0.1:4000/?time-zone=UTC&worker-count=4") - require.Nil(t, err) - rc := config.GetDefaultReplicaConfig() - rc.Filter = &config.FilterConfig{ - Rules: []string{"test.t1"}, - } - f, err := filter.NewFilter(rc) - require.Nil(t, err) - sink, err := newMySQLSink(ctx, changefeed, sinkURI, f, rc, map[string]string{}) - require.Nil(t, err) - - ddl1 := &model.DDLEvent{ - StartTs: 1000, - CommitTs: 1010, - TableInfo: &model.SimpleTableInfo{ - Schema: "test", - Table: "t1", - }, - Type: timodel.ActionAddColumn, - Query: "ALTER TABLE test.t1 ADD COLUMN a int", - } - ddl2 := &model.DDLEvent{ - StartTs: 1020, - CommitTs: 1030, - TableInfo: &model.SimpleTableInfo{ - Schema: "test", - Table: "t2", - }, - Type: timodel.ActionAddColumn, - Query: "ALTER TABLE test.t1 ADD COLUMN a int", - } - - err = sink.EmitDDLEvent(ctx, ddl1) - require.Nil(t, err) - err = sink.EmitDDLEvent(ctx, ddl2) - require.True(t, cerror.ErrDDLEventIgnored.Equal(err)) - // DDL execute failed, but error can be ignored - err = sink.EmitDDLEvent(ctx, ddl1) - require.Nil(t, err) - - err = sink.Close(ctx) - require.Nil(t, err) -} - -func TestNeedSwitchDB(t *testing.T) { - testCases := []struct { - ddl *model.DDLEvent - needSwitch bool - }{ - { - &model.DDLEvent{ - TableInfo: &model.SimpleTableInfo{ - Schema: "", - }, - Type: timodel.ActionCreateTable, - }, - false, - }, - { - &model.DDLEvent{ - TableInfo: &model.SimpleTableInfo{ - Schema: "golang", - }, - Type: timodel.ActionCreateSchema, - }, - false, - }, - { - &model.DDLEvent{ - TableInfo: &model.SimpleTableInfo{ - Schema: "golang", - }, - Type: timodel.ActionDropSchema, - }, - false, - }, - { - &model.DDLEvent{ - TableInfo: &model.SimpleTableInfo{ - Schema: "golang", - }, - Type: timodel.ActionCreateTable, - }, - true, - }, - } - - for _, tc := range testCases { - require.Equal(t, tc.needSwitch, needSwitchDB(tc.ddl)) - } -} - -func TestNewMySQLSink(t *testing.T) { - dbIndex := 0 - mockGetDBConn := func(ctx context.Context, dsnStr string) (*sql.DB, error) { - defer func() { - dbIndex++ - }() - if dbIndex == 0 { - // test db - db, err := mockTestDB() - require.Nil(t, err) - return db, nil - } - // normal db - db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) - mock.ExpectClose() - require.Nil(t, err) - return db, nil - } - backupGetDBConn := GetDBConnImpl - GetDBConnImpl = mockGetDBConn - defer func() { - GetDBConnImpl = backupGetDBConn - }() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - changefeed := "test-changefeed" - sinkURI, err := url.Parse("mysql://127.0.0.1:4000/?time-zone=UTC&worker-count=4") - require.Nil(t, err) - rc := config.GetDefaultReplicaConfig() - f, err := filter.NewFilter(rc) - require.Nil(t, err) - sink, err := newMySQLSink(ctx, changefeed, sinkURI, f, rc, map[string]string{}) - require.Nil(t, err) - err = sink.Close(ctx) - require.Nil(t, err) -} - -func TestMySQLSinkClose(t *testing.T) { - dbIndex := 0 - mockGetDBConn := func(ctx context.Context, dsnStr string) (*sql.DB, error) { - defer func() { - dbIndex++ - }() - if dbIndex == 0 { - // test db - db, err := mockTestDB() - require.Nil(t, err) - return db, nil - } - // normal db - db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) - mock.ExpectClose() - require.Nil(t, err) - return db, nil - } - backupGetDBConn := GetDBConnImpl - GetDBConnImpl = mockGetDBConn - defer func() { - GetDBConnImpl = backupGetDBConn - }() - - ctx := context.Background() - - changefeed := "test-changefeed" - sinkURI, err := url.Parse("mysql://127.0.0.1:4000/?time-zone=UTC&worker-count=4") - require.Nil(t, err) - rc := config.GetDefaultReplicaConfig() - f, err := filter.NewFilter(rc) - require.Nil(t, err) - - // test sink.Close will work correctly even if the ctx pass in has not been cancel - sink, err := newMySQLSink(ctx, changefeed, sinkURI, f, rc, map[string]string{}) - require.Nil(t, err) - err = sink.Close(ctx) - require.Nil(t, err) -} - -func TestMySQLSinkFlushResovledTs(t *testing.T) { - dbIndex := 0 - mockGetDBConn := func(ctx context.Context, dsnStr string) (*sql.DB, error) { - defer func() { - dbIndex++ - }() - if dbIndex == 0 { - // test db - db, err := mockTestDB() - require.Nil(t, err) - return db, nil - } - // normal db - db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) - mock.ExpectBegin() - mock.ExpectExec("REPLACE INTO `s1`.`t1`(`a`) VALUES (?)"). - WithArgs(1). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - mock.ExpectBegin() - mock.ExpectExec("REPLACE INTO `s1`.`t2`(`a`) VALUES (?)"). - WithArgs(1). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - mock.ExpectClose() - require.Nil(t, err) - return db, nil - } - backupGetDBConn := GetDBConnImpl - GetDBConnImpl = mockGetDBConn - defer func() { - GetDBConnImpl = backupGetDBConn - }() - - ctx := context.Background() - - changefeed := "test-changefeed" - sinkURI, err := url.Parse("mysql://127.0.0.1:4000/?time-zone=UTC&worker-count=4") - require.Nil(t, err) - rc := config.GetDefaultReplicaConfig() - f, err := filter.NewFilter(rc) - require.Nil(t, err) - - // test sink.Close will work correctly even if the ctx pass in has not been cancel - si, err := newMySQLSink(ctx, changefeed, sinkURI, f, rc, map[string]string{}) - sink := si.(*mysqlSink) - require.Nil(t, err) - checkpoint, err := sink.FlushRowChangedEvents(ctx, model.TableID(1), 1) - require.Nil(t, err) - require.Equal(t, uint64(0), checkpoint) - rows := []*model.RowChangedEvent{ - { - Table: &model.TableName{Schema: "s1", Table: "t1", TableID: 1}, - CommitTs: 5, - Columns: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, Value: 1}, - }, - }, - } - err = sink.EmitRowChangedEvents(ctx, rows...) - require.Nil(t, err) - checkpoint, err = sink.FlushRowChangedEvents(ctx, model.TableID(1), 6) - require.True(t, checkpoint <= 5) - time.Sleep(500 * time.Millisecond) - require.Nil(t, err) - require.Equal(t, uint64(6), sink.getTableCheckpointTs(model.TableID(1))) - rows = []*model.RowChangedEvent{ - { - Table: &model.TableName{Schema: "s1", Table: "t2", TableID: 2}, - CommitTs: 4, - Columns: []*model.Column{ - {Name: "a", Type: mysql.TypeLong, Flag: model.HandleKeyFlag | model.PrimaryKeyFlag, Value: 1}, - }, - }, - } - err = sink.EmitRowChangedEvents(ctx, rows...) - require.Nil(t, err) - checkpoint, err = sink.FlushRowChangedEvents(ctx, model.TableID(2), 5) - require.True(t, checkpoint <= 5) - time.Sleep(500 * time.Millisecond) - require.Nil(t, err) - require.Equal(t, uint64(5), sink.getTableCheckpointTs(model.TableID(2))) - err = sink.Close(ctx) - require.Nil(t, err) -} diff --git a/cdc/cdc/sink/mysql_worker.go b/cdc/cdc/sink/mysql_worker.go deleted file mode 100644 index 50456876..00000000 --- a/cdc/cdc/sink/mysql_worker.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package sink - -import ( - "context" - "runtime" - "sync" - - "github.com/pingcap/errors" - "github.com/pingcap/log" - "github.com/prometheus/client_golang/prometheus" - "github.com/tikv/migration/cdc/cdc/model" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/notify" - "go.uber.org/zap" -) - -type mysqlSinkWorker struct { - txnCh chan *model.SingleTableTxn - maxTxnRow int - bucket int - execDMLs func(context.Context, []*model.RowChangedEvent, uint64, int) error - metricBucketSize prometheus.Counter - receiver *notify.Receiver - closedCh chan struct{} -} - -func newMySQLSinkWorker( - maxTxnRow int, - bucket int, - metricBucketSize prometheus.Counter, - receiver *notify.Receiver, - execDMLs func(context.Context, []*model.RowChangedEvent, uint64, int) error, -) *mysqlSinkWorker { - return &mysqlSinkWorker{ - txnCh: make(chan *model.SingleTableTxn, 1024), - maxTxnRow: maxTxnRow, - bucket: bucket, - metricBucketSize: metricBucketSize, - execDMLs: execDMLs, - receiver: receiver, - closedCh: make(chan struct{}, 1), - } -} - -func (w *mysqlSinkWorker) appendTxn(ctx context.Context, txn *model.SingleTableTxn) { - if txn == nil { - return - } - select { - case <-ctx.Done(): - case w.txnCh <- txn: - } -} - -func (w *mysqlSinkWorker) appendFinishTxn(wg *sync.WaitGroup) { - // since worker will always fetch txns from txnCh, we don't need to worry the - // txnCh full and send is blocked. - wg.Add(1) - w.txnCh <- &model.SingleTableTxn{ - FinishWg: wg, - } -} - -func (w *mysqlSinkWorker) run(ctx context.Context) (err error) { - var ( - toExecRows []*model.RowChangedEvent - replicaID uint64 - txnNum int - ) - - // mark FinishWg before worker exits, all data txns can be omitted. - defer func() { - for { - select { - case txn := <-w.txnCh: - if txn.FinishWg != nil { - txn.FinishWg.Done() - } - default: - return - } - } - }() - - defer func() { - if r := recover(); r != nil { - buf := make([]byte, 4096) - stackSize := runtime.Stack(buf, false) - buf = buf[:stackSize] - err = cerror.ErrMySQLWorkerPanic.GenWithStack("mysql sink concurrent execute panic, stack: %v", string(buf)) - log.Error("mysql sink worker panic", zap.Reflect("r", r), zap.Stack("stack trace")) - } - }() - - flushRows := func() error { - if len(toExecRows) == 0 { - return nil - } - rows := make([]*model.RowChangedEvent, len(toExecRows)) - copy(rows, toExecRows) - err := w.execDMLs(ctx, rows, replicaID, w.bucket) - if err != nil { - txnNum = 0 - return err - } - toExecRows = toExecRows[:0] - w.metricBucketSize.Add(float64(txnNum)) - txnNum = 0 - return nil - } - - for { - select { - case <-ctx.Done(): - return errors.Trace(ctx.Err()) - case txn := <-w.txnCh: - if txn == nil { - return errors.Trace(flushRows()) - } - if txn.FinishWg != nil { - if err := flushRows(); err != nil { - return errors.Trace(err) - } - txn.FinishWg.Done() - continue - } - if txn.ReplicaID != replicaID || len(toExecRows)+len(txn.Rows) > w.maxTxnRow { - if err := flushRows(); err != nil { - txnNum++ - return errors.Trace(err) - } - } - replicaID = txn.ReplicaID - toExecRows = append(toExecRows, txn.Rows...) - txnNum++ - case <-w.receiver.C: - if err := flushRows(); err != nil { - return errors.Trace(err) - } - } - } -} - -// cleanup waits for notification from closedCh and consumes all txns from txnCh. -// The exit sequence is -// 1. producer(sink.flushRowChangedEvents goroutine) of txnCh exits -// 2. goroutine in 1 sends notification to closedCh of each sink worker -// 3. each sink worker receives the notification from closedCh and mark FinishWg as Done -func (w *mysqlSinkWorker) cleanup() { - <-w.closedCh - for { - select { - case txn := <-w.txnCh: - if txn.FinishWg != nil { - txn.FinishWg.Done() - } - default: - return - } - } -} diff --git a/cdc/cdc/sink/mysql_worker_test.go b/cdc/cdc/sink/mysql_worker_test.go deleted file mode 100644 index 17116899..00000000 --- a/cdc/cdc/sink/mysql_worker_test.go +++ /dev/null @@ -1,362 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package sink - -import ( - "context" - "fmt" - "sync" - "testing" - "time" - - "github.com/davecgh/go-spew/spew" - "github.com/pingcap/errors" - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/notify" - "github.com/tikv/migration/cdc/pkg/util/testleak" - "golang.org/x/sync/errgroup" -) - -func TestMysqlSinkWorker(t *testing.T) { - defer testleak.AfterTestT(t)() - tbl := &model.TableName{ - Schema: "test", - Table: "user", - TableID: 1, - IsPartition: false, - } - testCases := []struct { - txns []*model.SingleTableTxn - expectedOutputRows [][]*model.RowChangedEvent - exportedOutputReplicaIDs []uint64 - maxTxnRow int - }{ - { - txns: []*model.SingleTableTxn{}, - maxTxnRow: 4, - }, { - txns: []*model.SingleTableTxn{ - { - Table: tbl, - CommitTs: 1, - Rows: []*model.RowChangedEvent{{CommitTs: 1}}, - ReplicaID: 1, - }, - }, - expectedOutputRows: [][]*model.RowChangedEvent{{{CommitTs: 1}}}, - exportedOutputReplicaIDs: []uint64{1}, - maxTxnRow: 2, - }, { - txns: []*model.SingleTableTxn{ - { - Table: tbl, - CommitTs: 1, - Rows: []*model.RowChangedEvent{{CommitTs: 1}, {CommitTs: 1}, {CommitTs: 1}}, - ReplicaID: 1, - }, - }, - expectedOutputRows: [][]*model.RowChangedEvent{ - {{CommitTs: 1}, {CommitTs: 1}, {CommitTs: 1}}, - }, - exportedOutputReplicaIDs: []uint64{1}, - maxTxnRow: 2, - }, { - txns: []*model.SingleTableTxn{ - { - Table: tbl, - CommitTs: 1, - Rows: []*model.RowChangedEvent{{CommitTs: 1}, {CommitTs: 1}}, - ReplicaID: 1, - }, - { - Table: tbl, - CommitTs: 2, - Rows: []*model.RowChangedEvent{{CommitTs: 2}}, - ReplicaID: 1, - }, - { - Table: tbl, - CommitTs: 3, - Rows: []*model.RowChangedEvent{{CommitTs: 3}, {CommitTs: 3}}, - ReplicaID: 1, - }, - }, - expectedOutputRows: [][]*model.RowChangedEvent{ - {{CommitTs: 1}, {CommitTs: 1}, {CommitTs: 2}}, - {{CommitTs: 3}, {CommitTs: 3}}, - }, - exportedOutputReplicaIDs: []uint64{1, 1}, - maxTxnRow: 4, - }, { - txns: []*model.SingleTableTxn{ - { - Table: tbl, - CommitTs: 1, - Rows: []*model.RowChangedEvent{{CommitTs: 1}}, - ReplicaID: 1, - }, - { - Table: tbl, - CommitTs: 2, - Rows: []*model.RowChangedEvent{{CommitTs: 2}}, - ReplicaID: 2, - }, - { - Table: tbl, - CommitTs: 3, - Rows: []*model.RowChangedEvent{{CommitTs: 3}}, - ReplicaID: 3, - }, - }, - expectedOutputRows: [][]*model.RowChangedEvent{ - {{CommitTs: 1}}, - {{CommitTs: 2}}, - {{CommitTs: 3}}, - }, - exportedOutputReplicaIDs: []uint64{1, 2, 3}, - maxTxnRow: 4, - }, { - txns: []*model.SingleTableTxn{ - { - Table: tbl, - CommitTs: 1, - Rows: []*model.RowChangedEvent{{CommitTs: 1}}, - ReplicaID: 1, - }, - { - Table: tbl, - CommitTs: 2, - Rows: []*model.RowChangedEvent{{CommitTs: 2}, {CommitTs: 2}, {CommitTs: 2}}, - ReplicaID: 1, - }, - { - Table: tbl, - CommitTs: 3, - Rows: []*model.RowChangedEvent{{CommitTs: 3}}, - ReplicaID: 1, - }, - { - Table: tbl, - CommitTs: 4, - Rows: []*model.RowChangedEvent{{CommitTs: 4}}, - ReplicaID: 1, - }, - }, - expectedOutputRows: [][]*model.RowChangedEvent{ - {{CommitTs: 1}}, - {{CommitTs: 2}, {CommitTs: 2}, {CommitTs: 2}}, - {{CommitTs: 3}, {CommitTs: 4}}, - }, - exportedOutputReplicaIDs: []uint64{1, 1, 1}, - maxTxnRow: 2, - }, - } - ctx := context.Background() - - notifier := new(notify.Notifier) - for i, tc := range testCases { - cctx, cancel := context.WithCancel(ctx) - var outputRows [][]*model.RowChangedEvent - var outputReplicaIDs []uint64 - receiver, err := notifier.NewReceiver(-1) - require.Nil(t, err) - w := newMySQLSinkWorker(tc.maxTxnRow, 1, - bucketSizeCounter.WithLabelValues("capture", "changefeed", "1"), - receiver, - func(ctx context.Context, events []*model.RowChangedEvent, replicaID uint64, bucket int) error { - outputRows = append(outputRows, events) - outputReplicaIDs = append(outputReplicaIDs, replicaID) - return nil - }) - errg, cctx := errgroup.WithContext(cctx) - errg.Go(func() error { - return w.run(cctx) - }) - for _, txn := range tc.txns { - w.appendTxn(cctx, txn) - } - var wg sync.WaitGroup - w.appendFinishTxn(&wg) - // ensure all txns are fetched from txn channel in sink worker - time.Sleep(time.Millisecond * 100) - notifier.Notify() - wg.Wait() - cancel() - require.Equal(t, context.Canceled, errors.Cause(errg.Wait())) - require.Equal(t, tc.expectedOutputRows, outputRows, - fmt.Sprintf("case %v, %s, %s", i, spew.Sdump(outputRows), spew.Sdump(tc.expectedOutputRows))) - require.Equal(t, tc.exportedOutputReplicaIDs, outputReplicaIDs, tc.exportedOutputReplicaIDs, - fmt.Sprintf("case %v, %s, %s", i, spew.Sdump(outputReplicaIDs), spew.Sdump(tc.exportedOutputReplicaIDs))) - } -} - -func TestMySQLSinkWorkerExitWithError(t *testing.T) { - defer testleak.AfterTestT(t)() - tbl := &model.TableName{ - Schema: "test", - Table: "user", - TableID: 1, - IsPartition: false, - } - txns1 := []*model.SingleTableTxn{ - { - Table: tbl, - CommitTs: 1, - Rows: []*model.RowChangedEvent{{CommitTs: 1}}, - }, - { - Table: tbl, - CommitTs: 2, - Rows: []*model.RowChangedEvent{{CommitTs: 2}}, - }, - { - Table: tbl, - CommitTs: 3, - Rows: []*model.RowChangedEvent{{CommitTs: 3}}, - }, - { - Table: tbl, - CommitTs: 4, - Rows: []*model.RowChangedEvent{{CommitTs: 4}}, - }, - } - txns2 := []*model.SingleTableTxn{ - { - Table: tbl, - CommitTs: 5, - Rows: []*model.RowChangedEvent{{CommitTs: 5}}, - }, - { - Table: tbl, - CommitTs: 6, - Rows: []*model.RowChangedEvent{{CommitTs: 6}}, - }, - } - maxTxnRow := 1 - ctx := context.Background() - - errExecFailed := errors.New("sink worker exec failed") - notifier := new(notify.Notifier) - cctx, cancel := context.WithCancel(ctx) - receiver, err := notifier.NewReceiver(-1) - require.Nil(t, err) - w := newMySQLSinkWorker(maxTxnRow, 1, /*bucket*/ - bucketSizeCounter.WithLabelValues("capture", "changefeed", "1"), - receiver, - func(ctx context.Context, events []*model.RowChangedEvent, replicaID uint64, bucket int) error { - return errExecFailed - }) - errg, cctx := errgroup.WithContext(cctx) - errg.Go(func() error { - return w.run(cctx) - }) - // txn in txns1 will be sent to worker txnCh - for _, txn := range txns1 { - w.appendTxn(cctx, txn) - } - - // simulate notify sink worker to flush existing txns - var wg sync.WaitGroup - w.appendFinishTxn(&wg) - time.Sleep(time.Millisecond * 100) - // txn in txn2 will be blocked since the worker has exited - for _, txn := range txns2 { - w.appendTxn(cctx, txn) - } - notifier.Notify() - - // simulate sink shutdown and send closed singal to sink worker - w.closedCh <- struct{}{} - w.cleanup() - - // the flush notification wait group should be done - wg.Wait() - - cancel() - require.Equal(t, errExecFailed, errg.Wait()) -} - -func TestMySQLSinkWorkerExitCleanup(t *testing.T) { - defer testleak.AfterTestT(t)() - tbl := &model.TableName{ - Schema: "test", - Table: "user", - TableID: 1, - IsPartition: false, - } - txns1 := []*model.SingleTableTxn{ - { - Table: tbl, - CommitTs: 1, - Rows: []*model.RowChangedEvent{{CommitTs: 1}}, - }, - { - Table: tbl, - CommitTs: 2, - Rows: []*model.RowChangedEvent{{CommitTs: 2}}, - }, - } - txns2 := []*model.SingleTableTxn{ - { - Table: tbl, - CommitTs: 5, - Rows: []*model.RowChangedEvent{{CommitTs: 5}}, - }, - } - - maxTxnRow := 1 - ctx := context.Background() - - errExecFailed := errors.New("sink worker exec failed") - notifier := new(notify.Notifier) - cctx, cancel := context.WithCancel(ctx) - receiver, err := notifier.NewReceiver(-1) - require.Nil(t, err) - w := newMySQLSinkWorker(maxTxnRow, 1, /*bucket*/ - bucketSizeCounter.WithLabelValues("capture", "changefeed", "1"), - receiver, - func(ctx context.Context, events []*model.RowChangedEvent, replicaID uint64, bucket int) error { - return errExecFailed - }) - errg, cctx := errgroup.WithContext(cctx) - errg.Go(func() error { - err := w.run(cctx) - return err - }) - for _, txn := range txns1 { - w.appendTxn(cctx, txn) - } - - // sleep to let txns flushed by tick - time.Sleep(time.Millisecond * 100) - - // simulate more txns are sent to txnCh after the sink worker run has exited - for _, txn := range txns2 { - w.appendTxn(cctx, txn) - } - var wg sync.WaitGroup - w.appendFinishTxn(&wg) - notifier.Notify() - - // simulate sink shutdown and send closed singal to sink worker - w.closedCh <- struct{}{} - w.cleanup() - - // the flush notification wait group should be done - wg.Wait() - - cancel() - require.Equal(t, errExecFailed, errg.Wait()) -} diff --git a/cdc/cdc/sink/producer/kafka/config.go b/cdc/cdc/sink/producer/kafka/config.go deleted file mode 100644 index 05b62e2d..00000000 --- a/cdc/cdc/sink/producer/kafka/config.go +++ /dev/null @@ -1,297 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - "net/url" - "strconv" - "strings" - "time" - - "github.com/Shopify/sarama" - "github.com/pingcap/errors" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/pkg/config" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/security" - "github.com/tikv/migration/cdc/pkg/util" - "go.uber.org/zap" -) - -func init() { - sarama.MaxRequestSize = 1024 * 1024 * 1024 // 1GB -} - -// Config stores user specified Kafka producer configuration -type Config struct { - BrokerEndpoints []string - PartitionNum int32 - - // User should make sure that `replication-factor` not greater than the number of kafka brokers. - ReplicationFactor int16 - - Version string - MaxMessageBytes int - Compression string - ClientID string - Credential *security.Credential - SaslScram *security.SaslScram - // control whether to create topic - AutoCreate bool -} - -// NewConfig returns a default Kafka configuration -func NewConfig() *Config { - return &Config{ - Version: "2.4.0", - // MaxMessageBytes will be used to initialize producer - MaxMessageBytes: config.DefaultMaxMessageBytes, - ReplicationFactor: 1, - Compression: "none", - Credential: &security.Credential{}, - SaslScram: &security.SaslScram{}, - AutoCreate: true, - } -} - -// set the partition-num by the topic's partition count. -func (c *Config) setPartitionNum(realPartitionCount int32) error { - // user does not specify the `partition-num` in the sink-uri - if c.PartitionNum == 0 { - c.PartitionNum = realPartitionCount - return nil - } - - if c.PartitionNum < realPartitionCount { - log.Warn("number of partition specified in sink-uri is less than that of the actual topic. "+ - "Some partitions will not have messages dispatched to", - zap.Int32("sink-uri partitions", c.PartitionNum), - zap.Int32("topic partitions", realPartitionCount)) - return nil - } - - // Make sure that the user-specified `partition-num` is not greater than - // the real partition count, since messages would be dispatched to different - // partitions, this could prevent potential correctness problems. - if c.PartitionNum > realPartitionCount { - return cerror.ErrKafkaInvalidPartitionNum.GenWithStack( - "the number of partition (%d) specified in sink-uri is more than that of actual topic (%d)", - c.PartitionNum, realPartitionCount) - } - return nil -} - -// CompleteConfigsAndOpts the kafka producer configuration, replication configuration and opts. -func CompleteConfigsAndOpts(sinkURI *url.URL, producerConfig *Config, replicaConfig *config.ReplicaConfig, opts map[string]string) error { - producerConfig.BrokerEndpoints = strings.Split(sinkURI.Host, ",") - params := sinkURI.Query() - s := params.Get("partition-num") - if s != "" { - a, err := strconv.ParseInt(s, 10, 32) - if err != nil { - return err - } - producerConfig.PartitionNum = int32(a) - if producerConfig.PartitionNum <= 0 { - return cerror.ErrKafkaInvalidPartitionNum.GenWithStackByArgs(producerConfig.PartitionNum) - } - } - - s = params.Get("replication-factor") - if s != "" { - a, err := strconv.ParseInt(s, 10, 16) - if err != nil { - return err - } - producerConfig.ReplicationFactor = int16(a) - } - - s = params.Get("kafka-version") - if s != "" { - producerConfig.Version = s - } - - s = params.Get("max-message-bytes") - if s != "" { - a, err := strconv.Atoi(s) - if err != nil { - return err - } - producerConfig.MaxMessageBytes = a - opts["max-message-bytes"] = s - } - - s = params.Get("max-batch-size") - if s != "" { - opts["max-batch-size"] = s - } - - s = params.Get("compression") - if s != "" { - producerConfig.Compression = s - } - - producerConfig.ClientID = params.Get("kafka-client-id") - - s = params.Get("ca") - if s != "" { - producerConfig.Credential.CAPath = s - } - - s = params.Get("cert") - if s != "" { - producerConfig.Credential.CertPath = s - } - - s = params.Get("key") - if s != "" { - producerConfig.Credential.KeyPath = s - } - - s = params.Get("sasl-user") - if s != "" { - producerConfig.SaslScram.SaslUser = s - } - - s = params.Get("sasl-password") - if s != "" { - producerConfig.SaslScram.SaslPassword = s - } - - s = params.Get("sasl-mechanism") - if s != "" { - producerConfig.SaslScram.SaslMechanism = s - } - - s = params.Get("auto-create-topic") - if s != "" { - autoCreate, err := strconv.ParseBool(s) - if err != nil { - return err - } - producerConfig.AutoCreate = autoCreate - } - - s = params.Get(config.ProtocolKey) - if s != "" { - replicaConfig.Sink.Protocol = s - } - - s = params.Get("enable-tidb-extension") - if s != "" { - _, err := strconv.ParseBool(s) - if err != nil { - return err - } - if replicaConfig.Sink.Protocol != "canal-json" { - return cerror.WrapError(cerror.ErrKafkaInvalidConfig, errors.New("enable-tidb-extension only support canal-json protocol")) - } - opts["enable-tidb-extension"] = s - } - - return nil -} - -// newSaramaConfig return the default config and set the according version and metrics -func newSaramaConfig(ctx context.Context, c *Config) (*sarama.Config, error) { - config := sarama.NewConfig() - - version, err := sarama.ParseKafkaVersion(c.Version) - if err != nil { - return nil, cerror.WrapError(cerror.ErrKafkaInvalidVersion, err) - } - var role string - if util.IsOwnerFromCtx(ctx) { - role = "owner" - } else { - role = "processor" - } - captureAddr := util.CaptureAddrFromCtx(ctx) - changefeedID := util.ChangefeedIDFromCtx(ctx) - - config.ClientID, err = kafkaClientID(role, captureAddr, changefeedID, c.ClientID) - if err != nil { - return nil, errors.Trace(err) - } - config.Version = version - // See: https://kafka.apache.org/documentation/#replication - // When one of the brokers in a Kafka cluster is down, the partition leaders - // in this broker is broken, Kafka will election a new partition leader and - // replication logs, this process will last from a few seconds to a few minutes. - // Kafka cluster will not provide a writing service in this process. - // Time out in one minute. - config.Metadata.Retry.Max = 120 - config.Metadata.Retry.Backoff = 500 * time.Millisecond - // If it is not set, this means a metadata request against an unreachable - // cluster (all brokers are unreachable or unresponsive) can take up to - // `Net.[Dial|Read]Timeout * BrokerCount * (Metadata.Retry.Max + 1) + - // Metadata.Retry.Backoff * Metadata.Retry.Max` - // to fail. - // See: https://github.com/Shopify/sarama/issues/765 - // and https://github.com/tikv/migration/cdc/issues/3352. - config.Metadata.Timeout = 1 * time.Minute - - config.Producer.Partitioner = sarama.NewManualPartitioner - config.Producer.MaxMessageBytes = c.MaxMessageBytes - config.Producer.Return.Successes = true - config.Producer.Return.Errors = true - config.Producer.RequiredAcks = sarama.WaitForAll - // Time out in five minutes(600 * 500ms). - config.Producer.Retry.Max = 600 - config.Producer.Retry.Backoff = 500 * time.Millisecond - switch strings.ToLower(strings.TrimSpace(c.Compression)) { - case "none": - config.Producer.Compression = sarama.CompressionNone - case "gzip": - config.Producer.Compression = sarama.CompressionGZIP - case "snappy": - config.Producer.Compression = sarama.CompressionSnappy - case "lz4": - config.Producer.Compression = sarama.CompressionLZ4 - case "zstd": - config.Producer.Compression = sarama.CompressionZSTD - default: - log.Warn("Unsupported compression algorithm", zap.String("compression", c.Compression)) - config.Producer.Compression = sarama.CompressionNone - } - - // Time out in one minute(120 * 500ms). - config.Admin.Retry.Max = 120 - config.Admin.Retry.Backoff = 500 * time.Millisecond - config.Admin.Timeout = 1 * time.Minute - - if c.Credential != nil && len(c.Credential.CAPath) != 0 { - config.Net.TLS.Enable = true - config.Net.TLS.Config, err = c.Credential.ToTLSConfig() - if err != nil { - return nil, errors.Trace(err) - } - } - if c.SaslScram != nil && len(c.SaslScram.SaslUser) != 0 { - config.Net.SASL.Enable = true - config.Net.SASL.User = c.SaslScram.SaslUser - config.Net.SASL.Password = c.SaslScram.SaslPassword - config.Net.SASL.Mechanism = sarama.SASLMechanism(c.SaslScram.SaslMechanism) - if strings.EqualFold(c.SaslScram.SaslMechanism, "SCRAM-SHA-256") { - config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &security.XDGSCRAMClient{HashGeneratorFcn: security.SHA256} } - } else if strings.EqualFold(c.SaslScram.SaslMechanism, "SCRAM-SHA-512") { - config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &security.XDGSCRAMClient{HashGeneratorFcn: security.SHA512} } - } else { - return nil, errors.New("Unsupported sasl-mechanism, should be SCRAM-SHA-256 or SCRAM-SHA-512") - } - } - - return config, err -} diff --git a/cdc/cdc/sink/producer/kafka/config_test.go b/cdc/cdc/sink/producer/kafka/config_test.go deleted file mode 100644 index 1eb30a36..00000000 --- a/cdc/cdc/sink/producer/kafka/config_test.go +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - "fmt" - "net/url" - - "github.com/Shopify/sarama" - "github.com/pingcap/check" - "github.com/pingcap/errors" - "github.com/tikv/migration/cdc/pkg/config" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/security" - "github.com/tikv/migration/cdc/pkg/util" - "github.com/tikv/migration/cdc/pkg/util/testleak" -) - -func (s *kafkaSuite) TestNewSaramaConfig(c *check.C) { - defer testleak.AfterTest(c)() - ctx := context.Background() - config := NewConfig() - config.Version = "invalid" - _, err := newSaramaConfigImpl(ctx, config) - c.Assert(errors.Cause(err), check.ErrorMatches, "invalid version.*") - - ctx = util.SetOwnerInCtx(ctx) - config.Version = "2.6.0" - config.ClientID = "^invalid$" - _, err = newSaramaConfigImpl(ctx, config) - c.Assert(cerror.ErrKafkaInvalidClientID.Equal(err), check.IsTrue) - - config.ClientID = "test-kafka-client" - compressionCases := []struct { - algorithm string - expected sarama.CompressionCodec - }{ - {"none", sarama.CompressionNone}, - {"gzip", sarama.CompressionGZIP}, - {"snappy", sarama.CompressionSnappy}, - {"lz4", sarama.CompressionLZ4}, - {"zstd", sarama.CompressionZSTD}, - {"others", sarama.CompressionNone}, - } - for _, cc := range compressionCases { - config.Compression = cc.algorithm - cfg, err := newSaramaConfigImpl(ctx, config) - c.Assert(err, check.IsNil) - c.Assert(cfg.Producer.Compression, check.Equals, cc.expected) - } - - config.Credential = &security.Credential{ - CAPath: "/invalid/ca/path", - } - _, err = newSaramaConfigImpl(ctx, config) - c.Assert(errors.Cause(err), check.ErrorMatches, ".*no such file or directory") - - saslConfig := NewConfig() - saslConfig.Version = "2.6.0" - saslConfig.ClientID = "test-sasl-scram" - saslConfig.SaslScram = &security.SaslScram{ - SaslUser: "user", - SaslPassword: "password", - SaslMechanism: sarama.SASLTypeSCRAMSHA256, - } - - cfg, err := newSaramaConfigImpl(ctx, saslConfig) - c.Assert(err, check.IsNil) - c.Assert(cfg, check.NotNil) - c.Assert(cfg.Net.SASL.User, check.Equals, "user") - c.Assert(cfg.Net.SASL.Password, check.Equals, "password") - c.Assert(cfg.Net.SASL.Mechanism, check.Equals, sarama.SASLMechanism("SCRAM-SHA-256")) -} - -func (s *kafkaSuite) TestCompleteConfigByOpts(c *check.C) { - defer testleak.AfterTest(c) - cfg := NewConfig() - - // Normal config. - uriTemplate := "kafka://127.0.0.1:9092/kafka-test?kafka-version=2.6.0&max-batch-size=5" + - "&max-message-bytes=%s&partition-num=1&replication-factor=3" + - "&kafka-client-id=unit-test&auto-create-topic=false&compression=gzip" - maxMessageSize := "4096" // 4kb - uri := fmt.Sprintf(uriTemplate, maxMessageSize) - sinkURI, err := url.Parse(uri) - c.Assert(err, check.IsNil) - opts := make(map[string]string) - err = CompleteConfigsAndOpts(sinkURI, cfg, config.GetDefaultReplicaConfig(), opts) - c.Assert(err, check.IsNil) - c.Assert(cfg.PartitionNum, check.Equals, int32(1)) - c.Assert(cfg.ReplicationFactor, check.Equals, int16(3)) - c.Assert(cfg.Version, check.Equals, "2.6.0") - c.Assert(cfg.MaxMessageBytes, check.Equals, 4096) - expectedOpts := map[string]string{ - "max-message-bytes": maxMessageSize, - "max-batch-size": "5", - } - for k, v := range opts { - c.Assert(v, check.Equals, expectedOpts[k]) - } - - // Illegal replication-factor. - uri = "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&replication-factor=a" - sinkURI, err = url.Parse(uri) - c.Assert(err, check.IsNil) - cfg = NewConfig() - err = CompleteConfigsAndOpts(sinkURI, cfg, config.GetDefaultReplicaConfig(), opts) - c.Assert(errors.Cause(err), check.ErrorMatches, ".*invalid syntax.*") - - // Illegal max-message-bytes. - uri = "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&max-message-bytes=a" - sinkURI, err = url.Parse(uri) - c.Assert(err, check.IsNil) - cfg = NewConfig() - err = CompleteConfigsAndOpts(sinkURI, cfg, config.GetDefaultReplicaConfig(), opts) - c.Assert(errors.Cause(err), check.ErrorMatches, ".*invalid syntax.*") - - // Illegal enable-tidb-extension. - uri = "kafka://127.0.0.1:9092/abc?enable-tidb-extension=a&protocol=canal-json" - sinkURI, err = url.Parse(uri) - c.Assert(err, check.IsNil) - cfg = NewConfig() - err = CompleteConfigsAndOpts(sinkURI, cfg, config.GetDefaultReplicaConfig(), opts) - c.Assert(errors.Cause(err), check.ErrorMatches, ".*invalid syntax.*") - - // Illegal partition-num. - uri = "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=a" - sinkURI, err = url.Parse(uri) - c.Assert(err, check.IsNil) - cfg = NewConfig() - err = CompleteConfigsAndOpts(sinkURI, cfg, config.GetDefaultReplicaConfig(), opts) - c.Assert(errors.Cause(err), check.ErrorMatches, ".*invalid syntax.*") - - // Out of range partition-num. - uri = "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=0" - sinkURI, err = url.Parse(uri) - c.Assert(err, check.IsNil) - cfg = NewConfig() - err = CompleteConfigsAndOpts(sinkURI, cfg, config.GetDefaultReplicaConfig(), opts) - c.Assert(errors.Cause(err), check.ErrorMatches, ".*invalid partition num.*") - - // Use enable-tidb-extension on other protocols. - uri = "kafka://127.0.0.1:9092/abc?kafka-version=2.6.0&partition-num=1&enable-tidb-extension=true" - sinkURI, err = url.Parse(uri) - c.Assert(err, check.IsNil) - cfg = NewConfig() - err = CompleteConfigsAndOpts(sinkURI, cfg, config.GetDefaultReplicaConfig(), opts) - c.Assert(errors.Cause(err), check.ErrorMatches, ".*enable-tidb-extension only support canal-json protocol.*") - - // Test enable-tidb-extension. - uri = "kafka://127.0.0.1:9092/abc?enable-tidb-extension=true&protocol=canal-json" - sinkURI, err = url.Parse(uri) - c.Assert(err, check.IsNil) - cfg = NewConfig() - opts = make(map[string]string) - err = CompleteConfigsAndOpts(sinkURI, cfg, config.GetDefaultReplicaConfig(), opts) - c.Assert(err, check.IsNil) - expectedOpts = map[string]string{ - "enable-tidb-extension": "true", - } - for k, v := range opts { - c.Assert(v, check.Equals, expectedOpts[k]) - } -} - -func (s *kafkaSuite) TestSetPartitionNum(c *check.C) { - defer testleak.AfterTest(c)() - cfg := NewConfig() - err := cfg.setPartitionNum(2) - c.Assert(err, check.IsNil) - c.Assert(cfg.PartitionNum, check.Equals, int32(2)) - - cfg.PartitionNum = 1 - err = cfg.setPartitionNum(2) - c.Assert(err, check.IsNil) - c.Assert(cfg.PartitionNum, check.Equals, int32(1)) - - cfg.PartitionNum = 3 - err = cfg.setPartitionNum(2) - c.Assert(cerror.ErrKafkaInvalidPartitionNum.Equal(err), check.IsTrue) -} diff --git a/cdc/cdc/sink/producer/kafka/kafka.go b/cdc/cdc/sink/producer/kafka/kafka.go deleted file mode 100644 index c801ac4b..00000000 --- a/cdc/cdc/sink/producer/kafka/kafka.go +++ /dev/null @@ -1,470 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - "fmt" - "regexp" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/Shopify/sarama" - "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/sink/codec" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/kafka" - "github.com/tikv/migration/cdc/pkg/notify" - "go.uber.org/zap" -) - -const ( - // defaultPartitionNum specifies the default number of partitions when we create the topic. - defaultPartitionNum = 3 -) - -const ( - kafkaProducerRunning = 0 - kafkaProducerClosing = 1 -) - -type kafkaSaramaProducer struct { - // clientLock is used to protect concurrent access of asyncClient and syncClient. - // Since we don't close these two clients (which have an input chan) from the - // sender routine, data race or send on closed chan could happen. - clientLock sync.RWMutex - asyncClient sarama.AsyncProducer - syncClient sarama.SyncProducer - // producersReleased records whether asyncClient and syncClient have been closed properly - producersReleased bool - topic string - partitionNum int32 - - partitionOffset []struct { - flushed uint64 - sent uint64 - } - flushedNotifier *notify.Notifier - flushedReceiver *notify.Receiver - - failpointCh chan error - - closeCh chan struct{} - // atomic flag indicating whether the producer is closing - closing kafkaProducerClosingFlag -} - -type kafkaProducerClosingFlag = int32 - -func (k *kafkaSaramaProducer) AsyncSendMessage(ctx context.Context, message *codec.MQMessage, partition int32) error { - k.clientLock.RLock() - defer k.clientLock.RUnlock() - - // Checks whether the producer is closing. - // The atomic flag must be checked under `clientLock.RLock()` - if atomic.LoadInt32(&k.closing) == kafkaProducerClosing { - return nil - } - - msg := &sarama.ProducerMessage{ - Topic: k.topic, - Key: sarama.ByteEncoder(message.Key), - Value: sarama.ByteEncoder(message.Value), - Partition: partition, - } - msg.Metadata = atomic.AddUint64(&k.partitionOffset[partition].sent, 1) - - failpoint.Inject("KafkaSinkAsyncSendError", func() { - // simulate sending message to input channel successfully but flushing - // message to Kafka meets error - log.Info("failpoint error injected") - k.failpointCh <- errors.New("kafka sink injected error") - failpoint.Return(nil) - }) - - failpoint.Inject("SinkFlushDMLPanic", func() { - time.Sleep(time.Second) - log.Panic("SinkFlushDMLPanic") - }) - - select { - case <-ctx.Done(): - return ctx.Err() - case <-k.closeCh: - return nil - case k.asyncClient.Input() <- msg: - } - return nil -} - -func (k *kafkaSaramaProducer) SyncBroadcastMessage(ctx context.Context, message *codec.MQMessage) error { - k.clientLock.RLock() - defer k.clientLock.RUnlock() - msgs := make([]*sarama.ProducerMessage, k.partitionNum) - for i := 0; i < int(k.partitionNum); i++ { - msgs[i] = &sarama.ProducerMessage{ - Topic: k.topic, - Key: sarama.ByteEncoder(message.Key), - Value: sarama.ByteEncoder(message.Value), - Partition: int32(i), - } - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-k.closeCh: - return nil - default: - err := k.syncClient.SendMessages(msgs) - return cerror.WrapError(cerror.ErrKafkaSendMessage, err) - } -} - -func (k *kafkaSaramaProducer) Flush(ctx context.Context) error { - targetOffsets := make([]uint64, k.partitionNum) - for i := 0; i < len(k.partitionOffset); i++ { - targetOffsets[i] = atomic.LoadUint64(&k.partitionOffset[i].sent) - } - - noEventsToFLush := true - for i, target := range targetOffsets { - if target > atomic.LoadUint64(&k.partitionOffset[i].flushed) { - noEventsToFLush = false - break - } - } - if noEventsToFLush { - // no events to flush - return nil - } - - // checkAllPartitionFlushed checks whether data in each partition is flushed - checkAllPartitionFlushed := func() bool { - for i, target := range targetOffsets { - if target > atomic.LoadUint64(&k.partitionOffset[i].flushed) { - return false - } - } - return true - } - -flushLoop: - for { - select { - case <-ctx.Done(): - return ctx.Err() - case <-k.closeCh: - if checkAllPartitionFlushed() { - return nil - } - return cerror.ErrKafkaFlushUnfinished.GenWithStackByArgs() - case <-k.flushedReceiver.C: - if !checkAllPartitionFlushed() { - continue flushLoop - } - return nil - } - } -} - -func (k *kafkaSaramaProducer) GetPartitionNum() int32 { - return k.partitionNum -} - -// stop closes the closeCh to signal other routines to exit -// It SHOULD NOT be called under `clientLock`. -func (k *kafkaSaramaProducer) stop() { - if atomic.SwapInt32(&k.closing, kafkaProducerClosing) == kafkaProducerClosing { - return - } - log.Info("kafka producer closing...") - close(k.closeCh) -} - -// Close closes the sync and async clients. -func (k *kafkaSaramaProducer) Close() error { - k.stop() - - k.clientLock.Lock() - defer k.clientLock.Unlock() - - if k.producersReleased { - // We need to guard against double closing the clients, - // which could lead to panic. - return nil - } - k.producersReleased = true - // In fact close sarama sync client doesn't return any error. - // But close async client returns error if error channel is not empty, we - // don't populate this error to the upper caller, just add a log here. - err1 := k.syncClient.Close() - err2 := k.asyncClient.Close() - if err1 != nil { - log.Error("close sync client with error", zap.Error(err1)) - } - if err2 != nil { - log.Error("close async client with error", zap.Error(err2)) - } - return nil -} - -func (k *kafkaSaramaProducer) run(ctx context.Context) error { - defer func() { - k.flushedReceiver.Stop() - k.stop() - }() - for { - select { - case <-ctx.Done(): - return ctx.Err() - case <-k.closeCh: - return nil - case err := <-k.failpointCh: - log.Warn("receive from failpoint chan", zap.Error(err)) - return err - case msg := <-k.asyncClient.Successes(): - if msg == nil || msg.Metadata == nil { - continue - } - flushedOffset := msg.Metadata.(uint64) - atomic.StoreUint64(&k.partitionOffset[msg.Partition].flushed, flushedOffset) - k.flushedNotifier.Notify() - case err := <-k.asyncClient.Errors(): - // We should not wrap a nil pointer if the pointer is of a subtype of `error` - // because Go would store the type info and the resulted `error` variable would not be nil, - // which will cause the pkg/error library to malfunction. - if err == nil { - return nil - } - return cerror.WrapError(cerror.ErrKafkaAsyncSendMessage, err) - } - } -} - -var ( - newSaramaConfigImpl = newSaramaConfig - NewAdminClientImpl kafka.ClusterAdminClientCreator = kafka.NewSaramaAdminClient -) - -// NewKafkaSaramaProducer creates a kafka sarama producer -func NewKafkaSaramaProducer(ctx context.Context, topic string, config *Config, opts map[string]string, errCh chan error) (*kafkaSaramaProducer, error) { - log.Info("Starting kafka sarama producer ...", zap.Reflect("config", config)) - cfg, err := newSaramaConfigImpl(ctx, config) - if err != nil { - return nil, err - } - - admin, err := NewAdminClientImpl(config.BrokerEndpoints, cfg) - if err != nil { - return nil, cerror.WrapError(cerror.ErrKafkaNewSaramaProducer, err) - } - defer func() { - if err := admin.Close(); err != nil { - log.Warn("close kafka cluster admin failed", zap.Error(err)) - } - }() - - if err := validateMaxMessageBytesAndCreateTopic(admin, topic, config, cfg, opts); err != nil { - return nil, cerror.WrapError(cerror.ErrKafkaNewSaramaProducer, err) - } - - asyncClient, err := sarama.NewAsyncProducer(config.BrokerEndpoints, cfg) - if err != nil { - return nil, cerror.WrapError(cerror.ErrKafkaNewSaramaProducer, err) - } - syncClient, err := sarama.NewSyncProducer(config.BrokerEndpoints, cfg) - if err != nil { - return nil, cerror.WrapError(cerror.ErrKafkaNewSaramaProducer, err) - } - - notifier := new(notify.Notifier) - flushedReceiver, err := notifier.NewReceiver(50 * time.Millisecond) - if err != nil { - return nil, err - } - k := &kafkaSaramaProducer{ - asyncClient: asyncClient, - syncClient: syncClient, - topic: topic, - partitionNum: config.PartitionNum, - partitionOffset: make([]struct { - flushed uint64 - sent uint64 - }, config.PartitionNum), - flushedNotifier: notifier, - flushedReceiver: flushedReceiver, - closeCh: make(chan struct{}), - failpointCh: make(chan error, 1), - closing: kafkaProducerRunning, - } - go func() { - if err := k.run(ctx); err != nil && errors.Cause(err) != context.Canceled { - select { - case <-ctx.Done(): - return - case errCh <- err: - default: - log.Error("error channel is full", zap.Error(err)) - } - } - }() - return k, nil -} - -var ( - validClientID = regexp.MustCompile(`\A[A-Za-z0-9._-]+\z`) - commonInvalidChar = regexp.MustCompile(`[\?:,"]`) -) - -func kafkaClientID(role, captureAddr, changefeedID, configuredClientID string) (clientID string, err error) { - if configuredClientID != "" { - clientID = configuredClientID - } else { - clientID = fmt.Sprintf("TiCDC_sarama_producer_%s_%s_%s", role, captureAddr, changefeedID) - clientID = commonInvalidChar.ReplaceAllString(clientID, "_") - } - if !validClientID.MatchString(clientID) { - return "", cerror.ErrKafkaInvalidClientID.GenWithStackByArgs(clientID) - } - return -} - -func validateMaxMessageBytesAndCreateTopic(admin kafka.ClusterAdminClient, topic string, config *Config, saramaConfig *sarama.Config, opts map[string]string) error { - topics, err := admin.ListTopics() - if err != nil { - return cerror.WrapError(cerror.ErrKafkaNewSaramaProducer, err) - } - - info, exists := topics[topic] - // once we have found the topic, no matter `auto-create-topic`, make sure user input parameters are valid. - if exists { - // make sure that producer's `MaxMessageBytes` smaller than topic's `max.message.bytes` - topicMaxMessageBytes, err := getTopicMaxMessageBytes(admin, info) - if err != nil { - return cerror.WrapError(cerror.ErrKafkaNewSaramaProducer, err) - } - - if topicMaxMessageBytes < config.MaxMessageBytes { - log.Warn("topic's `max.message.bytes` less than the user set `max-message-bytes`,"+ - "use topic's `max.message.bytes` to initialize the Kafka producer", - zap.Int("max.message.bytes", topicMaxMessageBytes), - zap.Int("max-message-bytes", config.MaxMessageBytes)) - saramaConfig.Producer.MaxMessageBytes = topicMaxMessageBytes - } - opts["max-message-bytes"] = strconv.Itoa(saramaConfig.Producer.MaxMessageBytes) - - // no need to create the topic, but we would have to log user if they found enter wrong topic name later - if config.AutoCreate { - log.Warn("topic already exist, TiCDC will not create the topic", - zap.String("topic", topic), zap.Any("detail", info)) - } - - if err := config.setPartitionNum(info.NumPartitions); err != nil { - return errors.Trace(err) - } - - return nil - } - - if !config.AutoCreate { - return cerror.ErrKafkaInvalidConfig.GenWithStack("`auto-create-topic` is false, and topic not found") - } - - brokerMessageMaxBytes, err := getBrokerMessageMaxBytes(admin) - if err != nil { - log.Warn("TiCDC cannot find `message.max.bytes` from broker's configuration") - return errors.Trace(err) - } - - // when create the topic, `max.message.bytes` is decided by the broker, - // it would use broker's `message.max.bytes` to set topic's `max.message.bytes`. - // TiCDC need to make sure that the producer's `MaxMessageBytes` won't larger than - // broker's `message.max.bytes`. - if brokerMessageMaxBytes < config.MaxMessageBytes { - log.Warn("broker's `message.max.bytes` less than the user set `max-message-bytes`,"+ - "use broker's `message.max.bytes` to initialize the Kafka producer", - zap.Int("message.max.bytes", brokerMessageMaxBytes), - zap.Int("max-message-bytes", config.MaxMessageBytes)) - saramaConfig.Producer.MaxMessageBytes = brokerMessageMaxBytes - } - opts["max-message-bytes"] = strconv.Itoa(saramaConfig.Producer.MaxMessageBytes) - - // topic not exists yet, and user does not specify the `partition-num` in the sink uri. - if config.PartitionNum == 0 { - config.PartitionNum = defaultPartitionNum - log.Warn("partition-num is not set, use the default partition count", - zap.String("topic", topic), zap.Int32("partitions", config.PartitionNum)) - } - - err = admin.CreateTopic(topic, &sarama.TopicDetail{ - NumPartitions: config.PartitionNum, - ReplicationFactor: config.ReplicationFactor, - }, false) - // TODO identify the cause of "Topic with this name already exists" - if err != nil && !strings.Contains(err.Error(), "already exists") { - return cerror.WrapError(cerror.ErrKafkaNewSaramaProducer, err) - } - - log.Info("TiCDC create the topic", - zap.Int32("partition-num", config.PartitionNum), - zap.Int16("replication-factor", config.ReplicationFactor)) - - return nil -} - -func getBrokerMessageMaxBytes(admin kafka.ClusterAdminClient) (int, error) { - _, controllerID, err := admin.DescribeCluster() - if err != nil { - return 0, cerror.WrapError(cerror.ErrKafkaNewSaramaProducer, err) - } - - configEntries, err := admin.DescribeConfig(sarama.ConfigResource{ - Type: sarama.BrokerResource, - Name: strconv.Itoa(int(controllerID)), - ConfigNames: []string{kafka.BrokerMessageMaxBytesConfigName}, - }) - if err != nil { - return 0, cerror.WrapError(cerror.ErrKafkaNewSaramaProducer, err) - } - - if len(configEntries) == 0 || configEntries[0].Name != kafka.BrokerMessageMaxBytesConfigName { - return 0, cerror.ErrKafkaNewSaramaProducer.GenWithStack( - "since cannot find the `message.max.bytes` from the broker's configuration, " + - "ticdc decline to create the topic and changefeed to prevent potential error") - } - - result, err := strconv.Atoi(configEntries[0].Value) - if err != nil { - return 0, cerror.WrapError(cerror.ErrKafkaNewSaramaProducer, err) - } - - return result, nil -} - -func getTopicMaxMessageBytes(admin kafka.ClusterAdminClient, info sarama.TopicDetail) (int, error) { - if a, ok := info.ConfigEntries[kafka.TopicMaxMessageBytesConfigName]; ok { - result, err := strconv.Atoi(*a) - if err != nil { - return 0, cerror.WrapError(cerror.ErrKafkaNewSaramaProducer, err) - } - return result, nil - } - - return getBrokerMessageMaxBytes(admin) -} diff --git a/cdc/cdc/sink/producer/kafka/kafka_test.go b/cdc/cdc/sink/producer/kafka/kafka_test.go deleted file mode 100644 index 2bb8332f..00000000 --- a/cdc/cdc/sink/producer/kafka/kafka_test.go +++ /dev/null @@ -1,445 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kafka - -import ( - "context" - "strconv" - "strings" - "sync" - "testing" - "time" - - "github.com/Shopify/sarama" - "github.com/pingcap/check" - "github.com/pingcap/errors" - "github.com/tikv/migration/cdc/cdc/sink/codec" - "github.com/tikv/migration/cdc/pkg/kafka" - "github.com/tikv/migration/cdc/pkg/util/testleak" -) - -type kafkaSuite struct{} - -var _ = check.Suite(&kafkaSuite{}) - -func Test(t *testing.T) { check.TestingT(t) } - -func (s *kafkaSuite) TestClientID(c *check.C) { - defer testleak.AfterTest(c)() - testCases := []struct { - role string - addr string - changefeedID string - configuredID string - hasError bool - expected string - }{ - {"owner", "domain:1234", "123-121-121-121", "", false, "TiCDC_sarama_producer_owner_domain_1234_123-121-121-121"}, - {"owner", "127.0.0.1:1234", "123-121-121-121", "", false, "TiCDC_sarama_producer_owner_127.0.0.1_1234_123-121-121-121"}, - {"owner", "127.0.0.1:1234?:,\"", "123-121-121-121", "", false, "TiCDC_sarama_producer_owner_127.0.0.1_1234_____123-121-121-121"}, - {"owner", "中文", "123-121-121-121", "", true, ""}, - {"owner", "127.0.0.1:1234", "123-121-121-121", "cdc-changefeed-1", false, "cdc-changefeed-1"}, - } - for _, tc := range testCases { - id, err := kafkaClientID(tc.role, tc.addr, tc.changefeedID, tc.configuredID) - if tc.hasError { - c.Assert(err, check.NotNil) - } else { - c.Assert(err, check.IsNil) - c.Assert(id, check.Equals, tc.expected) - } - } -} - -func (s *kafkaSuite) TestNewSaramaProducer(c *check.C) { - defer testleak.AfterTest(c)() - ctx, cancel := context.WithCancel(context.Background()) - - topic := kafka.DefaultMockTopicName - leader := sarama.NewMockBroker(c, 2) - defer leader.Close() - metadataResponse := new(sarama.MetadataResponse) - metadataResponse.AddBroker(leader.Addr(), leader.BrokerID()) - metadataResponse.AddTopicPartition(topic, 0, leader.BrokerID(), nil, nil, nil, sarama.ErrNoError) - metadataResponse.AddTopicPartition(topic, 1, leader.BrokerID(), nil, nil, nil, sarama.ErrNoError) - leader.Returns(metadataResponse) - leader.Returns(metadataResponse) - - prodSuccess := new(sarama.ProduceResponse) - prodSuccess.AddTopicPartition(topic, 0, sarama.ErrNoError) - prodSuccess.AddTopicPartition(topic, 1, sarama.ErrNoError) - // 200 async messages and 2 sync message, Kafka flush could be in batch, - // we can set flush.max.messages to 1 to control message count exactly. - for i := 0; i < 202; i++ { - leader.Returns(prodSuccess) - } - - errCh := make(chan error, 1) - config := NewConfig() - // Because the sarama mock broker is not compatible with version larger than 1.0.0 - // We use a smaller version in the following producer tests. - // Ref: https://github.com/Shopify/sarama/blob/89707055369768913defac030c15cf08e9e57925/async_producer_test.go#L1445-L1447 - config.Version = "0.9.0.0" - config.PartitionNum = int32(2) - config.AutoCreate = false - config.BrokerEndpoints = strings.Split(leader.Addr(), ",") - - newSaramaConfigImplBak := newSaramaConfigImpl - newSaramaConfigImpl = func(ctx context.Context, config *Config) (*sarama.Config, error) { - cfg, err := newSaramaConfigImplBak(ctx, config) - c.Assert(err, check.IsNil) - cfg.Producer.Flush.MaxMessages = 1 - return cfg, err - } - NewAdminClientImpl = kafka.NewMockAdminClient - defer func() { - NewAdminClientImpl = kafka.NewSaramaAdminClient - }() - - opts := make(map[string]string) - producer, err := NewKafkaSaramaProducer(ctx, topic, config, opts, errCh) - c.Assert(err, check.IsNil) - c.Assert(producer.GetPartitionNum(), check.Equals, int32(2)) - c.Assert(opts, check.HasKey, "max-message-bytes") - for i := 0; i < 100; i++ { - err = producer.AsyncSendMessage(ctx, &codec.MQMessage{ - Key: []byte("test-key-1"), - Value: []byte("test-value"), - }, int32(0)) - c.Assert(err, check.IsNil) - err = producer.AsyncSendMessage(ctx, &codec.MQMessage{ - Key: []byte("test-key-1"), - Value: []byte("test-value"), - }, int32(1)) - c.Assert(err, check.IsNil) - } - - // In TiCDC logic, resolved ts event will always notify the flush loop. Here we - // trigger the flushedNotifier periodically to prevent the flush loop block. - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - for { - select { - case <-ctx.Done(): - return - case <-time.After(time.Millisecond * 100): - producer.flushedNotifier.Notify() - } - } - }() - - err = producer.Flush(ctx) - c.Assert(err, check.IsNil) - expected := []struct { - flushed uint64 - sent uint64 - }{ - {100, 100}, - {100, 100}, - } - c.Assert(producer.partitionOffset, check.DeepEquals, expected) - select { - case err := <-errCh: - c.Fatalf("unexpected err: %s", err) - default: - } - // check no events to flush - err = producer.Flush(ctx) - c.Assert(err, check.IsNil) - - err = producer.SyncBroadcastMessage(ctx, &codec.MQMessage{ - Key: []byte("test-broadcast"), - Value: nil, - }) - c.Assert(err, check.IsNil) - - err = producer.Close() - c.Assert(err, check.IsNil) - // check reentrant close - err = producer.Close() - c.Assert(err, check.IsNil) - cancel() - wg.Wait() - - // check send messages when context is canceled or producer closed - err = producer.AsyncSendMessage(ctx, &codec.MQMessage{ - Key: []byte("cancel"), - Value: nil, - }, int32(0)) - if err != nil { - c.Assert(err, check.Equals, context.Canceled) - } - err = producer.SyncBroadcastMessage(ctx, &codec.MQMessage{ - Key: []byte("cancel"), - Value: nil, - }) - if err != nil { - c.Assert(err, check.Equals, context.Canceled) - } -} - -func (s *kafkaSuite) TestValidateMaxMessageBytesAndCreateTopic(c *check.C) { - defer testleak.AfterTest(c) - config := NewConfig() - adminClient := kafka.NewClusterAdminClientMockImpl() - defer func() { - _ = adminClient.Close() - }() - - // When topic exists and max message bytes is set correctly. - config.MaxMessageBytes = adminClient.GetDefaultMaxMessageBytes() - cfg, err := newSaramaConfigImpl(context.Background(), config) - c.Assert(err, check.IsNil) - opts := make(map[string]string) - err = validateMaxMessageBytesAndCreateTopic(adminClient, adminClient.GetDefaultMockTopicName(), config, cfg, opts) - c.Assert(err, check.IsNil) - c.Assert(opts["max-message-bytes"], check.Equals, strconv.Itoa(cfg.Producer.MaxMessageBytes)) - - // When topic exists and max message bytes is not set correctly. - // use the smaller one. - defaultMaxMessageBytes := adminClient.GetDefaultMaxMessageBytes() - config.MaxMessageBytes = defaultMaxMessageBytes + 1 - cfg, err = newSaramaConfigImpl(context.Background(), config) - c.Assert(err, check.IsNil) - opts = make(map[string]string) - err = validateMaxMessageBytesAndCreateTopic(adminClient, adminClient.GetDefaultMockTopicName(), config, cfg, opts) - c.Assert(err, check.IsNil) - c.Assert(cfg.Producer.MaxMessageBytes, check.Equals, defaultMaxMessageBytes) - c.Assert(opts["max-message-bytes"], check.Equals, strconv.Itoa(cfg.Producer.MaxMessageBytes)) - - config.MaxMessageBytes = defaultMaxMessageBytes - 1 - cfg, err = newSaramaConfigImpl(context.Background(), config) - c.Assert(err, check.IsNil) - opts = make(map[string]string) - err = validateMaxMessageBytesAndCreateTopic(adminClient, adminClient.GetDefaultMockTopicName(), config, cfg, opts) - c.Assert(err, check.IsNil) - c.Assert(cfg.Producer.MaxMessageBytes, check.Equals, config.MaxMessageBytes) - c.Assert(opts["max-message-bytes"], check.Equals, strconv.Itoa(cfg.Producer.MaxMessageBytes)) - - // When topic does not exist and auto-create is not enabled. - config.AutoCreate = false - cfg, err = newSaramaConfigImpl(context.Background(), config) - c.Assert(err, check.IsNil) - opts = make(map[string]string) - err = validateMaxMessageBytesAndCreateTopic(adminClient, "non-exist", config, cfg, opts) - c.Assert( - errors.Cause(err), - check.ErrorMatches, - ".*auto-create-topic` is false, and topic not found.*", - ) - - // When the topic does not exist, use the broker's configuration to create the topic. - // It is less than the value of broker. - config.AutoCreate = true - config.MaxMessageBytes = defaultMaxMessageBytes - 1 - cfg, err = newSaramaConfigImpl(context.Background(), config) - c.Assert(err, check.IsNil) - opts = make(map[string]string) - err = validateMaxMessageBytesAndCreateTopic(adminClient, "create-new-success", config, cfg, opts) - c.Assert(err, check.IsNil) - c.Assert(cfg.Producer.MaxMessageBytes, check.Equals, config.MaxMessageBytes) - c.Assert(opts["max-message-bytes"], check.Equals, strconv.Itoa(cfg.Producer.MaxMessageBytes)) - - // When the topic does not exist, use the broker's configuration to create the topic. - // It is larger than the value of broker. - config.MaxMessageBytes = defaultMaxMessageBytes + 1 - config.AutoCreate = true - cfg, err = newSaramaConfigImpl(context.Background(), config) - c.Assert(err, check.IsNil) - opts = make(map[string]string) - err = validateMaxMessageBytesAndCreateTopic(adminClient, "create-new-fail", config, cfg, opts) - c.Assert(err, check.IsNil) - c.Assert(cfg.Producer.MaxMessageBytes, check.Equals, defaultMaxMessageBytes) - c.Assert(opts["max-message-bytes"], check.Equals, strconv.Itoa(cfg.Producer.MaxMessageBytes)) - - // When the topic exists, but the topic does not store max message bytes info, - // the check of parameter succeeds. - // It is less than the value of broker. - config.MaxMessageBytes = defaultMaxMessageBytes - 1 - cfg, err = newSaramaConfigImpl(context.Background(), config) - c.Assert(err, check.IsNil) - detail := &sarama.TopicDetail{ - NumPartitions: 3, - // Does not contain max message bytes information. - ConfigEntries: make(map[string]*string), - } - err = adminClient.CreateTopic("test-topic", detail, false) - c.Assert(err, check.IsNil) - opts = make(map[string]string) - err = validateMaxMessageBytesAndCreateTopic(adminClient, "test-topic", config, cfg, opts) - c.Assert(err, check.IsNil) - c.Assert(cfg.Producer.MaxMessageBytes, check.Equals, config.MaxMessageBytes) - c.Assert(opts["max-message-bytes"], check.Equals, strconv.Itoa(cfg.Producer.MaxMessageBytes)) - - // When the topic exists, but the topic does not store max message bytes info, - // the check of parameter fails. - // It is larger than the value of broker. - config.MaxMessageBytes = defaultMaxMessageBytes + 1 - cfg, err = newSaramaConfigImpl(context.Background(), config) - c.Assert(err, check.IsNil) - opts = make(map[string]string) - err = validateMaxMessageBytesAndCreateTopic(adminClient, "test-topic", config, cfg, opts) - c.Assert(err, check.IsNil) - c.Assert(cfg.Producer.MaxMessageBytes, check.Equals, defaultMaxMessageBytes) - c.Assert(opts["max-message-bytes"], check.Equals, strconv.Itoa(cfg.Producer.MaxMessageBytes)) -} - -func (s *kafkaSuite) TestCreateProducerFailed(c *check.C) { - defer testleak.AfterTest(c)() - ctx := context.Background() - errCh := make(chan error, 1) - config := NewConfig() - config.Version = "invalid" - config.BrokerEndpoints = []string{"127.0.0.1:1111"} - topic := "topic" - NewAdminClientImpl = kafka.NewMockAdminClient - defer func() { - NewAdminClientImpl = kafka.NewSaramaAdminClient - }() - opts := make(map[string]string) - _, err := NewKafkaSaramaProducer(ctx, topic, config, opts, errCh) - c.Assert(errors.Cause(err), check.ErrorMatches, "invalid version.*") -} - -func (s *kafkaSuite) TestProducerSendMessageFailed(c *check.C) { - defer testleak.AfterTest(c)() - topic := kafka.DefaultMockTopicName - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - - leader := sarama.NewMockBroker(c, 2) - defer leader.Close() - metadataResponse := new(sarama.MetadataResponse) - metadataResponse.AddBroker(leader.Addr(), leader.BrokerID()) - metadataResponse.AddTopicPartition(topic, 0, leader.BrokerID(), nil, nil, nil, sarama.ErrNoError) - metadataResponse.AddTopicPartition(topic, 1, leader.BrokerID(), nil, nil, nil, sarama.ErrNoError) - leader.Returns(metadataResponse) - leader.Returns(metadataResponse) - - config := NewConfig() - // Because the sarama mock broker is not compatible with version larger than 1.0.0 - // We use a smaller version in the following producer tests. - // Ref: https://github.com/Shopify/sarama/blob/89707055369768913defac030c15cf08e9e57925/async_producer_test.go#L1445-L1447 - config.Version = "0.9.0.0" - config.PartitionNum = int32(2) - config.AutoCreate = false - config.BrokerEndpoints = strings.Split(leader.Addr(), ",") - - NewAdminClientImpl = kafka.NewMockAdminClient - defer func() { - NewAdminClientImpl = kafka.NewSaramaAdminClient - }() - - newSaramaConfigImplBak := newSaramaConfigImpl - newSaramaConfigImpl = func(ctx context.Context, config *Config) (*sarama.Config, error) { - cfg, err := newSaramaConfigImplBak(ctx, config) - c.Assert(err, check.IsNil) - cfg.Producer.Flush.MaxMessages = 1 - cfg.Producer.Retry.Max = 2 - cfg.Producer.MaxMessageBytes = 8 - return cfg, err - } - defer func() { - newSaramaConfigImpl = newSaramaConfigImplBak - }() - - errCh := make(chan error, 1) - opts := make(map[string]string) - producer, err := NewKafkaSaramaProducer(ctx, topic, config, opts, errCh) - c.Assert(opts, check.HasKey, "max-message-bytes") - defer func() { - err := producer.Close() - c.Assert(err, check.IsNil) - }() - - c.Assert(err, check.IsNil) - c.Assert(producer, check.NotNil) - - var wg sync.WaitGroup - - wg.Add(1) - go func() { - defer wg.Done() - for i := 0; i < 20; i++ { - err = producer.AsyncSendMessage(ctx, &codec.MQMessage{ - Key: []byte("test-key-1"), - Value: []byte("test-value"), - }, int32(0)) - c.Assert(err, check.IsNil) - } - }() - - wg.Add(1) - go func() { - defer wg.Done() - select { - case <-ctx.Done(): - c.Fatal("TestProducerSendMessageFailed timed out") - case err := <-errCh: - c.Assert(err, check.ErrorMatches, ".*too large.*") - } - }() - - wg.Wait() -} - -func (s *kafkaSuite) TestProducerDoubleClose(c *check.C) { - defer testleak.AfterTest(c)() - topic := kafka.DefaultMockTopicName - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - - leader := sarama.NewMockBroker(c, 2) - defer leader.Close() - metadataResponse := new(sarama.MetadataResponse) - metadataResponse.AddBroker(leader.Addr(), leader.BrokerID()) - metadataResponse.AddTopicPartition(topic, 0, leader.BrokerID(), nil, nil, nil, sarama.ErrNoError) - metadataResponse.AddTopicPartition(topic, 1, leader.BrokerID(), nil, nil, nil, sarama.ErrNoError) - leader.Returns(metadataResponse) - leader.Returns(metadataResponse) - - config := NewConfig() - // Because the sarama mock broker is not compatible with version larger than 1.0.0 - // We use a smaller version in the following producer tests. - // Ref: https://github.com/Shopify/sarama/blob/89707055369768913defac030c15cf08e9e57925/async_producer_test.go#L1445-L1447 - config.Version = "0.9.0.0" - config.PartitionNum = int32(2) - config.AutoCreate = false - config.BrokerEndpoints = strings.Split(leader.Addr(), ",") - - NewAdminClientImpl = kafka.NewMockAdminClient - defer func() { - NewAdminClientImpl = kafka.NewSaramaAdminClient - }() - - errCh := make(chan error, 1) - opts := make(map[string]string) - producer, err := NewKafkaSaramaProducer(ctx, topic, config, opts, errCh) - c.Assert(opts, check.HasKey, "max-message-bytes") - defer func() { - err := producer.Close() - c.Assert(err, check.IsNil) - }() - - c.Assert(err, check.IsNil) - c.Assert(producer, check.NotNil) - - err = producer.Close() - c.Assert(err, check.IsNil) - - err = producer.Close() - c.Assert(err, check.IsNil) -} diff --git a/cdc/cdc/sink/producer/mq_producer.go b/cdc/cdc/sink/producer/mq_producer.go deleted file mode 100644 index e406af7d..00000000 --- a/cdc/cdc/sink/producer/mq_producer.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package producer - -import ( - "context" - - "github.com/tikv/migration/cdc/cdc/sink/codec" -) - -// Producer is an interface of mq producer -type Producer interface { - // AsyncSendMessage sends a message asynchronously. - AsyncSendMessage(ctx context.Context, message *codec.MQMessage, partition int32) error - // SyncBroadcastMessage broadcasts a message synchronously. - SyncBroadcastMessage(ctx context.Context, message *codec.MQMessage) error - // Flush all the messages buffered in the client and wait until all messages have been successfully - // persisted. - Flush(ctx context.Context) error - // GetPartitionNum gets partition number of topic. - GetPartitionNum() int32 - // Close closes the producer and client(s). - Close() error -} diff --git a/cdc/cdc/sink/producer/pulsar/doc.go b/cdc/cdc/sink/producer/pulsar/doc.go deleted file mode 100644 index 4e557239..00000000 --- a/cdc/cdc/sink/producer/pulsar/doc.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package pulsar provider a pulsar based mq Producer implementation. -// -// SinkURL format like: -// pulsar://{token}@{host}/{topic}?xx=xxx -// -// config options see links below: -// https://godoc.org/github.com/apache/pulsar-client-go/pulsar#ClientOptions -// https://godoc.org/github.com/apache/pulsar-client-go/pulsar#ProducerOptions -// -// Notice: -// 1. All option in url queries start with lowercase chars, e.g. `tlsAllowInsecureConnection`, `maxConnectionsPerBroker`. -// 2. Use `auth` to config authentication plugin type, `auth.*` to config auth params. -// See: -// 1. https://pulsar.apache.org/docs/en/reference-cli-tools/#pulsar-client -// 2. https://github.com/apache/pulsar-client-go/tree/master/pulsar/internal/auth -// -// For example: -// pulsar://{host}/{topic}?auth=token&auth.token={token} -package pulsar diff --git a/cdc/cdc/sink/producer/pulsar/option.go b/cdc/cdc/sink/producer/pulsar/option.go deleted file mode 100644 index 72359087..00000000 --- a/cdc/cdc/sink/producer/pulsar/option.go +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package pulsar - -import ( - "encoding/json" - "fmt" - "net/url" - "strconv" - "strings" - "time" - - "github.com/apache/pulsar-client-go/pulsar" -) - -// Option is pulsar producer's option. -type Option struct { - clientOptions *pulsar.ClientOptions - producerOptions *pulsar.ProducerOptions -} - -const route = "$route" - -func parseSinkOptions(u *url.URL) (opt *Option, err error) { - switch u.Scheme { - case "pulsar", "pulsar+ssl": - default: - return nil, fmt.Errorf("unsupported pulsar scheme: %s", u.Scheme) - } - c, err := parseClientOption(u) - if err != nil { - return nil, err - } - p := parseProducerOptions(u) - opt = &Option{ - clientOptions: c, - producerOptions: p, - } - - p.MessageRouter = func(message *pulsar.ProducerMessage, metadata pulsar.TopicMetadata) int { - partition, _ := strconv.Atoi(message.Properties[route]) - delete(message.Properties, route) - return partition - } - return -} - -func parseClientOption(u *url.URL) (opt *pulsar.ClientOptions, err error) { - vs := values(u.Query()) - opt = &pulsar.ClientOptions{ - URL: (&url.URL{Scheme: u.Scheme, Host: u.Host}).String(), - ConnectionTimeout: vs.Duration("connectionTimeout"), - OperationTimeout: vs.Duration("operationTimeout"), - TLSTrustCertsFilePath: vs.Str("tlsTrustCertsFilePath"), - TLSAllowInsecureConnection: vs.Bool("tlsAllowInsecureConnection"), - TLSValidateHostname: vs.Bool("tlsValidateHostname"), - MaxConnectionsPerBroker: vs.Int("maxConnectionsPerBroker"), - } - auth := vs.Str("auth") - if auth == "" { - if u.User.Username() == "" { - // no auth - return opt, nil - } - // use token provider by default - opt.Authentication = pulsar.NewAuthenticationToken(u.User.Username()) - return opt, nil - } - param := jsonStr(vs.SubPathKV("auth")) - opt.Authentication, err = pulsar.NewAuthentication(auth, param) - if err != nil { - return nil, err - } - return opt, nil -} - -func parseProducerOptions(u *url.URL) (opt *pulsar.ProducerOptions) { - vs := values(u.Query()) - opt = &pulsar.ProducerOptions{ - Name: vs.Str("name"), - MaxPendingMessages: vs.Int("maxPendingMessages"), - DisableBatching: vs.Bool("disableBatching"), - BatchingMaxPublishDelay: vs.Duration("batchingMaxPublishDelay"), - BatchingMaxMessages: uint(vs.Int("tlsAllowInsecureConnection")), - Properties: vs.SubPathKV("properties"), - } - hashingScheme := vs.Str("hashingScheme") - switch hashingScheme { - case "JavaStringHash", "": - opt.HashingScheme = pulsar.JavaStringHash - case "Murmur3_32Hash": - opt.HashingScheme = pulsar.Murmur3_32Hash - } - compressionType := vs.Str("compressionType") - switch compressionType { - case "LZ4": - opt.CompressionType = pulsar.LZ4 - case "ZLib": - opt.CompressionType = pulsar.ZLib - case "ZSTD": - opt.CompressionType = pulsar.ZSTD - } - switch u.Path { - case "", "/": - opt.Topic = vs.Str("topic") - default: - opt.Topic = strings.Trim(u.Path, "/") - } - return opt -} - -type values url.Values - -func (vs values) Int(name string) int { - value, ok := vs[name] - if !ok { - return 0 - } - if len(value) == 0 { - return 0 - } - v, _ := strconv.Atoi(value[0]) - return v -} - -func (vs values) Duration(name string) time.Duration { - value, ok := vs[name] - if !ok { - return 0 - } - if len(value) == 0 { - return 0 - } - v, _ := time.ParseDuration(value[0]) - return v -} - -func (vs values) Bool(name string) bool { - value, ok := vs[name] - if !ok { - return false - } - if len(value) == 0 { - return true - } - v, _ := strconv.ParseBool(value[0]) - return v -} - -func (vs values) Str(name string) string { - value, ok := vs[name] - if !ok { - return "" - } - if len(value) == 0 { - return "" - } - return value[0] -} - -func (vs values) SubPathKV(prefix string) map[string]string { - prefix = prefix + "." - m := map[string]string{} - for name, value := range vs { - if !strings.HasPrefix(name, prefix) { - continue - } - var v string - if len(value) != 0 { - v = value[0] - } - m[name[len(prefix):]] = v - } - return m -} - -func jsonStr(m interface{}) string { - data, _ := json.Marshal(m) - return string(data) -} diff --git a/cdc/cdc/sink/producer/pulsar/producer.go b/cdc/cdc/sink/producer/pulsar/producer.go deleted file mode 100644 index 80b5e5cc..00000000 --- a/cdc/cdc/sink/producer/pulsar/producer.go +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package pulsar - -import ( - "context" - "net/url" - "strconv" - - "github.com/apache/pulsar-client-go/pulsar" - "github.com/pingcap/failpoint" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/sink/codec" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "go.uber.org/zap" -) - -// NewProducer create a pulsar producer. -func NewProducer(u *url.URL, errCh chan error) (*Producer, error) { - failpoint.Inject("MockPulsar", func() { - failpoint.Return(&Producer{ - errCh: errCh, - partitionNum: 4, - }, nil) - }) - - opt, err := parseSinkOptions(u) - if err != nil { - return nil, cerror.WrapError(cerror.ErrPulsarNewProducer, err) - } - client, err := pulsar.NewClient(*opt.clientOptions) - if err != nil { - return nil, cerror.WrapError(cerror.ErrPulsarNewProducer, err) - } - producer, err := client.CreateProducer(*opt.producerOptions) - if err != nil { - client.Close() - return nil, cerror.WrapError(cerror.ErrPulsarNewProducer, err) - } - partitions, err := client.TopicPartitions(opt.producerOptions.Topic) - if err != nil { - client.Close() - return nil, cerror.WrapError(cerror.ErrPulsarNewProducer, err) - } - return &Producer{ - errCh: errCh, - opt: *opt, - client: client, - producer: producer, - partitionNum: len(partitions), - }, nil -} - -// Producer provide a way to send msg to pulsar. -type Producer struct { - opt Option - client pulsar.Client - producer pulsar.Producer - errCh chan error - partitionNum int -} - -func createProperties(message *codec.MQMessage, partition int32) map[string]string { - properties := map[string]string{route: strconv.Itoa(int(partition))} - properties["ts"] = strconv.FormatUint(message.Ts, 10) - properties["type"] = strconv.Itoa(int(message.Type)) - properties["protocol"] = strconv.Itoa(int(message.Protocol)) - if message.Schema != nil { - properties["schema"] = *message.Schema - } - if message.Table != nil { - properties["table"] = *message.Table - } - return properties -} - -// SendMessage send key-value msg to target partition. -func (p *Producer) AsyncSendMessage(ctx context.Context, message *codec.MQMessage, partition int32) error { - p.producer.SendAsync(ctx, &pulsar.ProducerMessage{ - Payload: message.Value, - Key: string(message.Key), - Properties: createProperties(message, partition), - EventTime: message.PhysicalTime(), - }, p.errors) - return nil -} - -func (p *Producer) errors(_ pulsar.MessageID, _ *pulsar.ProducerMessage, err error) { - if err != nil { - select { - case p.errCh <- cerror.WrapError(cerror.ErrPulsarSendMessage, err): - default: - log.Error("error channel is full", zap.Error(err)) - } - } -} - -// SyncBroadcastMessage send key-value msg to all partition. -func (p *Producer) SyncBroadcastMessage(ctx context.Context, message *codec.MQMessage) error { - for partition := 0; partition < p.partitionNum; partition++ { - _, err := p.producer.Send(ctx, &pulsar.ProducerMessage{ - Payload: message.Value, - Key: string(message.Key), - Properties: createProperties(message, int32(partition)), - EventTime: message.PhysicalTime(), - }) - if err != nil { - return cerror.WrapError(cerror.ErrPulsarSendMessage, p.producer.Flush()) - } - } - return nil -} - -// Flush flushes all in memory msgs to server. -func (p *Producer) Flush(_ context.Context) error { - return cerror.WrapError(cerror.ErrPulsarSendMessage, p.producer.Flush()) -} - -// GetPartitionNum got current topic's partition size. -func (p *Producer) GetPartitionNum() int32 { - return int32(p.partitionNum) -} - -// Close closes the producer and client. -func (p *Producer) Close() error { - err := p.producer.Flush() - if err != nil { - return cerror.WrapError(cerror.ErrPulsarSendMessage, err) - } - p.producer.Close() - p.client.Close() - return nil -} diff --git a/cdc/cdc/sink/simple_mysql_tester.go b/cdc/cdc/sink/simple_mysql_tester.go deleted file mode 100644 index 0b191a43..00000000 --- a/cdc/cdc/sink/simple_mysql_tester.go +++ /dev/null @@ -1,262 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package sink - -import ( - "context" - "database/sql" - "fmt" - "net/url" - "strings" - "sync" - - dmysql "github.com/go-sql-driver/mysql" - "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/config" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/errorutil" - "github.com/tikv/migration/cdc/pkg/filter" - "github.com/tikv/migration/cdc/pkg/quotes" - "go.uber.org/zap" -) - -func init() { - failpoint.Inject("SimpleMySQLSinkTester", func() { - sinkIniterMap["simple-mysql"] = func(ctx context.Context, changefeedID model.ChangeFeedID, sinkURI *url.URL, - filter *filter.Filter, config *config.ReplicaConfig, opts map[string]string, errCh chan error) (Sink, error) { - return newSimpleMySQLSink(ctx, sinkURI, config) - } - }) -} - -type simpleMySQLSink struct { - enableOldValue bool - enableCheckOldValue bool - db *sql.DB - rowsBuffer []*model.RowChangedEvent - rowsBufferLock sync.Mutex -} - -func newSimpleMySQLSink(ctx context.Context, sinkURI *url.URL, config *config.ReplicaConfig) (*simpleMySQLSink, error) { - var db *sql.DB - - // dsn format of the driver: - // [username[:password]@][protocol[(address)]]/dbname[?param1=value1&...¶mN=valueN] - username := sinkURI.User.Username() - password, _ := sinkURI.User.Password() - port := sinkURI.Port() - if username == "" { - username = "root" - } - if port == "" { - port = "4000" - } - - dsnStr := fmt.Sprintf("%s:%s@tcp(%s:%s)/?multiStatements=true", username, password, sinkURI.Hostname(), port) - dsn, err := dmysql.ParseDSN(dsnStr) - if err != nil { - return nil, cerror.WrapError(cerror.ErrMySQLInvalidConfig, err) - } - - // create test db used for parameter detection - if dsn.Params == nil { - dsn.Params = make(map[string]string, 1) - } - testDB, err := sql.Open("mysql", dsn.FormatDSN()) - if err != nil { - return nil, errors.Annotate( - cerror.WrapError(cerror.ErrMySQLConnectionError, err), "fail to open MySQL connection when configuring sink") - } - defer testDB.Close() - - db, err = sql.Open("mysql", dsnStr) - if err != nil { - return nil, errors.Annotate( - cerror.WrapError(cerror.ErrMySQLConnectionError, err), "fail to open MySQL connection") - } - err = db.PingContext(ctx) - if err != nil { - return nil, errors.Annotate( - cerror.WrapError(cerror.ErrMySQLConnectionError, err), "fail to open MySQL connection") - } - - sink := &simpleMySQLSink{ - db: db, - enableOldValue: config.EnableOldValue, - } - if strings.ToLower(sinkURI.Query().Get("check-old-value")) == "true" { - sink.enableCheckOldValue = true - log.Info("the old value checker is enabled") - } - return sink, nil -} - -// EmitRowChangedEvents sends Row Changed Event to Sink -// EmitRowChangedEvents may write rows to downstream directly; -func (s *simpleMySQLSink) EmitRowChangedEvents(ctx context.Context, rows ...*model.RowChangedEvent) error { - s.rowsBufferLock.Lock() - defer s.rowsBufferLock.Unlock() - s.rowsBuffer = append(s.rowsBuffer, rows...) - return nil -} - -func (s *simpleMySQLSink) executeRowChangedEvents(ctx context.Context, rows ...*model.RowChangedEvent) error { - var sql string - var args []interface{} - if s.enableOldValue { - for _, row := range rows { - if len(row.PreColumns) != 0 && len(row.Columns) != 0 { - // update - if s.enableCheckOldValue { - err := s.checkOldValue(ctx, row) - if err != nil { - return errors.Trace(err) - } - } - sql, args = prepareReplace(row.Table.QuoteString(), row.Columns, true, false /* translateToInsert */) - } else if len(row.PreColumns) == 0 { - // insert - sql, args = prepareReplace(row.Table.QuoteString(), row.Columns, true, false /* translateToInsert */) - } else if len(row.Columns) == 0 { - // delete - if s.enableCheckOldValue { - err := s.checkOldValue(ctx, row) - if err != nil { - return errors.Trace(err) - } - } - sql, args = prepareDelete(row.Table.QuoteString(), row.PreColumns, true) - } - _, err := s.db.ExecContext(ctx, sql, args...) - if err != nil { - return errors.Trace(err) - } - } - } else { - for _, row := range rows { - if row.IsDelete() { - sql, args = prepareDelete(row.Table.QuoteString(), row.PreColumns, true) - } else { - sql, args = prepareReplace(row.Table.QuoteString(), row.Columns, true, false) - } - _, err := s.db.ExecContext(ctx, sql, args...) - if err != nil { - return errors.Trace(err) - } - } - } - return nil -} - -// EmitDDLEvent sends DDL Event to Sink -// EmitDDLEvent should execute DDL to downstream synchronously -func (s *simpleMySQLSink) EmitDDLEvent(ctx context.Context, ddl *model.DDLEvent) error { - var sql string - if len(ddl.TableInfo.Table) == 0 { - sql = ddl.Query - } else { - sql = fmt.Sprintf("use %s;%s", ddl.TableInfo.Schema, ddl.Query) - } - _, err := s.db.ExecContext(ctx, sql) - if err != nil && errorutil.IsIgnorableMySQLDDLError(err) { - log.Info("execute DDL failed, but error can be ignored", zap.String("query", ddl.Query), zap.Error(err)) - return nil - } - return err -} - -// FlushRowChangedEvents flushes each row which of commitTs less than or equal to `resolvedTs` into downstream. -// TiCDC guarantees that all of Event which of commitTs less than or equal to `resolvedTs` are sent to Sink through `EmitRowChangedEvents` -func (s *simpleMySQLSink) FlushRowChangedEvents(ctx context.Context, _ model.TableID, resolvedTs uint64) (uint64, error) { - s.rowsBufferLock.Lock() - defer s.rowsBufferLock.Unlock() - newBuffer := make([]*model.RowChangedEvent, 0, len(s.rowsBuffer)) - for _, row := range s.rowsBuffer { - if row.CommitTs <= resolvedTs { - err := s.executeRowChangedEvents(ctx, row) - if err != nil { - return 0, err - } - } else { - newBuffer = append(newBuffer, row) - } - } - s.rowsBuffer = newBuffer - return resolvedTs, nil -} - -// EmitCheckpointTs sends CheckpointTs to Sink -// TiCDC guarantees that all Events **in the cluster** which of commitTs less than or equal `checkpointTs` are sent to downstream successfully. -func (s *simpleMySQLSink) EmitCheckpointTs(ctx context.Context, ts uint64) error { - // do nothing - return nil -} - -// Close closes the Sink -func (s *simpleMySQLSink) Close(ctx context.Context) error { - return s.db.Close() -} - -func (s *simpleMySQLSink) Barrier(ctx context.Context, tableID model.TableID) error { - return nil -} - -func prepareCheckSQL(quoteTable string, cols []*model.Column) (string, []interface{}) { - var builder strings.Builder - builder.WriteString("SELECT count(1) FROM " + quoteTable + " WHERE ") - - colNames, wargs := whereSlice(cols, true) - if len(wargs) == 0 { - return "", nil - } - args := make([]interface{}, 0, len(wargs)) - for i := 0; i < len(colNames); i++ { - if i > 0 { - builder.WriteString(" AND ") - } - if wargs[i] == nil { - builder.WriteString(quotes.QuoteName(colNames[i]) + " IS NULL") - } else { - builder.WriteString(quotes.QuoteName(colNames[i]) + " = ?") - args = append(args, wargs[i]) - } - } - builder.WriteString(" LIMIT 1;") - sql := builder.String() - return sql, args -} - -func (s *simpleMySQLSink) checkOldValue(ctx context.Context, row *model.RowChangedEvent) error { - sql, args := prepareCheckSQL(row.Table.QuoteString(), row.PreColumns) - result, err := s.db.QueryContext(ctx, sql, args...) - if err != nil { - return errors.Trace(err) - } - var count int - if result.Next() { - err := result.Scan(&count) - if err != nil { - return errors.Trace(err) - } - } - if count == 0 { - log.Error("can't pass the check, the old value of this row is not exist", zap.Any("row", row)) - return errors.New("check failed") - } - log.Debug("pass the old value check", zap.String("sql", sql), zap.Any("args", args), zap.Int("count", count)) - return nil -} diff --git a/cdc/cdc/sink/sink.go b/cdc/cdc/sink/sink.go index 71559f94..f87ca3d8 100644 --- a/cdc/cdc/sink/sink.go +++ b/cdc/cdc/sink/sink.go @@ -21,7 +21,6 @@ import ( "github.com/tikv/migration/cdc/cdc/model" "github.com/tikv/migration/cdc/pkg/config" cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/filter" ) // Sink options keys @@ -32,19 +31,19 @@ const ( // Sink is an abstraction for anything that a changefeed may emit into. type Sink interface { - // EmitRowChangedEvents sends Row Changed Event to Sink + // Emit,owChangedEvents sends Row Changed Event to Sink // EmitRowChangedEvents may write rows to downstream directly; // // EmitRowChangedEvents is thread-safe. // FIXME: some sink implementation, they should be. - EmitRowChangedEvents(ctx context.Context, rows ...*model.RowChangedEvent) error + EmitChangedEvents(ctx context.Context, rawKVEntries ...*model.RawKVEntry) error // EmitDDLEvent sends DDL Event to Sink // EmitDDLEvent should execute DDL to downstream synchronously // // EmitDDLEvent is thread-safe. // FIXME: some sink implementation, they should be. - EmitDDLEvent(ctx context.Context, ddl *model.DDLEvent) error + // EmitDDLEvent(ctx context.Context, ddl *model.DDLEvent) error // FlushRowChangedEvents flushes each row which of commitTs less than or // equal to `resolvedTs` into downstream. @@ -53,7 +52,7 @@ type Sink interface { // // FlushRowChangedEvents is thread-safe. // FIXME: some sink implementation, they should be. - FlushRowChangedEvents(ctx context.Context, tableID model.TableID, resolvedTs uint64) (uint64, error) + FlushChangedEvents(ctx context.Context, keyspanID model.KeySpanID, resolvedTs uint64) (uint64, error) // EmitCheckpointTs sends CheckpointTs to Sink. // TiCDC guarantees that all Events **in the cluster** which of commitTs @@ -74,66 +73,44 @@ type Sink interface { // the Barrier call returns. // // Barrier is thread-safe. - Barrier(ctx context.Context, tableID model.TableID) error + Barrier(ctx context.Context, keyspanID model.KeySpanID) error } var sinkIniterMap = make(map[string]sinkInitFunc) -type sinkInitFunc func(context.Context, model.ChangeFeedID, *url.URL, *filter.Filter, *config.ReplicaConfig, map[string]string, chan error) (Sink, error) +type sinkInitFunc func(context.Context, model.ChangeFeedID, *url.URL, *config.ReplicaConfig, map[string]string, chan error) (Sink, error) func init() { // register blackhole sink sinkIniterMap["blackhole"] = func(ctx context.Context, changefeedID model.ChangeFeedID, sinkURI *url.URL, - filter *filter.Filter, config *config.ReplicaConfig, opts map[string]string, errCh chan error) (Sink, error) { + config *config.ReplicaConfig, opts map[string]string, errCh chan error) (Sink, error) { return newBlackHoleSink(ctx, opts), nil } - // register mysql sink - sinkIniterMap["mysql"] = func(ctx context.Context, changefeedID model.ChangeFeedID, sinkURI *url.URL, - filter *filter.Filter, config *config.ReplicaConfig, opts map[string]string, errCh chan error) (Sink, error) { - return newMySQLSink(ctx, changefeedID, sinkURI, filter, config, opts) + sinkIniterMap["tikv"] = func(ctx context.Context, changefeedID model.ChangeFeedID, sinkURI *url.URL, + config *config.ReplicaConfig, opts map[string]string, errCh chan error) (Sink, error) { + return newTiKVSink(ctx, sinkURI, config, opts, errCh) } - sinkIniterMap["tidb"] = sinkIniterMap["mysql"] - sinkIniterMap["mysql+ssl"] = sinkIniterMap["mysql"] - sinkIniterMap["tidb+ssl"] = sinkIniterMap["mysql"] - - // register kafka sink - sinkIniterMap["kafka"] = func(ctx context.Context, changefeedID model.ChangeFeedID, sinkURI *url.URL, - filter *filter.Filter, config *config.ReplicaConfig, opts map[string]string, errCh chan error) (Sink, error) { - return newKafkaSaramaSink(ctx, sinkURI, filter, config, opts, errCh) - } - sinkIniterMap["kafka+ssl"] = sinkIniterMap["kafka"] - - // register pulsar sink - sinkIniterMap["pulsar"] = func(ctx context.Context, changefeedID model.ChangeFeedID, sinkURI *url.URL, - filter *filter.Filter, config *config.ReplicaConfig, opts map[string]string, errCh chan error) (Sink, error) { - return newPulsarSink(ctx, sinkURI, filter, config, opts, errCh) - } - sinkIniterMap["pulsar+ssl"] = sinkIniterMap["pulsar"] } // New creates a new sink with the sink-uri -func New(ctx context.Context, changefeedID model.ChangeFeedID, sinkURIStr string, filter *filter.Filter, config *config.ReplicaConfig, opts map[string]string, errCh chan error) (Sink, error) { +func New(ctx context.Context, changefeedID model.ChangeFeedID, sinkURIStr string, config *config.ReplicaConfig, opts map[string]string, errCh chan error) (Sink, error) { // parse sinkURI as a URI sinkURI, err := url.Parse(sinkURIStr) if err != nil { return nil, cerror.WrapError(cerror.ErrSinkURIInvalid, err) } if newSink, ok := sinkIniterMap[strings.ToLower(sinkURI.Scheme)]; ok { - return newSink(ctx, changefeedID, sinkURI, filter, config, opts, errCh) + return newSink(ctx, changefeedID, sinkURI, config, opts, errCh) } return nil, cerror.ErrSinkURIInvalid.GenWithStack("the sink scheme (%s) is not supported", sinkURI.Scheme) } // Validate sink if given valid parameters. func Validate(ctx context.Context, sinkURI string, cfg *config.ReplicaConfig, opts map[string]string) error { - sinkFilter, err := filter.NewFilter(cfg) - if err != nil { - return err - } errCh := make(chan error) // TODO: find a better way to verify a sinkURI is valid - s, err := New(ctx, "sink-verify", sinkURI, sinkFilter, cfg, opts, errCh) + s, err := New(ctx, "sink-verify", sinkURI, cfg, opts, errCh) if err != nil { return err } diff --git a/cdc/cdc/sink/sink_test.go b/cdc/cdc/sink/sink_test.go index 78ca7788..130f76c5 100644 --- a/cdc/cdc/sink/sink_test.go +++ b/cdc/cdc/sink/sink_test.go @@ -31,13 +31,11 @@ func TestValidateSink(t *testing.T) { replicateConfig := config.GetDefaultReplicaConfig() opts := make(map[string]string) - // test sink uri error - sinkURI := "mysql://root:111@127.0.0.1:3306/" + // test sink uri right + sinkURI := "tikv://127.0.0.1:3306/" err := Validate(ctx, sinkURI, replicateConfig, opts) - require.NotNil(t, err) - require.Contains(t, err.Error(), "fail to open MySQL connection") + require.Nil(t, err) - // test sink uri right sinkURI = "blackhole://" err = Validate(ctx, sinkURI, replicateConfig, opts) require.Nil(t, err) diff --git a/cdc/cdc/sink/syncpointStore.go b/cdc/cdc/sink/syncpointStore.go deleted file mode 100644 index ac25e224..00000000 --- a/cdc/cdc/sink/syncpointStore.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package sink - -import ( - "context" - "net/url" - "strings" - - "github.com/tikv/migration/cdc/cdc/model" - cerror "github.com/tikv/migration/cdc/pkg/errors" -) - -// SyncpointStore is an abstraction for anything that a changefeed may emit into. -type SyncpointStore interface { - // CreateSynctable create a table to record the syncpoints - CreateSynctable(ctx context.Context) error - - // SinkSyncpoint record the syncpoint(a map with ts) in downstream db - SinkSyncpoint(ctx context.Context, id string, checkpointTs uint64) error - - // Close closes the SyncpointSink - Close() error -} - -// NewSyncpointStore creates a new Spyncpoint sink with the sink-uri -func NewSyncpointStore(ctx context.Context, changefeedID model.ChangeFeedID, sinkURIStr string) (SyncpointStore, error) { - // parse sinkURI as a URI - sinkURI, err := url.Parse(sinkURIStr) - if err != nil { - return nil, cerror.WrapError(cerror.ErrSinkURIInvalid, err) - } - switch strings.ToLower(sinkURI.Scheme) { - case "mysql", "tidb", "mysql+ssl", "tidb+ssl": - return newMySQLSyncpointStore(ctx, changefeedID, sinkURI) - default: - return nil, cerror.ErrSinkURIInvalid.GenWithStack("the sink scheme (%s) is not supported", sinkURI.Scheme) - } -} diff --git a/cdc/cdc/sink/table_sink.go b/cdc/cdc/sink/table_sink.go deleted file mode 100644 index bc4c0f35..00000000 --- a/cdc/cdc/sink/table_sink.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package sink - -import ( - "context" - "sort" - - "github.com/pingcap/errors" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/redo" - "go.uber.org/zap" -) - -type tableSink struct { - tableID model.TableID - manager *Manager - buffer []*model.RowChangedEvent - redoManager redo.LogManager -} - -var _ Sink = (*tableSink)(nil) - -func (t *tableSink) EmitRowChangedEvents(ctx context.Context, rows ...*model.RowChangedEvent) error { - t.buffer = append(t.buffer, rows...) - t.manager.metricsTableSinkTotalRows.Add(float64(len(rows))) - if t.redoManager.Enabled() { - return t.redoManager.EmitRowChangedEvents(ctx, t.tableID, rows...) - } - return nil -} - -func (t *tableSink) EmitDDLEvent(ctx context.Context, ddl *model.DDLEvent) error { - // the table sink doesn't receive the DDL event - return nil -} - -// FlushRowChangedEvents flushes sorted rows to sink manager, note the resolvedTs -// is required to be no more than global resolvedTs, table barrierTs and table -// redo log watermarkTs. -func (t *tableSink) FlushRowChangedEvents(ctx context.Context, tableID model.TableID, resolvedTs uint64) (uint64, error) { - if tableID != t.tableID { - log.Panic("inconsistent table sink", - zap.Int64("tableID", tableID), zap.Int64("sinkTableID", t.tableID)) - } - i := sort.Search(len(t.buffer), func(i int) bool { - return t.buffer[i].CommitTs > resolvedTs - }) - if i == 0 { - return t.flushResolvedTs(ctx, resolvedTs) - } - resolvedRows := t.buffer[:i] - t.buffer = append(make([]*model.RowChangedEvent, 0, len(t.buffer[i:])), t.buffer[i:]...) - - err := t.manager.bufSink.EmitRowChangedEvents(ctx, resolvedRows...) - if err != nil { - return t.manager.getCheckpointTs(tableID), errors.Trace(err) - } - return t.flushResolvedTs(ctx, resolvedTs) -} - -func (t *tableSink) flushResolvedTs(ctx context.Context, resolvedTs uint64) (uint64, error) { - redoTs, err := t.flushRedoLogs(ctx, resolvedTs) - if err != nil { - return t.manager.getCheckpointTs(t.tableID), err - } - if redoTs < resolvedTs { - resolvedTs = redoTs - } - return t.manager.flushBackendSink(ctx, t.tableID, resolvedTs) -} - -// flushRedoLogs flush redo logs and returns redo log resolved ts which means -// all events before the ts have been persisted to redo log storage. -func (t *tableSink) flushRedoLogs(ctx context.Context, resolvedTs uint64) (uint64, error) { - if t.redoManager.Enabled() { - err := t.redoManager.FlushLog(ctx, t.tableID, resolvedTs) - if err != nil { - return 0, err - } - return t.redoManager.GetMinResolvedTs(), nil - } - return resolvedTs, nil -} - -func (t *tableSink) EmitCheckpointTs(ctx context.Context, ts uint64) error { - // the table sink doesn't receive the checkpoint event - return nil -} - -// Close once the method is called, no more events can be written to this table sink -func (t *tableSink) Close(ctx context.Context) error { - return t.manager.destroyTableSink(ctx, t.tableID) -} - -// Barrier is not used in table sink -func (t *tableSink) Barrier(ctx context.Context, tableID model.TableID) error { - return nil -} diff --git a/cdc/cdc/sink/tikv.go b/cdc/cdc/sink/tikv.go new file mode 100644 index 00000000..84bc8850 --- /dev/null +++ b/cdc/cdc/sink/tikv.go @@ -0,0 +1,370 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package sink + +import ( + "context" + "net/url" + "strconv" + "sync/atomic" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/log" + "github.com/twmb/murmur3" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" + + tikvconfig "github.com/tikv/client-go/v2/config" + "github.com/tikv/client-go/v2/rawkv" + "github.com/tikv/migration/cdc/cdc/model" + "github.com/tikv/migration/cdc/pkg/config" + "github.com/tikv/migration/cdc/pkg/notify" +) + +const ( + defaultConcurrency uint32 = 4 + defaultTikvByteSizeLimit int64 = 4 * 1024 * 1024 // 4MB +) + +type tikvSink struct { + workerNum uint32 + workerInput []chan struct { + rawKVEntry *model.RawKVEntry + resolvedTs uint64 + } + workerResolvedTs []uint64 + checkpointTs uint64 + resolvedNotifier *notify.Notifier + resolvedReceiver *notify.Receiver + + config *tikvconfig.Config + pdAddr []string + opts map[string]string + + statistics *Statistics +} + +func createTiKVSink( + ctx context.Context, + config *tikvconfig.Config, + pdAddr []string, + opts map[string]string, + errCh chan error, +) (*tikvSink, error) { + workerNum := defaultConcurrency + if s, ok := opts["concurrency"]; ok { + c, _ := strconv.Atoi(s) + workerNum = uint32(c) + } + workerInput := make([]chan struct { + rawKVEntry *model.RawKVEntry + resolvedTs uint64 + }, workerNum) + for i := 0; i < int(workerNum); i++ { + workerInput[i] = make(chan struct { + rawKVEntry *model.RawKVEntry + resolvedTs uint64 + }, 12800) + } + + notifier := new(notify.Notifier) + resolvedReceiver, err := notifier.NewReceiver(50 * time.Millisecond) + if err != nil { + return nil, err + } + k := &tikvSink{ + workerNum: workerNum, + workerInput: workerInput, + workerResolvedTs: make([]uint64, workerNum), + resolvedNotifier: notifier, + resolvedReceiver: resolvedReceiver, + + config: config, + pdAddr: pdAddr, + opts: opts, + + statistics: NewStatistics(ctx, "TiKVSink", opts), + } + + go func() { + if err := k.run(ctx); err != nil && errors.Cause(err) != context.Canceled { + select { + case <-ctx.Done(): + return + case errCh <- err: + default: + log.Error("error channel is full", zap.Error(err)) + } + } + }() + return k, nil +} + +func (k *tikvSink) dispatch(entry *model.RawKVEntry) uint32 { + hasher := murmur3.New32() + hasher.Write(entry.Key) + return hasher.Sum32() % k.workerNum +} + +func (k *tikvSink) EmitChangedEvents(ctx context.Context, rawKVEntries ...*model.RawKVEntry) error { + // log.Debug("(rawkv)tikvSink::EmitRowChangedEvents", zap.Any("events", events)) + rowsCount := 0 + for _, rawKVEntry := range rawKVEntries { + workerIdx := k.dispatch(rawKVEntry) + select { + case <-ctx.Done(): + return ctx.Err() + case k.workerInput[workerIdx] <- struct { + rawKVEntry *model.RawKVEntry + resolvedTs uint64 + }{rawKVEntry: rawKVEntry}: + } + rowsCount++ + } + k.statistics.AddRowsCount(rowsCount) + return nil +} + +func (k *tikvSink) FlushChangedEvents(ctx context.Context, keyspanID model.KeySpanID, resolvedTs uint64) (uint64, error) { + log.Debug("(rawkv)tikvSink::FlushRowChangedEvents", zap.Uint64("resolvedTs", resolvedTs), zap.Uint64("checkpointTs", k.checkpointTs)) + if resolvedTs <= k.checkpointTs { + return k.checkpointTs, nil + } + + for i := 0; i < int(k.workerNum); i++ { + select { + case <-ctx.Done(): + return 0, ctx.Err() + case k.workerInput[i] <- struct { + rawKVEntry *model.RawKVEntry + resolvedTs uint64 + }{resolvedTs: resolvedTs}: + } + } + + // waiting for all row events are sent to TiKV +flushLoop: + for { + select { + case <-ctx.Done(): + return 0, ctx.Err() + case <-k.resolvedReceiver.C: + for i := 0; i < int(k.workerNum); i++ { + if resolvedTs > atomic.LoadUint64(&k.workerResolvedTs[i]) { + continue flushLoop + } + } + break flushLoop + } + } + k.checkpointTs = resolvedTs + k.statistics.PrintStatus(ctx) + return k.checkpointTs, nil +} + +func (k *tikvSink) EmitCheckpointTs(ctx context.Context, ts uint64) error { + return nil +} + +func (k *tikvSink) EmitDDLEvent(ctx context.Context, ddl *model.DDLEvent) error { + return nil +} + +// Initialize registers Avro schemas for all tables +func (k *tikvSink) Initialize(ctx context.Context, tableInfo []*model.SimpleTableInfo) error { + // No longer need it for now + return nil +} + +func (k *tikvSink) Close(ctx context.Context) error { + return nil +} + +func (k *tikvSink) Barrier(cxt context.Context, keyspanID model.KeySpanID) error { + // Barrier does nothing because FlushRowChangedEvents in mq sink has flushed + // all buffered events forcedlly. + return nil +} + +func (k *tikvSink) run(ctx context.Context) error { + defer k.resolvedReceiver.Stop() + wg, ctx := errgroup.WithContext(ctx) + for i := uint32(0); i < k.workerNum; i++ { + workerIdx := i + wg.Go(func() error { + return k.runWorker(ctx, workerIdx) + }) + } + return wg.Wait() +} + +type innerBatch struct { + OpType model.OpType + Keys [][]byte + Values [][]byte +} + +type tikvBatcher struct { + Batches map[model.OpType]*innerBatch + count int + byteSize int64 +} + +func (b *tikvBatcher) Count() int { + return b.count +} + +func (b *tikvBatcher) ByteSize() int64 { + return b.byteSize +} + +func (b *tikvBatcher) Append(entry *model.RawKVEntry) { + log.Debug("(rawkv)tikvBatch::Append", zap.Any("event", entry)) + + opType := entry.OpType + _, ok := b.Batches[opType] + if !ok { + b.Batches[opType] = &innerBatch{ + OpType: opType, + Keys: [][]byte{entry.Key}, + Values: [][]byte{entry.Value}, + } + } + b.Batches[opType].Keys = append(b.Batches[opType].Keys, entry.Key[1:]) + b.Batches[opType].Values = append(b.Batches[opType].Values, entry.Value[1:]) + + b.count += 1 + b.byteSize += int64(len(entry.Key) + len(entry.Value)) +} + +func (b *tikvBatcher) Reset() { + b.Batches = map[model.OpType]*innerBatch{} + b.count = 0 + b.byteSize = 0 +} + +func (k *tikvSink) runWorker(ctx context.Context, workerIdx uint32) error { + log.Info("(rawkv)tikvSink worker start", zap.Uint32("workerIdx", workerIdx)) + input := k.workerInput[workerIdx] + + cli, err := rawkv.NewClient(ctx, k.pdAddr, k.config.Security) + if err != nil { + return err + } + defer cli.Close() + + tick := time.NewTicker(500 * time.Millisecond) + defer tick.Stop() + + batcher := tikvBatcher{ + Batches: map[model.OpType]*innerBatch{}, + } + + flushToTiKV := func() error { + return k.statistics.RecordBatchExecution(func() (int, error) { + log.Debug("(rawkv)tikvSink::flushToTiKV", zap.Any("batches", batcher.Batches)) + thisBatchSize := batcher.Count() + if thisBatchSize == 0 { + return 0, nil + } + + for _, batch := range batcher.Batches { + var err error + if batch.OpType == model.OpTypePut { + err = cli.BatchPut(ctx, batch.Keys, batch.Values, nil) + } else if batch.OpType == model.OpTypeDelete { + err = cli.BatchDelete(ctx, batch.Keys) + } + if err != nil { + return 0, err + } + log.Debug("(rawkv)TiKVSink flushed", zap.Int("thisBatchSize", thisBatchSize), zap.Any("batch", batch)) + } + batcher.Reset() + return thisBatchSize, nil + }) + } + for { + var e struct { + rawKVEntry *model.RawKVEntry + resolvedTs uint64 + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-tick.C: + if err := flushToTiKV(); err != nil { + return errors.Trace(err) + } + continue + case e = <-input: + } + if e.rawKVEntry == nil { + if e.resolvedTs != 0 { + log.Debug("(rawkv)tikvSink::runWorker push workerResolvedTs", zap.Uint32("workerIdx", workerIdx), zap.Uint64("event.resolvedTs", e.resolvedTs)) + if err := flushToTiKV(); err != nil { + return errors.Trace(err) + } + + atomic.StoreUint64(&k.workerResolvedTs[workerIdx], e.resolvedTs) + k.resolvedNotifier.Notify() + } + continue + } + log.Debug("(rawkv)tikvSink::runWorker append event", zap.Uint32("workerIdx", workerIdx), zap.Any("event", e.rawKVEntry)) + batcher.Append(e.rawKVEntry) + + if batcher.ByteSize() >= defaultTikvByteSizeLimit { + if err := flushToTiKV(); err != nil { + return errors.Trace(err) + } + } + } +} + +func parseTiKVUri(sinkURI *url.URL, opts map[string]string) (*tikvconfig.Config, []string, error) { + config := tikvconfig.DefaultConfig() + + var pdAddr []string + if sinkURI.Opaque != "" { + pdAddr = append(pdAddr, "http://"+sinkURI.Opaque) + } else { + pdAddr = append(pdAddr, "http://127.0.0.1:2379") + } + + s := sinkURI.Query().Get("concurrency") + if s != "" { + _, err := strconv.Atoi(s) + if err != nil { + return nil, nil, err + } + opts["concurrency"] = s + } + + return &config, pdAddr, nil +} + +func newTiKVSink(ctx context.Context, sinkURI *url.URL, _ *config.ReplicaConfig, opts map[string]string, errCh chan error) (*tikvSink, error) { + config, pdAddr, err := parseTiKVUri(sinkURI, opts) + if err != nil { + return nil, errors.Trace(err) + } + + sink, err := createTiKVSink(ctx, config, pdAddr, opts, errCh) + if err != nil { + return nil, errors.Trace(err) + } + return sink, nil +} diff --git a/cdc/cdc/sink/txns_heap.go b/cdc/cdc/sink/txns_heap.go deleted file mode 100644 index ba6a5a87..00000000 --- a/cdc/cdc/sink/txns_heap.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package sink - -import ( - "container/heap" - - "github.com/tikv/migration/cdc/cdc/model" -) - -type innerTxnsHeap []innerHeapEntry - -type innerHeapEntry struct { - ts uint64 - bucket int -} - -func (h innerTxnsHeap) Len() int { return len(h) } -func (h innerTxnsHeap) Less(i, j int) bool { return h[i].ts < h[j].ts } -func (h innerTxnsHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } - -func (h *innerTxnsHeap) Push(x interface{}) { - // Push and Pop use pointer receivers because they modify the slice's length, - // not just its contents. - *h = append(*h, x.(innerHeapEntry)) -} - -func (h *innerTxnsHeap) Pop() interface{} { - old := *h - n := len(old) - x := old[n-1] - *h = old[0 : n-1] - return x -} - -type txnsHeap struct { - inner *innerTxnsHeap - txnsGroup [][]*model.SingleTableTxn -} - -func newTxnsHeap(txnsMap map[model.TableID][]*model.SingleTableTxn) *txnsHeap { - txnsGroup := make([][]*model.SingleTableTxn, 0, len(txnsMap)) - for _, txns := range txnsMap { - txnsGroup = append(txnsGroup, txns) - } - inner := make(innerTxnsHeap, 0, len(txnsGroup)) - heap.Init(&inner) - for bucket, txns := range txnsGroup { - if len(txns) == 0 { - continue - } - entry := innerHeapEntry{ts: txns[0].CommitTs, bucket: bucket} - heap.Push(&inner, entry) - } - return &txnsHeap{inner: &inner, txnsGroup: txnsGroup} -} - -func (h *txnsHeap) iter(fn func(txn *model.SingleTableTxn)) { - for { - if h.inner.Len() == 0 { - break - } - minEntry := heap.Pop(h.inner).(innerHeapEntry) - bucket := minEntry.bucket - fn(h.txnsGroup[bucket][0]) - h.txnsGroup[bucket] = h.txnsGroup[bucket][1:] - if len(h.txnsGroup[bucket]) > 0 { - heap.Push(h.inner, innerHeapEntry{ - ts: h.txnsGroup[bucket][0].CommitTs, - bucket: bucket, - }) - } - } -} diff --git a/cdc/cdc/sink/txns_heap_test.go b/cdc/cdc/sink/txns_heap_test.go deleted file mode 100644 index 8f8bb363..00000000 --- a/cdc/cdc/sink/txns_heap_test.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package sink - -import ( - "github.com/pingcap/check" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/util/testleak" -) - -type TxnsHeapSuite struct{} - -var _ = check.Suite(&TxnsHeapSuite{}) - -func (s TxnsHeapSuite) TestTxnsHeap(c *check.C) { - defer testleak.AfterTest(c)() - testCases := []struct { - txnsMap map[model.TableID][]*model.SingleTableTxn - expected []*model.SingleTableTxn - }{{ - txnsMap: nil, - expected: nil, - }, { - txnsMap: map[model.TableID][]*model.SingleTableTxn{ - 1: { - {CommitTs: 1}, {CommitTs: 3}, {CommitTs: 5}, {CommitTs: 7}, {CommitTs: 9}, - }, - 2: { - {CommitTs: 1}, {CommitTs: 10}, {CommitTs: 15}, {CommitTs: 15}, {CommitTs: 15}, - }, - 3: { - {CommitTs: 1}, {CommitTs: 1}, {CommitTs: 1}, {CommitTs: 2}, {CommitTs: 3}, - }, - }, - expected: []*model.SingleTableTxn{ - {CommitTs: 1}, - {CommitTs: 1}, - {CommitTs: 1}, - {CommitTs: 1}, - {CommitTs: 1}, - {CommitTs: 2}, - {CommitTs: 3}, - {CommitTs: 3}, - {CommitTs: 5}, - {CommitTs: 7}, - {CommitTs: 9}, - {CommitTs: 10}, - {CommitTs: 15}, - {CommitTs: 15}, - {CommitTs: 15}, - }, - }} - - for _, tc := range testCases { - h := newTxnsHeap(tc.txnsMap) - i := 0 - h.iter(func(txn *model.SingleTableTxn) { - c.Assert(txn, check.DeepEquals, tc.expected[i]) - i++ - }) - } -} diff --git a/cdc/cdc/sorter/encoding/key.go b/cdc/cdc/sorter/encoding/key.go deleted file mode 100644 index f74461b0..00000000 --- a/cdc/cdc/sorter/encoding/key.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package encoding - -import ( - "encoding/binary" - - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" - "go.uber.org/zap" -) - -// DecodeKey decodes a key to uniqueID, tableID, startTs, CRTs. -func DecodeKey(key []byte) (uniqueID uint32, tableID uint64, startTs, CRTs uint64) { - // uniqueID, tableID, CRTs, startTs, Key, Put/Delete - // uniqueID - uniqueID = binary.BigEndian.Uint32(key) - // table ID - tableID = binary.BigEndian.Uint64(key[4:]) - // CRTs - CRTs = binary.BigEndian.Uint64(key[12:]) - if len(key) >= 28 { - // startTs - startTs = binary.BigEndian.Uint64(key[20:]) - } - return -} - -// EncodeTsKey encodes uniqueID, tableID, CRTs. -func EncodeTsKey(uniqueID uint32, tableID uint64, ts uint64) []byte { - // uniqueID, tableID, CRTs. - buf := make([]byte, 0, 4+8+8) - uint64Buf := [8]byte{} - // uniqueID - binary.BigEndian.PutUint32(uint64Buf[:], uniqueID) - buf = append(buf, uint64Buf[:4]...) - // tableID - binary.BigEndian.PutUint64(uint64Buf[:], tableID) - buf = append(buf, uint64Buf[:]...) - // CRTs - binary.BigEndian.PutUint64(uint64Buf[:], ts) - return append(buf, uint64Buf[:]...) -} - -// EncodeKey encodes a key according to event. -// Format: uniqueID, tableID, CRTs, startTs, Put/Delete, Key. -func EncodeKey(uniqueID uint32, tableID uint64, event *model.PolymorphicEvent) []byte { - if event.RawKV == nil { - log.Panic("rawkv must not be nil", zap.Any("event", event)) - } - // uniqueID, tableID, CRTs, startTs, Put/Delete, Key - length := 4 + 8 + 8 + 8 + 2 + len(event.RawKV.Key) - buf := make([]byte, 0, length) - uint64Buf := [8]byte{} - // uniqueID - binary.BigEndian.PutUint32(uint64Buf[:], uniqueID) - buf = append(buf, uint64Buf[:4]...) - // table ID - binary.BigEndian.PutUint64(uint64Buf[:], tableID) - buf = append(buf, uint64Buf[:]...) - // CRTs - binary.BigEndian.PutUint64(uint64Buf[:], event.CRTs) - buf = append(buf, uint64Buf[:]...) - // startTs - binary.BigEndian.PutUint64(uint64Buf[:], event.StartTs) - buf = append(buf, uint64Buf[:]...) - // Let Delete < Put - binary.BigEndian.PutUint16(uint64Buf[:], ^uint16(event.RawKV.OpType)) - buf = append(buf, uint64Buf[:2]...) - // key - return append(buf, event.RawKV.Key...) -} diff --git a/cdc/cdc/sorter/encoding/key_test.go b/cdc/cdc/sorter/encoding/key_test.go deleted file mode 100644 index 1a56d747..00000000 --- a/cdc/cdc/sorter/encoding/key_test.go +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package encoding - -import ( - "bytes" - "testing" - - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" -) - -func TestEncodeKey(t *testing.T) { - t.Parallel() - mustLess := func(uniqueIDa, uniqueIDb uint32, tableIDa, tableIDb uint64, a, b *model.PolymorphicEvent) { - keya, keyb := EncodeKey(uniqueIDa, tableIDa, a), EncodeKey(uniqueIDb, tableIDb, b) - require.Equal(t, bytes.Compare(keya, keyb), -1) - require.Equal(t, len(keya), cap(keya)) - require.Equal(t, len(keyb), cap(keyb)) - } - - // UID - mustLess( - 0, 1, - 0, 0, - model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypeDelete, - Key: []byte{1}, - StartTs: 1, - CRTs: 2, - }), - model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypeDelete, - Key: []byte{1}, - StartTs: 1, - CRTs: 2, - }), - ) - - // TableID - mustLess( - 0, 0, - 0, 1, - model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypeDelete, - Key: []byte{1}, - StartTs: 1, - CRTs: 2, - }), - model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypeDelete, - Key: []byte{1}, - StartTs: 1, - CRTs: 2, - }), - ) - // OpType: OpTypeDelete < OpTypePut - mustLess( - 0, 0, - 1, 1, - model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypeDelete, - Key: []byte{1}, - StartTs: 1, - CRTs: 2, - }), - model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypePut, - Key: []byte{1}, - StartTs: 1, - CRTs: 2, - }), - ) - // CRTs - mustLess( - 0, 0, - 1, 1, - model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypePut, - Key: []byte{1}, - StartTs: 1, - CRTs: 2, - }), - model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypePut, - Key: []byte{1}, - StartTs: 1, - CRTs: 3, - }), - ) - // Key - mustLess( - 0, 0, - 1, 1, - model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypePut, - Key: []byte{1}, - StartTs: 1, - CRTs: 3, - }), - model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypePut, - Key: []byte{2}, - StartTs: 1, - CRTs: 3, - }), - ) - // StartTs - mustLess( - 0, 0, - 1, 1, - model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypePut, - Key: []byte{2}, - StartTs: 1, - CRTs: 3, - }), - model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypePut, - Key: []byte{2}, - StartTs: 2, - CRTs: 3, - }), - ) -} - -func TestEncodeTsKey(t *testing.T) { - t.Parallel() - mustLess := func(uniqueIDa, uniqueIDb uint32, tableIDa, tableIDb uint64, a *model.PolymorphicEvent, b uint64) { - keya, keyb := EncodeKey(uniqueIDa, tableIDa, a), EncodeTsKey(uniqueIDb, tableIDb, b) - require.Equal(t, bytes.Compare(keya, keyb), -1) - require.Equal(t, len(keya), cap(keya)) - require.Equal(t, len(keyb), cap(keyb)) - } - - // UID - mustLess( - 0, 1, - 0, 0, - model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypeDelete, - Key: []byte{1}, - StartTs: 1, - CRTs: 2, - }), - 0, - ) - - // TableID - mustLess( - 0, 0, - 0, 1, - model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypeDelete, - Key: []byte{1}, - StartTs: 1, - CRTs: 2, - }), - 0, - ) - mustLess( - 0, 0, - 1, 1, - model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypeDelete, - Key: []byte{1}, - StartTs: 1, - CRTs: 2, - }), - 3, - ) -} - -func TestDecodeKey(t *testing.T) { - t.Parallel() - key := EncodeKey(1, 2, model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypePut, - Key: []byte{3}, - StartTs: 4, - CRTs: 5, - })) - uid, tableID, startTs, CRTs := DecodeKey(key) - require.EqualValues(t, 1, uid) - require.EqualValues(t, 2, tableID) - require.EqualValues(t, 4, startTs) - require.EqualValues(t, 5, CRTs) - - key = EncodeTsKey(1, 2, 3) - uid, tableID, startTs, CRTs = DecodeKey(key) - require.EqualValues(t, 1, uid) - require.EqualValues(t, 2, tableID) - require.EqualValues(t, 0, startTs) - require.EqualValues(t, 3, CRTs) -} diff --git a/cdc/cdc/sorter/encoding/value.go b/cdc/cdc/sorter/encoding/value.go deleted file mode 100644 index 6be51bd4..00000000 --- a/cdc/cdc/sorter/encoding/value.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package encoding - -import ( - "github.com/pingcap/errors" - "github.com/tikv/migration/cdc/cdc/model" -) - -// SerializerDeserializer is the interface encodes and decodes model.PolymorphicEvent. -type SerializerDeserializer interface { - Marshal(event *model.PolymorphicEvent, bytes []byte) ([]byte, error) - Unmarshal(event *model.PolymorphicEvent, bytes []byte) ([]byte, error) -} - -// MsgPackGenSerde encodes model.PolymorphicEvent into bytes and decodes -// model.PolymorphicEvent from bytes. -type MsgPackGenSerde struct{} - -// Marshal encodes model.PolymorphicEvent into bytes. -func (m *MsgPackGenSerde) Marshal(event *model.PolymorphicEvent, bytes []byte) ([]byte, error) { - bytes = bytes[:0] - return event.RawKV.MarshalMsg(bytes) -} - -// Unmarshal decodes model.PolymorphicEvent from bytes. -func (m *MsgPackGenSerde) Unmarshal(event *model.PolymorphicEvent, bytes []byte) ([]byte, error) { - if event.RawKV == nil { - event.RawKV = new(model.RawKVEntry) - } - - bytes, err := event.RawKV.UnmarshalMsg(bytes) - if err != nil { - return nil, errors.Trace(err) - } - - event.StartTs = event.RawKV.StartTs - event.CRTs = event.RawKV.CRTs - - return bytes, nil -} diff --git a/cdc/cdc/sorter/leveldb/buffer.go b/cdc/cdc/sorter/leveldb/buffer.go deleted file mode 100644 index e15896e9..00000000 --- a/cdc/cdc/sorter/leveldb/buffer.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package leveldb - -import ( - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/sorter/leveldb/message" - "go.uber.org/zap" -) - -// outputBuffer a struct that facilitate leveldb table sorter. -type outputBuffer struct { - // A slice of keys need to be deleted. - deleteKeys []message.Key - // A slice of resolved events that have the same commit ts. - resolvedEvents []*model.PolymorphicEvent - - advisedCapacity int -} - -func newOutputBuffer(advisedCapacity int) *outputBuffer { - return &outputBuffer{ - deleteKeys: make([]message.Key, 0, advisedCapacity), - resolvedEvents: make([]*model.PolymorphicEvent, 0, advisedCapacity), - advisedCapacity: advisedCapacity, - } -} - -// maybeShrink try to shrink slices to the advised capacity. -func (b *outputBuffer) maybeShrink() { - if len(b.deleteKeys) < b.advisedCapacity { - if cap(b.deleteKeys) > b.advisedCapacity { - buf := make([]message.Key, 0, b.advisedCapacity) - buf = append(buf, b.deleteKeys...) - b.deleteKeys = buf - } - } - if len(b.resolvedEvents) < b.advisedCapacity { - if cap(b.resolvedEvents) > b.advisedCapacity { - buf := make([]*model.PolymorphicEvent, 0, b.advisedCapacity) - buf = append(buf, b.resolvedEvents...) - b.resolvedEvents = buf - } - } -} - -// In place left shift resolved events slice. After the call, -// `index` will become the first element in the slice -func (b *outputBuffer) shiftResolvedEvents(index int) { - if index > len(b.resolvedEvents) { - log.Panic("index out of range", zap.Int("len", len(b.resolvedEvents))) - } - if index != 0 { - length := len(b.resolvedEvents) - for left, right := 0, index; right < length; right++ { - b.resolvedEvents[left] = b.resolvedEvents[right] - // Set original element to nil to help GC. - b.resolvedEvents[right] = nil - left++ - } - b.resolvedEvents = b.resolvedEvents[:length-index] - } -} - -// appendResolvedEvent appends resolved events to the buffer. -func (b *outputBuffer) appendResolvedEvent(event *model.PolymorphicEvent) { - if len(b.resolvedEvents) > 0 { - if b.resolvedEvents[0].CRTs != event.CRTs { - log.Panic("commit ts must be equal", - zap.Uint64("newCommitTs", event.CRTs), - zap.Uint64("commitTs", b.resolvedEvents[0].CRTs)) - } - } - b.resolvedEvents = append(b.resolvedEvents, event) -} - -// appendDeleteKey appends to-be-deleted keys to the buffer. -func (b *outputBuffer) appendDeleteKey(key message.Key) { - b.deleteKeys = append(b.deleteKeys, key) -} - -// resetDeleteKey reset deleteKeys to a zero len slice. -func (b *outputBuffer) resetDeleteKey() { - b.deleteKeys = b.deleteKeys[:0] -} - -// len returns the length of resolvedEvents and delete keys. -func (b *outputBuffer) len() (int, int) { - return len(b.resolvedEvents), len(b.deleteKeys) -} diff --git a/cdc/cdc/sorter/leveldb/buffer_test.go b/cdc/cdc/sorter/leveldb/buffer_test.go deleted file mode 100644 index cdb2c180..00000000 --- a/cdc/cdc/sorter/leveldb/buffer_test.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package leveldb - -import ( - "testing" - - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/sorter/leveldb/message" -) - -func TestOutputBufferMaybeShrink(t *testing.T) { - t.Parallel() - advisedCapacity := 4 - buf := newOutputBuffer(advisedCapacity) - require.Equal(t, 0, len(buf.resolvedEvents)) - require.Equal(t, 0, len(buf.deleteKeys)) - require.Equal(t, advisedCapacity, cap(buf.resolvedEvents)) - require.Equal(t, advisedCapacity, cap(buf.deleteKeys)) - - // len == cap == advisedCapacity. - buf.resolvedEvents = make([]*model.PolymorphicEvent, advisedCapacity) - buf.resolvedEvents[0] = model.NewResolvedPolymorphicEvent(0, 1) - buf.deleteKeys = make([]message.Key, advisedCapacity) - buf.deleteKeys[0] = message.Key([]byte{1}) - resolvedEvents := append([]*model.PolymorphicEvent{}, buf.resolvedEvents...) - deleteKeys := append([]message.Key{}, buf.deleteKeys...) - - buf.maybeShrink() - require.Equal(t, advisedCapacity, len(buf.resolvedEvents)) - require.Equal(t, advisedCapacity, cap(buf.resolvedEvents)) - require.EqualValues(t, resolvedEvents, buf.resolvedEvents) - require.EqualValues(t, deleteKeys, buf.deleteKeys) - - // len < cap == 2*advisedCapacity. - buf.resolvedEvents = make([]*model.PolymorphicEvent, 2*advisedCapacity-1, 2*advisedCapacity) - buf.resolvedEvents[0] = model.NewResolvedPolymorphicEvent(0, 1) - buf.deleteKeys = make([]message.Key, 2*advisedCapacity-1, 2*advisedCapacity) - buf.deleteKeys[0] = message.Key([]byte{1}) - resolvedEvents = append([]*model.PolymorphicEvent{}, buf.resolvedEvents...) - deleteKeys = append([]message.Key{}, buf.deleteKeys...) - - buf.maybeShrink() - require.Equal(t, 2*advisedCapacity-1, len(buf.resolvedEvents)) - require.Equal(t, 2*advisedCapacity-1, len(buf.deleteKeys)) - require.EqualValues(t, resolvedEvents, buf.resolvedEvents) - require.EqualValues(t, deleteKeys, buf.deleteKeys) - - // len < cap/2 == advisedCapacity. - buf.resolvedEvents = make([]*model.PolymorphicEvent, advisedCapacity-1, 2*advisedCapacity) - buf.resolvedEvents[0] = model.NewResolvedPolymorphicEvent(0, 1) - buf.deleteKeys = make([]message.Key, advisedCapacity-1, 2*advisedCapacity) - buf.deleteKeys[0] = message.Key([]byte{1}) - resolvedEvents = append([]*model.PolymorphicEvent{}, buf.resolvedEvents...) - deleteKeys = append([]message.Key{}, buf.deleteKeys...) - - buf.maybeShrink() - require.Equal(t, advisedCapacity-1, len(buf.resolvedEvents)) - require.Equal(t, advisedCapacity-1, len(buf.deleteKeys)) - require.EqualValues(t, resolvedEvents, buf.resolvedEvents) - require.EqualValues(t, deleteKeys, buf.deleteKeys) -} - -func TestOutputBufferShiftResolvedEvents(t *testing.T) { - t.Parallel() - advisedCapacity := 64 - buf := newOutputBuffer(advisedCapacity) - - events := make([]*model.PolymorphicEvent, advisedCapacity) - for i := range events { - events[i] = &model.PolymorphicEvent{CRTs: uint64(1)} - } - - for i := 0; i < advisedCapacity; i++ { - buf.resolvedEvents = append([]*model.PolymorphicEvent{}, events...) - buf.shiftResolvedEvents(i) - require.EqualValues(t, buf.resolvedEvents, events[i:]) - } -} diff --git a/cdc/cdc/sorter/leveldb/cleaner.go b/cdc/cdc/sorter/leveldb/cleaner.go deleted file mode 100644 index 9af58046..00000000 --- a/cdc/cdc/sorter/leveldb/cleaner.go +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package leveldb - -import ( - "context" - "sync" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/sorter/encoding" - "github.com/tikv/migration/cdc/cdc/sorter/leveldb/message" - "github.com/tikv/migration/cdc/pkg/actor" - actormsg "github.com/tikv/migration/cdc/pkg/actor/message" - "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/db" - "go.uber.org/zap" - "golang.org/x/time/rate" -) - -// CleanerActor is an actor that can clean up table data asynchronously. -type CleanerActor struct { - id actor.ID - db db.DB - wbSize int - - deleteCount int - compact *CompactScheduler - - closedWg *sync.WaitGroup - - limiter *rate.Limiter - router *actor.Router -} - -var _ actor.Actor = (*CleanerActor)(nil) - -// NewCleanerActor returns a cleaner actor. -func NewCleanerActor( - id int, db db.DB, router *actor.Router, compact *CompactScheduler, - cfg *config.DBConfig, wg *sync.WaitGroup, -) (*CleanerActor, actor.Mailbox, error) { - wg.Add(1) - wbSize := 500 // default write batch size. - if (cfg.CleanupSpeedLimit / 2) < wbSize { - // wb size must be less than speed limit, otherwise it is easily - // rate-limited. - wbSize = cfg.CleanupSpeedLimit / 2 - } - limiter := rate.NewLimiter(rate.Limit(cfg.CleanupSpeedLimit), wbSize*2) - mb := actor.NewMailbox(actor.ID(id), cfg.Concurrency) - return &CleanerActor{ - id: actor.ID(id), - db: db, - wbSize: wbSize, - compact: compact, - closedWg: wg, - limiter: limiter, - router: router, - }, mb, nil -} - -// Poll implements actor.Actor. -func (clean *CleanerActor) Poll(ctx context.Context, tasks []actormsg.Message) bool { - select { - case <-ctx.Done(): - clean.close(ctx.Err()) - return false - default: - } - - reschedulePos := -1 - rescheduleDelay := time.Duration(0) - batch := clean.db.Batch(0) -TASKS: - for pos := range tasks { - var task message.Task - msg := tasks[pos] - switch msg.Tp { - case actormsg.TypeSorterTask: - task = msg.SorterTask - case actormsg.TypeStop: - clean.close(nil) - return false - default: - log.Panic("unexpected message", zap.Any("message", msg)) - } - if !task.Cleanup { - log.Panic("unexpected message", zap.Any("message", msg)) - } - - start := encoding.EncodeTsKey(task.UID, task.TableID, 0) - limit := encoding.EncodeTsKey(task.UID, task.TableID+1, 0) - iter := clean.db.Iterator(start, limit) - - // Force writes the first batch if the task is rescheduled (rate limited). - force := task.CleanupRatelimited - - for hasNext := iter.Seek(start); hasNext; hasNext = iter.Next() { - batch.Delete(iter.Key()) - - // TODO it's similar to LevelActor.maybeWrite, - // they should be unified. - if int(batch.Count()) >= clean.wbSize { - delay, err := clean.writeRateLimited(batch, force) - if err != nil { - log.Panic("db error", - zap.Error(err), zap.Uint64("id", uint64(clean.id))) - } - if delay != 0 { - // Rate limited, break and reschedule tasks. - // After the delay, this batch can be write forcibly. - reschedulePos = pos - rescheduleDelay = delay - err := iter.Release() - if err != nil { - log.Panic("db error", - zap.Error(err), zap.Uint64("id", uint64(clean.id))) - } - break TASKS - } - force = false - } - } - // Release iterator and snapshot in time. - err := iter.Release() - if err != nil { - log.Panic("db error", - zap.Error(err), zap.Uint64("id", uint64(clean.id))) - } - // Ignore rate limit and force write remaining kv. - _, err = clean.writeRateLimited(batch, true) - if err != nil { - log.Panic("db error", - zap.Error(err), zap.Uint64("id", uint64(clean.id))) - } - } - - // Reschedule rate limited tasks. - if reschedulePos >= 0 { - clean.reschedule(ctx, tasks[reschedulePos:], rescheduleDelay) - } - - return true -} - -func (clean *CleanerActor) close(err error) { - log.Info("cleaner actor quit", - zap.Uint64("ID", uint64(clean.id)), zap.Error(err)) - clean.closedWg.Done() -} - -func (clean *CleanerActor) writeRateLimited( - batch db.Batch, force bool, -) (time.Duration, error) { - count := int(batch.Count()) - // Skip rate limiter, if force write. - if !force { - reservation := clean.limiter.ReserveN(time.Now(), count) - if reservation != nil { - if !reservation.OK() { - log.Panic("write batch too large", - zap.Int("wbSize", count), - zap.Int("limit", clean.limiter.Burst())) - } - delay := reservation.Delay() - if delay != 0 { - // Rate limited, wait. - return delay, nil - } - } - } - clean.deleteCount += int(batch.Count()) - err := batch.Commit() - if err != nil { - return 0, errors.Trace(err) - } - batch.Reset() - // Schedule a compact task when there are too many deletion. - if clean.compact.maybeCompact(clean.id, clean.deleteCount) { - // Reset delete key count if schedule compaction successfully. - clean.deleteCount = 0 - } - return 0, nil -} - -func (clean *CleanerActor) reschedule( - ctx context.Context, tasks []actormsg.Message, delay time.Duration, -) { - id := clean.id - msgs := append([]actormsg.Message{}, tasks...) - // Reschedule tasks respect after delay. - time.AfterFunc(delay, func() { - for i := range msgs { - // Mark the first task is rescheduled due to rate limit. - if i == 0 { - msgs[i].SorterTask.CleanupRatelimited = true - } - // Blocking send to ensure that no tasks are lost. - err := clean.router.SendB(ctx, id, msgs[i]) - if err != nil { - log.Warn("drop table clean-up task", - zap.Uint64("tableID", msgs[i].SorterTask.TableID)) - } - } - }) -} diff --git a/cdc/cdc/sorter/leveldb/cleaner_test.go b/cdc/cdc/sorter/leveldb/cleaner_test.go deleted file mode 100644 index d968d8ff..00000000 --- a/cdc/cdc/sorter/leveldb/cleaner_test.go +++ /dev/null @@ -1,394 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package leveldb - -import ( - "context" - "encoding/hex" - "fmt" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/sorter/encoding" - "github.com/tikv/migration/cdc/cdc/sorter/leveldb/message" - "github.com/tikv/migration/cdc/pkg/actor" - actormsg "github.com/tikv/migration/cdc/pkg/actor/message" - "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/db" -) - -func makeCleanTask(uid uint32, tableID uint64) []actormsg.Message { - return []actormsg.Message{actormsg.SorterMessage(message.Task{ - UID: uid, - TableID: tableID, - Cleanup: true, - })} -} - -func prepareData(t *testing.T, db db.DB, data [][]int) { - wb := db.Batch(0) - for _, d := range data { - count, uid, tableID := d[0], d[1], d[2] - for k := 0; k < count; k++ { - key := encoding.EncodeKey( - uint32(uid), uint64(tableID), - model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypeDelete, - Key: []byte{byte(k)}, - StartTs: 1, - CRTs: 2, - })) - wb.Put(key, key) - } - } - require.Nil(t, wb.Commit()) -} - -func TestCleanerPoll(t *testing.T) { - t.Parallel() - ctx := context.Background() - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - cfg.Count = 1 - - db, err := db.OpenLevelDB(ctx, 1, t.TempDir(), cfg) - require.Nil(t, err) - closedWg := new(sync.WaitGroup) - compact := NewCompactScheduler(actor.NewRouter(t.Name()), cfg) - clean, _, err := NewCleanerActor(1, db, nil, compact, cfg, closedWg) - require.Nil(t, err) - - // Put data to db. - // * 1 key of uid1 table1 - // * 3 key of uid2 table1 - // * 2 key of uid3 table2 - // * 1 key of uid4 table2 - data := [][]int{ - {1, 1, 1}, - {3, 2, 1}, - {2, 3, 2}, - {1, 4, 2}, - } - prepareData(t, db, data) - - // Ensure there are some key/values belongs to uid2 table1. - start := encoding.EncodeTsKey(2, 1, 0) - limit := encoding.EncodeTsKey(2, 2, 0) - iter := db.Iterator(start, limit) - require.True(t, iter.First()) - require.Nil(t, iter.Release()) - - // Clean up uid2 table1 - closed := !clean.Poll(ctx, makeCleanTask(2, 1)) - require.False(t, closed) - - // Ensure no key/values belongs to uid2 table1 - iter = db.Iterator(start, limit) - require.False(t, iter.First()) - require.Nil(t, iter.Release()) - - // Ensure uid1 table1 is untouched. - start = encoding.EncodeTsKey(1, 1, 0) - limit = encoding.EncodeTsKey(1, 2, 0) - iter = db.Iterator(start, limit) - require.True(t, iter.First()) - require.Nil(t, iter.Release()) - - // Ensure uid3 table2 is untouched. - start = encoding.EncodeTsKey(3, 2, 0) - limit = encoding.EncodeTsKey(3, 3, 0) - iter = db.Iterator(start, limit) - require.True(t, iter.First()) - require.Nil(t, iter.Release()) - - // Clean up uid3 table2 - closed = !clean.Poll(ctx, makeCleanTask(3, 2)) - require.False(t, closed) - - // Ensure no key/values belongs to uid3 table2 - iter = db.Iterator(start, limit) - require.False(t, iter.First()) - require.Nil(t, iter.Release()) - - // Ensure uid4 table2 is untouched. - start = encoding.EncodeTsKey(4, 2, 0) - limit = encoding.EncodeTsKey(4, 3, 0) - iter = db.Iterator(start, limit) - require.True(t, iter.First()) - require.Nil(t, iter.Release()) - - // Close leveldb. - closed = !clean.Poll(ctx, []actormsg.Message{actormsg.StopMessage()}) - require.True(t, closed) - closedWg.Wait() - require.Nil(t, db.Close()) -} - -func TestCleanerContextCancel(t *testing.T) { - t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - cfg.Count = 1 - - db, err := db.OpenLevelDB(ctx, 1, t.TempDir(), cfg) - require.Nil(t, err) - closedWg := new(sync.WaitGroup) - compact := NewCompactScheduler(actor.NewRouter(t.Name()), cfg) - clean, _, err := NewCleanerActor(1, db, nil, compact, cfg, closedWg) - require.Nil(t, err) - - cancel() - tasks := makeCleanTask(1, 1) - closed := !clean.Poll(ctx, tasks) - require.True(t, closed) - closedWg.Wait() - require.Nil(t, db.Close()) -} - -func TestCleanerWriteRateLimited(t *testing.T) { - t.Parallel() - ctx := context.Background() - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - cfg.Count = 1 - cfg.CleanupSpeedLimit = 4 - // wbSize = cleanup speed limit / 2 - - db, err := db.OpenLevelDB(ctx, 1, t.TempDir(), cfg) - require.Nil(t, err) - closedWg := new(sync.WaitGroup) - compact := NewCompactScheduler(actor.NewRouter(t.Name()), cfg) - clean, _, err := NewCleanerActor(1, db, nil, compact, cfg, closedWg) - require.Nil(t, err) - - // Put data to db. - // * 1 key of uid1 table1 - // * 3 key of uid2 table1 - // * 2 key of uid3 table2 - // * 1 key of uid4 table2 - data := [][]int{ - {1, 1, 1}, - {3, 2, 1}, - {2, 3, 2}, - {1, 4, 2}, - } - prepareData(t, db, data) - - keys := [][]byte{} - start := encoding.EncodeTsKey(0, 0, 0) - limit := encoding.EncodeTsKey(5, 0, 0) - iter := db.Iterator(start, limit) - for iter.Next() { - key := append([]byte{}, iter.Key()...) - keys = append(keys, key) - } - require.Nil(t, iter.Release()) - require.Equal(t, 7, len(keys), "%v", keys) - - // Must speed limited. - wb := db.Batch(0) - var delay time.Duration - var count int - for { - for i := 0; i < cfg.CleanupSpeedLimit/2; i++ { - wb.Delete(keys[i]) - } - delay, err = clean.writeRateLimited(wb, false) - require.Nil(t, err) - if delay != 0 { - break - } - count++ - } - - // Sleep and write again. - time.Sleep(delay * 4) - delay, err = clean.writeRateLimited(wb, false) - require.EqualValues(t, 0, delay) - require.Nil(t, err) - - // Force write ignores speed limit. - for i := 0; i < count*2; i++ { - delay, err = clean.writeRateLimited(wb, true) - require.EqualValues(t, 0, delay) - require.Nil(t, err) - } - - // Close leveldb. - closed := !clean.Poll(ctx, []actormsg.Message{actormsg.StopMessage()}) - require.True(t, closed) - closedWg.Wait() - require.Nil(t, db.Close()) -} - -func TestCleanerTaskRescheduled(t *testing.T) { - t.Parallel() - ctx := context.Background() - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - cfg.Count = 1 - cfg.CleanupSpeedLimit = 4 - // wbSize = cleanup speed limit / 2 - - db, err := db.OpenLevelDB(ctx, 1, t.TempDir(), cfg) - require.Nil(t, err) - closedWg := new(sync.WaitGroup) - router := actor.NewRouter(t.Name()) - compact := NewCompactScheduler(actor.NewRouter(t.Name()), cfg) - clean, mb, err := NewCleanerActor(1, db, router, compact, cfg, closedWg) - require.Nil(t, err) - router.InsertMailbox4Test(actor.ID(1), mb) - require.Nil(t, router.SendB(ctx, actor.ID(1), actormsg.TickMessage())) - receiveTimeout := func() (actormsg.Message, bool) { - for i := 0; i < 10; i++ { // 2s - time.Sleep(200 * time.Millisecond) - task, ok := mb.Receive() - if ok { - return task, ok - } - } - return mb.Receive() - } - mustReceive := func() actormsg.Message { - task, ok := receiveTimeout() - if !ok { - t.Fatal("timeout") - } - return task - } - _ = mustReceive() - - // Put data to db. - // * 8 key of uid1 table1 - // * 2 key of uid2 table1 - // * 2 key of uid3 table2 - data := [][]int{ - {8, 1, 1}, - {2, 2, 1}, - {2, 3, 2}, - } - prepareData(t, db, data) - - tasks := makeCleanTask(1, 1) - tasks = append(tasks, makeCleanTask(2, 1)...) - tasks = append(tasks, makeCleanTask(3, 2)...) - - // All tasks must be rescheduled. - closed := !clean.Poll(ctx, tasks) - require.False(t, closed) - // uid1 table1 - task := mustReceive() - msg := makeCleanTask(1, 1) - msg[0].SorterTask.CleanupRatelimited = true - require.EqualValues(t, msg[0], task) - tasks[0] = task - // uid2 tabl2 - task = mustReceive() - msg = makeCleanTask(2, 1) - require.EqualValues(t, msg[0], task) - tasks[1] = task - // uid3 tabl2 - task = mustReceive() - msg = makeCleanTask(3, 2) - require.EqualValues(t, msg[0], task) - tasks[2] = task - - // Reschedule tasks. - // All tasks can finish eventually. - closed = !clean.Poll(ctx, tasks) - require.False(t, closed) - for { - task, ok := receiveTimeout() - if !ok { - break - } - closed := !clean.Poll(ctx, []actormsg.Message{task}) - require.False(t, closed) - } - - // Ensure all data are deleted. - start := encoding.EncodeTsKey(0, 0, 0) - limit := encoding.EncodeTsKey(4, 0, 0) - iter := db.Iterator(start, limit) - require.False(t, iter.First(), fmt.Sprintln(hex.EncodeToString(iter.Key()))) - require.Nil(t, iter.Release()) - - // Close leveldb. - closed = !clean.Poll(ctx, []actormsg.Message{actormsg.StopMessage()}) - require.True(t, closed) - closedWg.Wait() - // TODO: find a better to test if iterators are leaked. - // stats := leveldb.DBStats{} - // require.Nil(t, db.Stats(&stats)) - // require.Zero(t, stats.AliveIterators) - require.Nil(t, db.Close()) -} - -func TestCleanerCompact(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - cfg.Count = 1 - - id := 1 - db, err := db.OpenLevelDB(ctx, id, t.TempDir(), cfg) - require.Nil(t, err) - closedWg := new(sync.WaitGroup) - compactRouter := actor.NewRouter(t.Name()) - compactMB := actor.NewMailbox(actor.ID(id), 1) - compactRouter.InsertMailbox4Test(compactMB.ID(), compactMB) - compact := NewCompactScheduler(compactRouter, cfg) - cleaner, _, err := NewCleanerActor(id, db, nil, compact, cfg, closedWg) - require.Nil(t, err) - - // Lower compactThreshold to speed up tests. - compact.compactThreshold = 2 - cleaner.wbSize = 1 - - // Put data to db. - // * 1 key of uid1 table1 - // * 2 key of uid2 table1 - data := [][]int{ - {1, 1, 1}, - {2, 2, 1}, - } - prepareData(t, db, data) - - // Empty task must not trigger compact. - closed := !cleaner.Poll(ctx, makeCleanTask(0, 0)) - require.False(t, closed) - _, ok := compactMB.Receive() - require.False(t, ok) - - // Delete 2 keys must trigger compact. - closed = !cleaner.Poll(ctx, makeCleanTask(2, 1)) - require.False(t, closed) - _, ok = compactMB.Receive() - require.True(t, ok) - - // Delete 1 key must not trigger compact. - closed = !cleaner.Poll(ctx, makeCleanTask(1, 1)) - require.False(t, closed) - _, ok = compactMB.Receive() - require.False(t, ok) - - // Close db. - closed = !cleaner.Poll(ctx, []actormsg.Message{actormsg.StopMessage()}) - require.True(t, closed) - closedWg.Wait() - require.Nil(t, db.Close()) -} diff --git a/cdc/cdc/sorter/leveldb/compactor.go b/cdc/cdc/sorter/leveldb/compactor.go deleted file mode 100644 index d4c3d1be..00000000 --- a/cdc/cdc/sorter/leveldb/compactor.go +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package leveldb - -import ( - "bytes" - "context" - "strconv" - "sync" - "time" - - "github.com/pingcap/log" - "github.com/prometheus/client_golang/prometheus" - "github.com/tikv/migration/cdc/pkg/actor" - actormsg "github.com/tikv/migration/cdc/pkg/actor/message" - "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/db" - cerrors "github.com/tikv/migration/cdc/pkg/errors" - "go.uber.org/zap" -) - -// CompactActor is an actor that compacts db. -// It GCs delete kv entries and reclaim disk space. -type CompactActor struct { - id actor.ID - db db.DB - closedWg *sync.WaitGroup - - metricCompactDuration prometheus.Observer -} - -var _ actor.Actor = (*CompactActor)(nil) - -// NewCompactActor returns a compactor actor. -func NewCompactActor( - id int, db db.DB, wg *sync.WaitGroup, captureAddr string, -) (*CompactActor, actor.Mailbox, error) { - wg.Add(1) - idTag := strconv.Itoa(id) - // Compact is CPU intensive, set capacity to 1 to reduce unnecessary tasks. - mb := actor.NewMailbox(actor.ID(id), 1) - return &CompactActor{ - id: actor.ID(id), - db: db, - closedWg: wg, - - metricCompactDuration: sorterCompactDurationHistogram.WithLabelValues(captureAddr, idTag), - }, mb, nil -} - -// Poll implements actor.Actor. -func (c *CompactActor) Poll(ctx context.Context, tasks []actormsg.Message) bool { - select { - case <-ctx.Done(): - c.close(ctx.Err()) - return false - default: - } - - // Only compact once for every batch. - for pos := range tasks { - msg := tasks[pos] - switch msg.Tp { - case actormsg.TypeTick: - case actormsg.TypeStop: - c.close(nil) - return false - default: - log.Panic("unexpected message", zap.Any("message", msg)) - } - } - - // A range that is large enough to cover entire db effectively. - // See see sorter/encoding/key.go. - start, end := []byte{0x0}, bytes.Repeat([]byte{0xff}, 128) - now := time.Now() - if err := c.db.Compact(start, end); err != nil { - log.Error("db compact error", zap.Error(err)) - } - c.metricCompactDuration.Observe(time.Since(now).Seconds()) - - return true -} - -func (c *CompactActor) close(err error) { - log.Info("compactor actor quit", - zap.Uint64("ID", uint64(c.id)), zap.Error(err)) - c.closedWg.Done() -} - -// NewCompactScheduler returns a new compact scheduler. -func NewCompactScheduler( - router *actor.Router, cfg *config.DBConfig, -) *CompactScheduler { - return &CompactScheduler{ - router: router, - compactThreshold: cfg.CompactionDeletionThreshold, - } -} - -// CompactScheduler schedules compact tasks to compactors. -type CompactScheduler struct { - // A router to compactors. - router *actor.Router - // The number of delete keys that triggers compact. - compactThreshold int -} - -func (s *CompactScheduler) maybeCompact(id actor.ID, deleteCount int) bool { - if deleteCount < s.compactThreshold { - return false - } - err := s.router.Send(id, actormsg.TickMessage()) - // An ongoing compaction may block compactor and cause channel full, - // skip send the task as there is a pending task. - if err != nil && cerrors.ErrMailboxFull.NotEqual(err) { - log.Warn("schedule compact failed", zap.Error(err)) - return false - } - return true -} diff --git a/cdc/cdc/sorter/leveldb/compactor_test.go b/cdc/cdc/sorter/leveldb/compactor_test.go deleted file mode 100644 index 75e9c575..00000000 --- a/cdc/cdc/sorter/leveldb/compactor_test.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package leveldb - -import ( - "context" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/pkg/actor" - actormsg "github.com/tikv/migration/cdc/pkg/actor/message" - "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/db" -) - -type mockCompactDB struct { - db.DB - compact chan struct{} -} - -func (m *mockCompactDB) Compact(start, end []byte) error { - m.compact <- struct{}{} - return nil -} - -func TestCompactorPoll(t *testing.T) { - t.Parallel() - ctx := context.Background() - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - cfg.Count = 1 - - db, err := db.OpenLevelDB(ctx, 1, t.TempDir(), cfg) - require.Nil(t, err) - mockDB := mockCompactDB{DB: db, compact: make(chan struct{}, 1)} - closedWg := new(sync.WaitGroup) - compactor, _, err := NewCompactActor(1, &mockDB, closedWg, "") - require.Nil(t, err) - - closed := !compactor.Poll(ctx, []actormsg.Message{actormsg.TickMessage()}) - require.False(t, closed) - select { - case <-time.After(5 * time.Second): - t.Fatal("Must trigger compact") - case <-mockDB.compact: - } - - // Close leveldb. - closed = !compactor.Poll(ctx, []actormsg.Message{actormsg.StopMessage()}) - require.True(t, closed) - closedWg.Wait() - require.Nil(t, db.Close()) -} - -func TestComactorContextCancel(t *testing.T) { - t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - cfg.Count = 1 - - db, err := db.OpenLevelDB(ctx, 1, t.TempDir(), cfg) - require.Nil(t, err) - closedWg := new(sync.WaitGroup) - ldb, _, err := NewCompactActor(0, db, closedWg, "") - require.Nil(t, err) - - cancel() - closed := !ldb.Poll(ctx, []actormsg.Message{actormsg.TickMessage()}) - require.True(t, closed) - closedWg.Wait() - require.Nil(t, db.Close()) -} - -func TestScheduleCompact(t *testing.T) { - t.Parallel() - router := actor.NewRouter(t.Name()) - mb := actor.NewMailbox(actor.ID(1), 1) - router.InsertMailbox4Test(mb.ID(), mb) - compact := NewCompactScheduler( - router, config.GetDefaultServerConfig().Debug.DB) - compact.compactThreshold = 2 - - // Too few deletion, should not trigger compact. - require.False(t, compact.maybeCompact(mb.ID(), 1)) - _, ok := mb.Receive() - require.False(t, ok) - // Must trigger compact. - require.True(t, compact.maybeCompact(mb.ID(), 3)) - msg, ok := mb.Receive() - require.True(t, ok) - require.EqualValues(t, actormsg.TickMessage(), msg) - - // Skip sending unnecessary tasks. - require.True(t, compact.maybeCompact(mb.ID(), 3)) - require.True(t, compact.maybeCompact(mb.ID(), 3)) - msg, ok = mb.Receive() - require.True(t, ok) - require.EqualValues(t, actormsg.TickMessage(), msg) - _, ok = mb.Receive() - require.False(t, ok) -} diff --git a/cdc/cdc/sorter/leveldb/leveldb.go b/cdc/cdc/sorter/leveldb/leveldb.go deleted file mode 100644 index 07d120a9..00000000 --- a/cdc/cdc/sorter/leveldb/leveldb.go +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package leveldb - -import ( - "container/list" - "context" - "strconv" - "sync" - "time" - - "github.com/pingcap/log" - "github.com/prometheus/client_golang/prometheus" - "github.com/tikv/migration/cdc/cdc/sorter/leveldb/message" - "github.com/tikv/migration/cdc/pkg/actor" - actormsg "github.com/tikv/migration/cdc/pkg/actor/message" - "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/db" - cerrors "github.com/tikv/migration/cdc/pkg/errors" - "go.uber.org/zap" - "golang.org/x/sync/semaphore" -) - -// Queue of IterRequest -type iterQueue struct { - *list.List - // TableID set. - tables map[tableKey]struct{} -} - -type iterItem struct { - key tableKey - req *message.IterRequest -} - -type tableKey struct { - UID uint32 - TableID uint64 -} - -func (q *iterQueue) push(uid uint32, tableID uint64, req *message.IterRequest) { - key := tableKey{UID: uid, TableID: tableID} - _, ok := q.tables[key] - if ok { - log.Panic("A table should not issue two concurrent iterator requests", - zap.Uint64("tableID", tableID), - zap.Uint32("UID", uid), - zap.Uint64("resolvedTs", req.ResolvedTs)) - } - q.tables[key] = struct{}{} - q.List.PushBack(iterItem{req: req, key: key}) -} - -func (q *iterQueue) pop() (*message.IterRequest, bool) { - item := q.List.Front() - if item == nil { - return nil, false - } - q.List.Remove(item) - req := item.Value.(iterItem) - delete(q.tables, req.key) - return req.req, true -} - -// DBActor is a db actor, it reads, writes and deletes key value pair in its db. -type DBActor struct { - id actor.ID - db db.DB - wb db.Batch - wbSize int - wbCap int - iterSem *semaphore.Weighted - iterQ iterQueue - - deleteCount int - compact *CompactScheduler - - closedWg *sync.WaitGroup - - metricWriteDuration prometheus.Observer - metricWriteBytes prometheus.Observer -} - -var _ actor.Actor = (*DBActor)(nil) - -// NewDBActor returns a db actor. -func NewDBActor( - id int, db db.DB, cfg *config.DBConfig, compact *CompactScheduler, - wg *sync.WaitGroup, captureAddr string, -) (*DBActor, actor.Mailbox, error) { - idTag := strconv.Itoa(id) - // Write batch size should be larger than block size to save CPU. - const writeBatchSizeFactor = 16 - wbSize := cfg.BlockSize * writeBatchSizeFactor - // Double batch capacity to avoid memory reallocation. - const writeBatchCapFactor = 2 - wbCap := wbSize * writeBatchCapFactor - wb := db.Batch(wbCap) - // IterCount limits the total number of opened iterators to release db - // resources in time. - iterSema := semaphore.NewWeighted(int64(cfg.Concurrency)) - mb := actor.NewMailbox(actor.ID(id), cfg.Concurrency) - wg.Add(1) - - return &DBActor{ - id: actor.ID(id), - db: db, - wb: wb, - iterSem: iterSema, - iterQ: iterQueue{ - List: list.New(), - tables: make(map[tableKey]struct{}), - }, - wbSize: wbSize, - wbCap: wbCap, - compact: compact, - - closedWg: wg, - - metricWriteDuration: sorterWriteDurationHistogram.WithLabelValues(captureAddr, idTag), - metricWriteBytes: sorterWriteBytesHistogram.WithLabelValues(captureAddr, idTag), - }, mb, nil -} - -func (ldb *DBActor) close(err error) { - log.Info("db actor quit", zap.Uint64("ID", uint64(ldb.id)), zap.Error(err)) - ldb.closedWg.Done() -} - -func (ldb *DBActor) maybeWrite(force bool) error { - bytes := len(ldb.wb.Repr()) - if bytes >= ldb.wbSize || (force && bytes != 0) { - startTime := time.Now() - err := ldb.wb.Commit() - if err != nil { - return cerrors.ErrLevelDBSorterError.GenWithStackByArgs(err) - } - ldb.metricWriteDuration.Observe(time.Since(startTime).Seconds()) - ldb.metricWriteBytes.Observe(float64(bytes)) - - // Reset write batch or reclaim memory if it grows too large. - if cap(ldb.wb.Repr()) <= ldb.wbCap { - ldb.wb.Reset() - } else { - ldb.wb = ldb.db.Batch(ldb.wbCap) - } - - // Schedule a compact task when there are too many deletion. - if ldb.compact.maybeCompact(ldb.id, ldb.deleteCount) { - // Reset delete key count if schedule compaction successfully. - ldb.deleteCount = 0 - } - } - return nil -} - -// Batch acquire iterators for requests in the queue. -func (ldb *DBActor) acquireIterators() { - for { - succeed := ldb.iterSem.TryAcquire(1) - if !succeed { - break - } - req, ok := ldb.iterQ.pop() - if !ok { - ldb.iterSem.Release(1) - break - } - - iterCh := req.IterCh - iterRange := req.Range - iter := ldb.db.Iterator(iterRange[0], iterRange[1]) - iterCh <- &message.LimitedIterator{ - Iterator: iter, - Sema: ldb.iterSem, - ResolvedTs: req.ResolvedTs, - } - close(iterCh) - } -} - -// Poll implements actor.Actor. -// It handles tasks by writing kv, deleting kv and taking iterators. -func (ldb *DBActor) Poll(ctx context.Context, tasks []actormsg.Message) bool { - select { - case <-ctx.Done(): - ldb.close(ctx.Err()) - return false - default: - } - requireIter := false - for i := range tasks { - var task message.Task - msg := tasks[i] - switch msg.Tp { - case actormsg.TypeTick: - continue - case actormsg.TypeSorterTask: - task = msg.SorterTask - case actormsg.TypeStop: - ldb.close(nil) - return false - default: - log.Panic("unexpected message", zap.Any("message", msg)) - } - - for k, v := range task.Events { - if len(v) != 0 { - ldb.wb.Put([]byte(k), v) - } else { - // Delete the key if value is empty - ldb.wb.Delete([]byte(k)) - ldb.deleteCount++ - } - - // Do not force write, batching for efficiency. - if err := ldb.maybeWrite(false); err != nil { - log.Panic("db error", zap.Error(err)) - } - } - if task.IterReq != nil { - // Append to slice for later batch acquiring iterators. - ldb.iterQ.push(task.UID, task.TableID, task.IterReq) - requireIter = true - } - } - - // Force write only if there is a task requires an iterator. - if err := ldb.maybeWrite(requireIter); err != nil { - log.Panic("db error", zap.Error(err)) - } - ldb.acquireIterators() - - return true -} diff --git a/cdc/cdc/sorter/leveldb/leveldb_test.go b/cdc/cdc/sorter/leveldb/leveldb_test.go deleted file mode 100644 index 7f4e3cc4..00000000 --- a/cdc/cdc/sorter/leveldb/leveldb_test.go +++ /dev/null @@ -1,418 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package leveldb - -import ( - "bytes" - "context" - "math/rand" - "sort" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/sorter/leveldb/message" - "github.com/tikv/migration/cdc/pkg/actor" - actormsg "github.com/tikv/migration/cdc/pkg/actor/message" - "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/db" - "github.com/tikv/migration/cdc/pkg/leakutil" - "go.uber.org/goleak" -) - -func TestMain(m *testing.M) { - leakutil.SetUpLeakTest(m, - goleak.IgnoreTopFunction("github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain")) -} - -func TestMaybeWrite(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - cfg.Count = 1 - - db, err := db.OpenLevelDB(ctx, 1, t.TempDir(), cfg) - require.Nil(t, err) - closedWg := new(sync.WaitGroup) - compact := NewCompactScheduler(actor.NewRouter(t.Name()), cfg) - ldb, _, err := NewDBActor(0, db, cfg, compact, closedWg, "") - require.Nil(t, err) - - // Empty batch - err = ldb.maybeWrite(false) - require.Nil(t, err) - - // None empty batch - ldb.wb.Put([]byte("abc"), []byte("abc")) - err = ldb.maybeWrite(false) - require.Nil(t, err) - require.EqualValues(t, ldb.wb.Count(), 1) - - // None empty batch - err = ldb.maybeWrite(true) - require.Nil(t, err) - require.EqualValues(t, ldb.wb.Count(), 0) - - ldb.wb.Put([]byte("abc"), []byte("abc")) - ldb.wbSize = 1 - require.Greater(t, len(ldb.wb.Repr()), ldb.wbSize) - err = ldb.maybeWrite(false) - require.Nil(t, err) - require.EqualValues(t, ldb.wb.Count(), 0) - - // Close db. - closed := !ldb.Poll(ctx, []actormsg.Message{actormsg.StopMessage()}) - require.True(t, closed) - closedWg.Wait() - require.Nil(t, db.Close()) -} - -func TestCompact(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - cfg.Count = 1 - - id := 1 - db, err := db.OpenLevelDB(ctx, id, t.TempDir(), cfg) - require.Nil(t, err) - closedWg := new(sync.WaitGroup) - compactRouter := actor.NewRouter(t.Name()) - compactMB := actor.NewMailbox(actor.ID(id), 1) - compactRouter.InsertMailbox4Test(compactMB.ID(), compactMB) - compact := NewCompactScheduler(compactRouter, cfg) - ldb, _, err := NewDBActor(id, db, cfg, compact, closedWg, "") - require.Nil(t, err) - - // Lower compactThreshold to speed up tests. - compact.compactThreshold = 2 - - // Empty task must not trigger compact. - task, iterCh := makeTask(make(map[message.Key][]byte), [][]byte{{0x00}, {0xff}}) - closed := !ldb.Poll(ctx, task) - require.False(t, closed) - <-iterCh - _, ok := compactMB.Receive() - require.False(t, ok) - - // Delete 3 keys must trigger compact. - dels := map[message.Key][]byte{"a": {}, "b": {}, "c": {}} - task, iterCh = makeTask(dels, [][]byte{{0x00}, {0xff}}) - closed = !ldb.Poll(ctx, task) - require.False(t, closed) - <-iterCh - _, ok = compactMB.Receive() - require.True(t, ok) - - // Delete 1 key must not trigger compact. - dels = map[message.Key][]byte{"a": {}} - task, iterCh = makeTask(dels, [][]byte{{0x00}, {0xff}}) - closed = !ldb.Poll(ctx, task) - require.False(t, closed) - <-iterCh - _, ok = compactMB.Receive() - require.False(t, ok) - - // Close db. - closed = !ldb.Poll(ctx, []actormsg.Message{actormsg.StopMessage()}) - require.True(t, closed) - closedWg.Wait() - require.Nil(t, db.Close()) -} - -func makeTask(events map[message.Key][]byte, rg [][]byte) ([]actormsg.Message, chan *message.LimitedIterator) { - var iterReq *message.IterRequest - var iterCh chan *message.LimitedIterator - if len(rg) != 0 { - iterCh = make(chan *message.LimitedIterator, 1) - iterReq = &message.IterRequest{ - Range: [2][]byte{rg[0], rg[1]}, - IterCh: iterCh, - } - } - return []actormsg.Message{actormsg.SorterMessage(message.Task{ - Events: events, - IterReq: iterReq, - })}, iterCh -} - -func TestPutReadDelete(t *testing.T) { - t.Parallel() - - ctx := context.Background() - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - cfg.Count = 1 - - db, err := db.OpenLevelDB(ctx, 1, t.TempDir(), cfg) - require.Nil(t, err) - closedWg := new(sync.WaitGroup) - compact := NewCompactScheduler(actor.NewRouter(t.Name()), cfg) - ldb, _, err := NewDBActor(0, db, cfg, compact, closedWg, "") - require.Nil(t, err) - - // Put only. - tasks, iterCh := makeTask(map[message.Key][]byte{"key": {}}, nil) - require.Nil(t, iterCh) - closed := !ldb.Poll(ctx, tasks) - require.False(t, closed) - - // Put and read. - tasks, iterCh = makeTask(map[message.Key][]byte{"key": []byte("value")}, - [][]byte{{0x00}, {0xff}}) - closed = !ldb.Poll(ctx, tasks) - require.False(t, closed) - iter, ok := <-iterCh - require.True(t, ok) - require.NotNil(t, iter) - ok = iter.Seek([]byte("")) - require.True(t, ok) - require.EqualValues(t, iter.Key(), "key") - ok = iter.Next() - require.False(t, ok) - require.Nil(t, iter.Release()) - - // Read only. - tasks, iterCh = makeTask(make(map[message.Key][]byte), [][]byte{{0x00}, {0xff}}) - closed = !ldb.Poll(ctx, tasks) - require.False(t, closed) - iter, ok = <-iterCh - require.True(t, ok) - require.NotNil(t, iter) - ok = iter.Seek([]byte("")) - require.True(t, ok) - require.EqualValues(t, iter.Key(), "key") - ok = iter.Next() - require.False(t, ok) - require.Nil(t, iter.Release()) - - // Delete and read. - tasks, iterCh = makeTask(map[message.Key][]byte{"key": {}}, [][]byte{{0x00}, {0xff}}) - closed = !ldb.Poll(ctx, tasks) - require.False(t, closed) - iter, ok = <-iterCh - require.True(t, ok) - require.NotNil(t, iter) - ok = iter.Seek([]byte("")) - require.False(t, ok, string(iter.Key())) - require.Nil(t, iter.Release()) - - // Close db. - closed = !ldb.Poll(ctx, []actormsg.Message{actormsg.StopMessage()}) - require.True(t, closed) - closedWg.Wait() - require.Nil(t, db.Close()) -} - -func TestAcquireIterators(t *testing.T) { - ctx := context.Background() - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - cfg.Count = 1 - - db, err := db.OpenLevelDB(ctx, 1, t.TempDir(), cfg) - require.Nil(t, err) - closedWg := new(sync.WaitGroup) - - // Set max iterator count to 1. - cfg.Concurrency = 1 - compact := NewCompactScheduler(actor.NewRouter(t.Name()), cfg) - ldb, _, err := NewDBActor(0, db, cfg, compact, closedWg, "") - require.Nil(t, err) - - // Poll two tasks. - tasks, iterCh1 := makeTask(make(map[message.Key][]byte), [][]byte{{0x00}, {0xff}}) - tasks[0].SorterTask.TableID = 1 - tasks2, iterCh2 := makeTask(make(map[message.Key][]byte), [][]byte{{0x00}, {0xff}}) - tasks2[0].SorterTask.TableID = 2 - tasks = append(tasks, tasks2...) - closed := !ldb.Poll(ctx, tasks) - require.False(t, closed) - iter, ok := <-iterCh1 - require.True(t, ok) - require.NotNil(t, iter) - - // Require iterator is not allow for now. - closed = !ldb.Poll(ctx, []actormsg.Message{actormsg.TickMessage()}) - require.False(t, closed) - select { - case <-iterCh2: - require.FailNow(t, "should not acquire an iterator") - default: - } - - // Release iter and iterCh2 should be able to receive an iterator. - require.Nil(t, iter.Release()) - closed = !ldb.Poll(ctx, []actormsg.Message{actormsg.TickMessage()}) - require.False(t, closed) - iter, ok = <-iterCh2 - require.True(t, ok) - require.Nil(t, iter.Release()) - - // Close db. - closed = !ldb.Poll(ctx, []actormsg.Message{actormsg.StopMessage()}) - require.True(t, closed) - closedWg.Wait() - require.Nil(t, db.Close()) -} - -type sortedMap struct { - // sorted keys - kvs map[message.Key][]byte -} - -func (s *sortedMap) put(k message.Key, v []byte) { - s.kvs[k] = v -} - -func (s *sortedMap) delete(k message.Key) { - delete(s.kvs, k) -} - -func (s *sortedMap) iter(start, end message.Key) []message.Key { - keys := make([]message.Key, 0) - for k := range s.kvs { - key := k - // [start, end) - if bytes.Compare([]byte(key), []byte(start)) >= 0 && - bytes.Compare([]byte(key), []byte(end)) < 0 { - keys = append(keys, key) - } - } - sort.Sort(sortableKeys(keys)) - return keys -} - -type sortableKeys []message.Key - -func (x sortableKeys) Len() int { return len(x) } -func (x sortableKeys) Less(i, j int) bool { return bytes.Compare([]byte(x[i]), []byte(x[j])) < 0 } -func (x sortableKeys) Swap(i, j int) { x[i], x[j] = x[j], x[i] } - -func TestModelChecking(t *testing.T) { - t.Parallel() - - seed := time.Now().Unix() - rd := rand.New(rand.NewSource(seed)) - ctx := context.Background() - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - cfg.Count = 1 - - db, err := db.OpenLevelDB(ctx, 1, t.TempDir(), cfg) - require.Nil(t, err) - closedWg := new(sync.WaitGroup) - compact := NewCompactScheduler(actor.NewRouter(t.Name()), cfg) - ldb, _, err := NewDBActor(0, db, cfg, compact, closedWg, "") - require.Nil(t, err) - - minKey := message.Key("") - maxKey := message.Key(bytes.Repeat([]byte{0xff}, 100)) - randKey := func() []byte { - // At least 10 bytes. - key := make([]byte, rd.Intn(90)+10) - n, err := rd.Read(key) - require.Greater(t, n, 0) - require.Nil(t, err) - return key - } - model := sortedMap{kvs: make(map[message.Key][]byte)} - // Prepare 100 key value pairs. - for i := 0; i < 100; i++ { - key := randKey() - value := key - - // Put to model. - model.put(message.Key(key), value) - // Put to db. - tasks, _ := makeTask(map[message.Key][]byte{message.Key(key): value}, nil) - closed := !ldb.Poll(ctx, tasks) - require.False(t, closed) - } - - // 100 random tests. - for i := 0; i < 100; i++ { - // [1, 4] ops - ops := rd.Intn(4) + 1 - for j := 0; j < ops; j++ { - switch rd.Intn(2) { - // 0 for put. - case 0: - key := randKey() - value := key - - model.put(message.Key(key), value) - tasks, _ := makeTask(map[message.Key][]byte{message.Key(key): value}, nil) - closed := !ldb.Poll(ctx, tasks) - require.False(t, closed) - - // 1 for delete. - case 1: - keys := model.iter(minKey, maxKey) - delKey := keys[rd.Intn(len(keys))] - model.delete(delKey) - tasks, _ := makeTask(map[message.Key][]byte{delKey: {}}, nil) - closed := !ldb.Poll(ctx, tasks) - require.False(t, closed) - } - } - - tasks, iterCh := makeTask(map[message.Key][]byte{}, - [][]byte{[]byte(minKey), []byte(maxKey)}) - closed := !ldb.Poll(ctx, tasks) - require.False(t, closed) - iter := <-iterCh - iter.Seek([]byte(minKey)) - keys := model.iter(minKey, maxKey) - for idx, key := range keys { - require.EqualValues(t, key, iter.Key()) - require.EqualValues(t, model.kvs[key], iter.Value()) - ok := iter.Next() - require.Equal(t, ok, idx != len(keys)-1, - "index %d, len(model): %d, seed: %d", idx, len(model.kvs), seed) - } - require.Nil(t, iter.Release()) - } - - // Close db. - closed := !ldb.Poll(ctx, []actormsg.Message{actormsg.StopMessage()}) - require.True(t, closed) - require.Nil(t, db.Close()) -} - -func TestContextCancel(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - cfg.Count = 1 - - db, err := db.OpenLevelDB(ctx, 1, t.TempDir(), cfg) - require.Nil(t, err) - closedWg := new(sync.WaitGroup) - compact := NewCompactScheduler(actor.NewRouter(t.Name()), cfg) - ldb, _, err := NewDBActor(0, db, cfg, compact, closedWg, "") - require.Nil(t, err) - - cancel() - tasks, _ := makeTask(map[message.Key][]byte{"key": {}}, [][]byte{{0x00}, {0xff}}) - closed := !ldb.Poll(ctx, tasks) - require.True(t, closed) - closedWg.Wait() - require.Nil(t, db.Close()) -} diff --git a/cdc/cdc/sorter/leveldb/message/task.go b/cdc/cdc/sorter/leveldb/message/task.go deleted file mode 100644 index ad32d016..00000000 --- a/cdc/cdc/sorter/leveldb/message/task.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package message - -import ( - "fmt" - - "github.com/pingcap/errors" - "github.com/tikv/migration/cdc/cdc/sorter/encoding" - "github.com/tikv/migration/cdc/pkg/db" - "golang.org/x/sync/semaphore" -) - -// Task is a db actor task. It carries write and read request. -type Task struct { - UID uint32 - TableID uint64 - - // encoded key -> serde.marshal(event) - // If a value is empty, it deletes the key/value entry in db. - Events map[Key][]byte - // Requests an iterator when it is not nil. - IterReq *IterRequest - - // For clean-up table task. - Cleanup bool - CleanupRatelimited bool -} - -// IterRequest contains parameters that necessary to build an iterator. -type IterRequest struct { - UID uint32 - - // The resolved ts at the time of issuing the request. - ResolvedTs uint64 - // Range of a requested iterator. - Range [2][]byte - // Must be buffered channel to avoid blocking. - IterCh chan *LimitedIterator `json:"-"` // Make Task JSON printable. -} - -// Key is the key that is written to db. -type Key string - -// String returns a pretty printed string. -func (k Key) String() string { - uid, tableID, startTs, CRTs := encoding.DecodeKey([]byte(k)) - return fmt.Sprintf( - "uid: %d, tableID: %d, startTs: %d, CRTs: %d", - uid, tableID, startTs, CRTs) -} - -// LimitedIterator is a wrapper of db.Iterator that has a sema to limit -// the total number of alive iterator. -type LimitedIterator struct { - db.Iterator - Sema *semaphore.Weighted - ResolvedTs uint64 -} - -// Release resources of the snapshot. -func (s *LimitedIterator) Release() error { - s.Sema.Release(1) - return errors.Trace(s.Iterator.Release()) -} - -// NewCleanupTask returns a clean up task to clean up table data. -func NewCleanupTask(uid uint32, tableID uint64) Task { - return Task{ - TableID: tableID, - UID: uid, - Cleanup: true, - } -} diff --git a/cdc/cdc/sorter/leveldb/message/task_test.go b/cdc/cdc/sorter/leveldb/message/task_test.go deleted file mode 100644 index 777c9456..00000000 --- a/cdc/cdc/sorter/leveldb/message/task_test.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package message - -import ( - "testing" - - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/sorter/encoding" -) - -func TestPrint(t *testing.T) { - t.Parallel() - event := model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypeDelete, - Key: []byte{1}, - StartTs: 3, - CRTs: 4, - }) - - require.Equal(t, "uid: 1, tableID: 2, startTs: 3, CRTs: 4", - Key(encoding.EncodeKey(1, 2, event)).String()) - require.Equal(t, "uid: 1, tableID: 2, startTs: 0, CRTs: 3", - Key(encoding.EncodeTsKey(1, 2, 3)).String()) -} - -func TestNewCleanupTask(t *testing.T) { - t.Parallel() - task := NewCleanupTask(1, 2) - require.True(t, task.Cleanup) - require.EqualValues(t, 1, task.UID) - require.EqualValues(t, 2, task.TableID) -} diff --git a/cdc/cdc/sorter/leveldb/metrics.go b/cdc/cdc/sorter/leveldb/metrics.go deleted file mode 100644 index ff12d5c7..00000000 --- a/cdc/cdc/sorter/leveldb/metrics.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package leveldb - -import ( - "github.com/prometheus/client_golang/prometheus" -) - -var ( - sorterWriteBytesHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "ticdc", - Subsystem: "sorter", - Name: "db_write_bytes", - Help: "Bucketed histogram of sorter write batch bytes", - Buckets: prometheus.ExponentialBuckets(16, 2.0, 20), - }, []string{"capture", "id"}) - - sorterWriteDurationHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "ticdc", - Subsystem: "sorter", - Name: "db_write_duration_seconds", - Help: "Bucketed histogram of sorter write duration", - Buckets: prometheus.ExponentialBuckets(0.004, 2.0, 20), - }, []string{"capture", "id"}) - - sorterCompactDurationHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "ticdc", - Subsystem: "sorter", - Name: "db_compact_duration_seconds", - Help: "Bucketed histogram of sorter manual compact duration", - Buckets: prometheus.ExponentialBuckets(0.004, 2.0, 20), - }, []string{"capture", "id"}) - - sorterIterReadDurationHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "ticdc", - Subsystem: "sorter", - Name: "db_iter_read_duration_seconds", - Help: "Bucketed histogram of db sorter iterator read duration", - Buckets: prometheus.ExponentialBuckets(0.004, 2.0, 20), - }, []string{"capture", "id", "call"}) - - sorterCleanupKVCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "ticdc", - Subsystem: "sorter", - Name: "db_cleanup_kv_total", - Help: "The total number of cleaned up kv entries", - }, []string{"capture", "id"}) -) - -// InitMetrics registers all metrics in this file -func InitMetrics(registry *prometheus.Registry) { - registry.MustRegister(sorterWriteDurationHistogram) - registry.MustRegister(sorterCompactDurationHistogram) - registry.MustRegister(sorterWriteBytesHistogram) - registry.MustRegister(sorterIterReadDurationHistogram) - registry.MustRegister(sorterCleanupKVCounter) -} diff --git a/cdc/cdc/sorter/leveldb/system/system.go b/cdc/cdc/sorter/leveldb/system/system.go deleted file mode 100644 index 2bd96497..00000000 --- a/cdc/cdc/sorter/leveldb/system/system.go +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package system - -import ( - "context" - "encoding/binary" - "hash/fnv" - "sync" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/log" - lsorter "github.com/tikv/migration/cdc/cdc/sorter/leveldb" - "github.com/tikv/migration/cdc/pkg/actor" - "github.com/tikv/migration/cdc/pkg/actor/message" - "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/db" - cerrors "github.com/tikv/migration/cdc/pkg/errors" - "go.uber.org/zap" -) - -// The interval of collecting db metrics. -const defaultMetricInterval = 15 * time.Second - -// State of a system. -type sysState int - -const ( - sysStateInit sysState = iota - sysStateStarted - sysStateStopped -) - -// System manages db sorter resource. -type System struct { - dbs []db.DB - dbSystem *actor.System - dbRouter *actor.Router - cleanSystem *actor.System - cleanRouter *actor.Router - compactSystem *actor.System - compactRouter *actor.Router - compactSched *lsorter.CompactScheduler - dir string - cfg *config.DBConfig - closedCh chan struct{} - closedWg *sync.WaitGroup - - state sysState - stateMu *sync.Mutex -} - -// NewSystem returns a system. -func NewSystem(dir string, cfg *config.DBConfig) *System { - dbSystem, dbRouter := actor.NewSystemBuilder("sorter"). - WorkerNumber(cfg.Count).Build() - cleanSystem, cleanRouter := actor.NewSystemBuilder("cleaner"). - WorkerNumber(cfg.Count).Build() - compactSystem, compactRouter := actor.NewSystemBuilder("compactor"). - WorkerNumber(cfg.Count).Build() - compactSched := lsorter.NewCompactScheduler(compactRouter, cfg) - return &System{ - dbSystem: dbSystem, - dbRouter: dbRouter, - cleanSystem: cleanSystem, - cleanRouter: cleanRouter, - compactSystem: compactSystem, - compactRouter: compactRouter, - compactSched: compactSched, - dir: dir, - cfg: cfg, - closedCh: make(chan struct{}), - closedWg: new(sync.WaitGroup), - state: sysStateInit, - stateMu: new(sync.Mutex), - } -} - -// ActorID returns an ActorID correspond with tableID. -func (s *System) ActorID(tableID uint64) actor.ID { - h := fnv.New64() - b := [8]byte{} - binary.LittleEndian.PutUint64(b[:], tableID) - h.Write(b[:]) - return actor.ID(h.Sum64() % uint64(s.cfg.Count)) -} - -// Router returns db actors router. -func (s *System) Router() *actor.Router { - return s.dbRouter -} - -// CleanerRouter returns cleaner actors router. -func (s *System) CleanerRouter() *actor.Router { - return s.cleanRouter -} - -// CompactScheduler returns compaction scheduler. -func (s *System) CompactScheduler() *lsorter.CompactScheduler { - return s.compactSched -} - -// broadcase messages to actors in the router. -// Caveats it may lose messages quietly. -func (s *System) broadcast(ctx context.Context, router *actor.Router, msg message.Message) { - dbCount := s.cfg.Count - for id := 0; id < dbCount; id++ { - err := router.SendB(ctx, actor.ID(id), msg) - if err != nil { - log.Warn("broadcast message failed", - zap.Int("ID", id), zap.Any("message", msg)) - } - } -} - -// Start starts a system. -func (s *System) Start(ctx context.Context) error { - s.stateMu.Lock() - defer s.stateMu.Unlock() - if s.state == sysStateStarted { - // Already started. - return nil - } else if s.state == sysStateStopped { - return cerrors.ErrStartAStoppedLevelDBSystem.GenWithStackByArgs() - } - s.state = sysStateStarted - - s.compactSystem.Start(ctx) - s.dbSystem.Start(ctx) - s.cleanSystem.Start(ctx) - captureAddr := config.GetGlobalServerConfig().AdvertiseAddr - dbCount := s.cfg.Count - for id := 0; id < dbCount; id++ { - // Open db. - db, err := db.OpenPebble(ctx, id, s.dir, s.cfg) - if err != nil { - return errors.Trace(err) - } - s.dbs = append(s.dbs, db) - // Create and spawn compactor actor. - compactor, cmb, err := - lsorter.NewCompactActor(id, db, s.closedWg, captureAddr) - if err != nil { - return errors.Trace(err) - } - err = s.compactSystem.Spawn(cmb, compactor) - if err != nil { - return errors.Trace(err) - } - // Create and spawn db actor. - dbac, dbmb, err := - lsorter.NewDBActor(id, db, s.cfg, s.compactSched, s.closedWg, captureAddr) - if err != nil { - return errors.Trace(err) - } - err = s.dbSystem.Spawn(dbmb, dbac) - if err != nil { - return errors.Trace(err) - } - // Create and spawn cleaner actor. - clac, clmb, err := lsorter.NewCleanerActor( - id, db, s.cleanRouter, s.compactSched, s.cfg, s.closedWg) - if err != nil { - return errors.Trace(err) - } - err = s.cleanSystem.Spawn(clmb, clac) - if err != nil { - return errors.Trace(err) - } - } - s.closedWg.Add(1) - go func() { - defer s.closedWg.Done() - metricsTimer := time.NewTimer(defaultMetricInterval) - defer metricsTimer.Stop() - for { - select { - case <-ctx.Done(): - return - case <-s.closedCh: - return - case <-metricsTimer.C: - collectMetrics(s.dbs, captureAddr) - metricsTimer.Reset(defaultMetricInterval) - } - } - }() - return nil -} - -// Stop stops a system. -func (s *System) Stop() error { - s.stateMu.Lock() - defer s.stateMu.Unlock() - switch s.state { - case sysStateStopped: - // Already stopped. - return nil - case sysStateInit: - // Not started. - return nil - } - s.state = sysStateStopped - - // TODO caller should pass context. - deadline := time.Now().Add(1 * time.Second) - ctx, cancel := context.WithDeadline(context.Background(), deadline) - defer cancel() - // Close actors - s.broadcast(ctx, s.dbRouter, message.StopMessage()) - s.broadcast(ctx, s.cleanRouter, message.StopMessage()) - s.broadcast(ctx, s.compactRouter, message.StopMessage()) - // Close metrics goroutine. - close(s.closedCh) - // Wait actors and metrics goroutine. - s.closedWg.Wait() - - // Stop systems. - err := s.dbSystem.Stop() - if err != nil { - return errors.Trace(err) - } - err = s.cleanSystem.Stop() - if err != nil { - return errors.Trace(err) - } - err = s.compactSystem.Stop() - if err != nil { - return errors.Trace(err) - } - - // Close dbs. - for _, db := range s.dbs { - err = db.Close() - if err != nil { - log.Warn("db close error", zap.Error(err)) - } - } - return nil -} - -func collectMetrics(dbs []db.DB, captureAddr string) { - for i := range dbs { - db := dbs[i] - db.CollectMetrics(captureAddr, i) - } -} diff --git a/cdc/cdc/sorter/leveldb/system/system_test.go b/cdc/cdc/sorter/leveldb/system/system_test.go deleted file mode 100644 index 3905d3ec..00000000 --- a/cdc/cdc/sorter/leveldb/system/system_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package system - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/pkg/config" -) - -func TestSystemStartStop(t *testing.T) { - t.Parallel() - ctx := context.Background() - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - cfg.Count = 1 - - sys := NewSystem(t.TempDir(), cfg) - require.Nil(t, sys.Start(ctx)) - require.Nil(t, sys.Stop()) - - // Close it again. - require.Nil(t, sys.Stop()) - // Start a closed system. - require.Error(t, sys.Start(ctx)) -} - -func TestSystemStopUnstarted(t *testing.T) { - t.Parallel() - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - cfg.Count = 1 - - sys := NewSystem(t.TempDir(), cfg) - require.Nil(t, sys.Stop()) -} - -func TestCollectMetrics(t *testing.T) { - t.Parallel() - ctx := context.Background() - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - cfg.Count = 2 - - sys := NewSystem(t.TempDir(), cfg) - require.Nil(t, sys.Start(ctx)) - collectMetrics(sys.dbs, "") - require.Nil(t, sys.Stop()) -} - -func TestActorID(t *testing.T) { - t.Parallel() - ctx := context.Background() - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - cfg.Count = 2 - - sys := NewSystem(t.TempDir(), cfg) - require.Nil(t, sys.Start(ctx)) - id1 := sys.ActorID(1) - id2 := sys.ActorID(1) - // tableID to actor ID must be deterministic. - require.Equal(t, id1, id2) - require.Nil(t, sys.Stop()) -} diff --git a/cdc/cdc/sorter/leveldb/table_sorter.go b/cdc/cdc/sorter/leveldb/table_sorter.go deleted file mode 100644 index 345a43f6..00000000 --- a/cdc/cdc/sorter/leveldb/table_sorter.go +++ /dev/null @@ -1,723 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package leveldb - -import ( - "context" - "math" - "sync/atomic" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/log" - "github.com/prometheus/client_golang/prometheus" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/sorter" - "github.com/tikv/migration/cdc/cdc/sorter/encoding" - "github.com/tikv/migration/cdc/cdc/sorter/leveldb/message" - "github.com/tikv/migration/cdc/pkg/actor" - actormsg "github.com/tikv/migration/cdc/pkg/actor/message" - "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/db" - "github.com/tikv/migration/cdc/pkg/util" - "go.uber.org/zap" -) - -const ( - // Capacity of db sorter input and output channels. - sorterInputCap, sorterOutputCap = 64, 64 - // Max size of received event batch. - batchReceiveEventSize = 32 -) - -var levelDBSorterIDAlloc uint32 = 0 - -func allocID() uint32 { - return atomic.AddUint32(&levelDBSorterIDAlloc, 1) -} - -// Sorter accepts out-of-order raw kv entries and output sorted entries -type Sorter struct { - actorID actor.ID - router *actor.Router - compact *CompactScheduler - uid uint32 - tableID uint64 - serde *encoding.MsgPackGenSerde - - iterMaxAliveDuration time.Duration - iterFirstSlowDuration time.Duration - - lastSentResolvedTs uint64 - lastEvent *model.PolymorphicEvent - - inputCh chan *model.PolymorphicEvent - outputCh chan *model.PolymorphicEvent - - closed int32 - - metricTotalEventsKV prometheus.Counter - metricTotalEventsResolvedTs prometheus.Counter - metricIterDuration prometheus.ObserverVec - metricIterReadDuration prometheus.Observer - metricIterNextDuration prometheus.Observer -} - -// NewSorter creates a new Sorter -func NewSorter( - ctx context.Context, tableID int64, startTs uint64, - router *actor.Router, actorID actor.ID, compact *CompactScheduler, - cfg *config.DBConfig, -) *Sorter { - captureAddr := util.CaptureAddrFromCtx(ctx) - changefeedID := util.ChangefeedIDFromCtx(ctx) - metricIterDuration := sorterIterReadDurationHistogram.MustCurryWith( - prometheus.Labels{"capture": captureAddr, "id": changefeedID}) - return &Sorter{ - actorID: actorID, - router: router, - compact: compact, - uid: allocID(), - tableID: uint64(tableID), - lastSentResolvedTs: startTs, - serde: &encoding.MsgPackGenSerde{}, - - iterMaxAliveDuration: time.Duration(cfg.IteratorMaxAliveDuration) * time.Millisecond, - iterFirstSlowDuration: time.Duration(cfg.IteratorSlowReadDuration) * time.Millisecond, - - inputCh: make(chan *model.PolymorphicEvent, sorterInputCap), - outputCh: make(chan *model.PolymorphicEvent, sorterOutputCap), - - metricTotalEventsKV: sorter.EventCount.WithLabelValues(captureAddr, changefeedID, "kv"), - metricTotalEventsResolvedTs: sorter.EventCount.WithLabelValues(captureAddr, changefeedID, "resolved"), - metricIterDuration: metricIterDuration, - metricIterReadDuration: metricIterDuration.WithLabelValues("read"), - metricIterNextDuration: metricIterDuration.WithLabelValues("next"), - } -} - -func (ls *Sorter) waitInput(ctx context.Context) (*model.PolymorphicEvent, error) { - select { - case <-ctx.Done(): - return nil, errors.Trace(ctx.Err()) - case ev := <-ls.inputCh: - return ev, nil - } -} - -func (ls *Sorter) waitInputOutput( - ctx context.Context, -) (*model.PolymorphicEvent, error) { - // A dummy event for detecting whether output is available. - dummyEvent := model.NewResolvedPolymorphicEvent(0, 0) - select { - // Prefer receiving input events. - case ev := <-ls.inputCh: - return ev, nil - default: - select { - case <-ctx.Done(): - return nil, errors.Trace(ctx.Err()) - case ev := <-ls.inputCh: - return ev, nil - case ls.outputCh <- dummyEvent: - return nil, nil - } - } -} - -// wait input or output becomes available. -// It returns -// 1) the max commit ts of received new events, -// 2) the max resolved ts of new resolvedTs events, -// 3) number of received new events, -// 4) error. -// -// If input is available, it batches newly received events. -// If output available, it sends a dummy resolved ts event and returns. -func (ls *Sorter) wait( - ctx context.Context, waitOutput bool, events []*model.PolymorphicEvent, -) (uint64, uint64, int, error) { - batchSize := len(events) - if batchSize <= 0 { - log.Panic("batch size must be larger than 0") - } - maxCommitTs, maxResolvedTs := uint64(0), uint64(0) - inputCount, kvEventCount, resolvedEventCount := 0, 0, 0 - appendInputEvent := func(ev *model.PolymorphicEvent) { - if ls.lastSentResolvedTs != 0 && ev.CRTs < ls.lastSentResolvedTs { - // Since TiKV/Puller may send out of order or duplicated events, - // we should not panic here. - // Regression is not a common case, use warn level to rise our - // attention. - log.Warn("commit ts < resolved ts", - zap.Uint64("lastSentResolvedTs", ls.lastSentResolvedTs), - zap.Any("event", ev), zap.Uint64("regionID", ev.RegionID())) - return - } - if ev.RawKV.OpType == model.OpTypeResolved { - if maxResolvedTs < ev.CRTs { - maxResolvedTs = ev.CRTs - } - resolvedEventCount++ - } else { - if maxCommitTs < ev.CRTs { - maxCommitTs = ev.CRTs - } - events[inputCount] = ev - inputCount++ - kvEventCount++ - } - } - - if waitOutput { - // Wait intput and output. - ev, err := ls.waitInputOutput(ctx) - if err != nil { - atomic.StoreInt32(&ls.closed, 1) - close(ls.outputCh) - return 0, 0, 0, errors.Trace(ctx.Err()) - } - if ev == nil { - // No input event and output is available. - return maxCommitTs, maxResolvedTs, 0, nil - } - appendInputEvent(ev) - } else { - // Wait input only. - ev, err := ls.waitInput(ctx) - if err != nil { - atomic.StoreInt32(&ls.closed, 1) - close(ls.outputCh) - return 0, 0, 0, errors.Trace(ctx.Err()) - } - appendInputEvent(ev) - } - - // Batch receive events -BATCH: - for inputCount < batchSize { - select { - case ev := <-ls.inputCh: - appendInputEvent(ev) - default: - break BATCH - } - } - ls.metricTotalEventsKV.Add(float64(kvEventCount)) - ls.metricTotalEventsResolvedTs.Add(float64(resolvedEventCount)) - - // Release buffered events to help GC reclaim memory. - for i := inputCount; i < batchSize; i++ { - events[i] = nil - } - return maxCommitTs, maxResolvedTs, inputCount, nil -} - -// buildTask build a task for writing new events and delete outputted events. -func (ls *Sorter) buildTask( - events []*model.PolymorphicEvent, deleteKeys []message.Key, -) (message.Task, error) { - writes := make(map[message.Key][]byte) - for i := range events { - event := events[i] - if event.RawKV.OpType == model.OpTypeResolved { - continue - } - - key := encoding.EncodeKey(ls.uid, ls.tableID, event) - value := []byte{} - var err error - value, err = ls.serde.Marshal(event, value) - if err != nil { - return message.Task{}, errors.Trace(err) - } - writes[message.Key(key)] = value - } - - // Delete keys of outputted resolved events. - for i := range deleteKeys { - writes[deleteKeys[i]] = []byte{} - } - - return message.Task{ - UID: ls.uid, - TableID: ls.tableID, - Events: writes, - }, nil -} - -// output nonblocking outputs an event. Caller should retry when it returns false. -func (ls *Sorter) output(event *model.PolymorphicEvent) bool { - if ls.lastEvent == nil { - ls.lastEvent = event - } - if ls.lastEvent.CRTs > event.CRTs { - log.Panic("regression", - zap.Any("lastEntry", ls.lastEvent), zap.Any("event", event), - zap.Uint64("regionID", event.RegionID())) - } - select { - case ls.outputCh <- event: - ls.lastEvent = event - return true - default: - return false - } -} - -// outputResolvedTs nonblocking outputs a resolved ts event. -func (ls *Sorter) outputResolvedTs(rts model.Ts) { - ok := ls.output(model.NewResolvedPolymorphicEvent(0, rts)) - if ok { - ls.lastSentResolvedTs = rts - } -} - -// outputBufferedResolvedEvents nonblocking output resolved events and -// resolved ts that are buffered in outputBuffer. -// It pops outputted events in the buffer and append their key to deleteKeys. -func (ls *Sorter) outputBufferedResolvedEvents( - buffer *outputBuffer, sendResolvedTsHint bool, -) { - hasRemainEvents := false - // Index of remaining output events - remainIdx := 0 - // Commit ts of the last outputted events. - lastCommitTs := uint64(0) - for idx := range buffer.resolvedEvents { - event := buffer.resolvedEvents[idx] - ok := ls.output(event) - if !ok { - hasRemainEvents = true - break - } - lastCommitTs = event.CRTs - - // Delete sent events. - key := encoding.EncodeKey(ls.uid, ls.tableID, event) - buffer.appendDeleteKey(message.Key(key)) - remainIdx = idx + 1 - } - // Remove outputted events. - buffer.shiftResolvedEvents(remainIdx) - - // If all buffered resolved events are sent, send its resolved ts too. - if sendResolvedTsHint && lastCommitTs != 0 && !hasRemainEvents { - ls.outputResolvedTs(lastCommitTs) - } -} - -// outputIterEvents nonblocking output resolved events that are buffered -// in leveldb. -// It appends outputted events's key to outputBuffer deleteKeys to delete them -// later, and appends resolved events to outputBuffer resolvedEvents to send -// them later. -// -// It returns: -// * a bool to indicate whether it has read the last Next or not. -// * a uint64, if it is not 0, it means all resolved events before the ts -// are outputted. -// * an error if it occurs. -// -// Note: outputBuffer must be empty. -func (ls *Sorter) outputIterEvents( - iter db.Iterator, hasReadLastNext bool, buffer *outputBuffer, - resolvedTs uint64, -) (bool, uint64, error) { - lenResolvedEvents, lenDeleteKeys := buffer.len() - if lenDeleteKeys > 0 || lenResolvedEvents > 0 { - log.Panic("buffer is not empty", - zap.Int("deleteKeys", lenDeleteKeys), - zap.Int("resolvedEvents", lenResolvedEvents)) - } - - // Commit ts of buffered resolved events. - commitTs := uint64(0) - start := time.Now() - lastNext := start - if hasReadLastNext { - // We have read the last key/value, move the Next. - iter.Next() - ls.metricIterNextDuration.Observe(time.Since(start).Seconds()) - } // else the last is not read, we need to skip calling Next and read again. - hasReadNext := true - hasNext := iter.Valid() - for ; hasNext; hasNext = iter.Next() { - now := time.Now() - ls.metricIterNextDuration.Observe(now.Sub(lastNext).Seconds()) - lastNext = now - - if iter.Error() != nil { - return false, 0, errors.Trace(iter.Error()) - } - event := new(model.PolymorphicEvent) - _, err := ls.serde.Unmarshal(event, iter.Value()) - if err != nil { - return false, 0, errors.Trace(err) - } - if commitTs > event.CRTs || commitTs > resolvedTs { - log.Panic("event commit ts regression", - zap.Any("event", event), zap.Stringer("key", message.Key(iter.Key())), - zap.Uint64("ts", commitTs), zap.Uint64("resolvedTs", resolvedTs)) - } - - if commitTs == 0 { - commitTs = event.CRTs - } - // Group resolved events that has the same commit ts. - if commitTs == event.CRTs { - buffer.appendResolvedEvent(event) - continue - } - // As a new event belongs to a new txn group, we need to output all - // buffered events before append the event. - ls.outputBufferedResolvedEvents(buffer, true) - lenResolvedEvents, _ = buffer.len() - if lenResolvedEvents > 0 { - // Output blocked, skip append new event. - // This means we have not read Next. - hasReadNext = false - break - } - - // Append new event to the buffer. - commitTs = event.CRTs - buffer.appendResolvedEvent(event) - } - elapsed := time.Since(start) - ls.metricIterReadDuration.Observe(elapsed.Seconds()) - - // Try shrink buffer to release memory. - buffer.maybeShrink() - - // Events have not been sent, buffer them and output them later. - // Do not let outputBufferedResolvedEvents output resolved ts, instead we - // output resolved ts here. - sendResolvedTsHint := false - ls.outputBufferedResolvedEvents(buffer, sendResolvedTsHint) - lenResolvedEvents, _ = buffer.len() - - // Skip output resolved ts if there is any buffered resolved event. - if lenResolvedEvents != 0 { - return hasReadNext, 0, nil - } - - if !hasNext && resolvedTs != 0 { - // Iter is exhausted and there is no resolved event (up to max - // resolved ts), output max resolved ts and return an exhausted - // resolved ts. - ls.outputResolvedTs(resolvedTs) - return hasReadNext, resolvedTs, nil - } - if commitTs != 0 { - // All buffered resolved events are outputted, - // output last commit ts. - ls.outputResolvedTs(commitTs) - } - - return hasReadNext, 0, nil -} - -type pollState struct { - // Buffer for receiveing new events from AddEntry. - eventsBuf []*model.PolymorphicEvent - // Buffer for resolved events and to-be-deleted events. - outputBuf *outputBuffer - // The maximum commit ts for all events. - maxCommitTs uint64 - // The maximum commit ts for all resolved ts events. - maxResolvedTs uint64 - // All resolved events before the resolved ts are outputted. - exhaustedResolvedTs uint64 - - // Compactor actor ID. - actorID actor.ID - // A scheduler that triggers db compaction to speed up Iterator.First(). - compact *CompactScheduler - // A threshold of triggering db compaction. - iterFirstSlowDuration time.Duration - // A timestamp when iterator was created. - // Iterator is released once it execced `iterMaxAliveDuration`. - iterAliveTime time.Time - iterMaxAliveDuration time.Duration - // A channel for receiving iterator asynchronously. - iterCh chan *message.LimitedIterator - // A iterator for reading resolved events, up to the `iterResolvedTs`. - iter *message.LimitedIterator - iterResolvedTs uint64 - // A flag to mark whether the current position has been read. - iterHasRead bool - - metricIterFirst prometheus.Observer - metricIterRelease prometheus.Observer -} - -func (state *pollState) hasResolvedEvents() bool { - // It has resolved events, if 1) it has buffer resolved events, - lenResolvedEvents, _ := state.outputBuf.len() - if lenResolvedEvents > 0 { - return true - } - // or 2) there are some events that can be resolved. - // -------|-----------------|-------------|-------> time - // exhaustedResolvedTs - // maxCommitTs - // maxResolvedTs - // -------|-----------------|-------------|-------> time - // exhaustedResolvedTs - // maxResolvedTs - // maxCommitTs - if state.exhaustedResolvedTs < state.maxCommitTs && - state.exhaustedResolvedTs < state.maxResolvedTs { - return true - } - - // Otherwise, there is no event can be resolved. - // -------|-----------------|-------------|-------> time - // maxCommitTs - // exhaustedResolvedTs - // maxResolvedTs - return false -} - -func (state *pollState) advanceMaxTs(maxCommitTs, maxResolvedTs uint64) { - // The max commit ts of all received events. - if maxCommitTs > state.maxCommitTs { - state.maxCommitTs = maxCommitTs - } - // The max resolved ts of all received resolvedTs events. - if maxResolvedTs > state.maxResolvedTs { - state.maxResolvedTs = maxResolvedTs - } -} - -// tryGetIterator tries to get an iterator. -// When it returns a request, caller must send it. -// When it returns true, it means there is an iterator that can be used. -func (state *pollState) tryGetIterator( - uid uint32, tableID uint64, -) (*message.IterRequest, bool) { - if state.iter != nil && state.iterCh != nil { - log.Panic("assert failed, there can only be one of iter or iterCh", - zap.Any("iter", state.iter), zap.Uint64("tableID", tableID), - zap.Uint32("uid", uid)) - } - - if state.iter != nil { - return nil, true - } - - if state.iterCh == nil { - // We haven't send request. - state.iterCh = make(chan *message.LimitedIterator, 1) - return &message.IterRequest{ - Range: [2][]byte{ - encoding.EncodeTsKey(uid, tableID, 0), - encoding.EncodeTsKey(uid, tableID, state.maxResolvedTs+1), - }, - ResolvedTs: state.maxResolvedTs, - IterCh: state.iterCh, - }, false - } - - // Try receive iterator. - select { - case iter := <-state.iterCh: - // Iterator received, reset state.iterCh - state.iterCh = nil - state.iter = iter - start := time.Now() - state.iterAliveTime = start - state.iterResolvedTs = iter.ResolvedTs - state.iterHasRead = false - state.iter.First() - duration := time.Since(start) - state.metricIterFirst.Observe(duration.Seconds()) - if duration >= state.iterFirstSlowDuration { - // Force trigger a compaction if Iterator.Fisrt is too slow. - state.compact.maybeCompact(state.actorID, int(math.MaxInt32)) - } - return nil, true - default: - // Iterator is not ready yet. - return nil, false - } -} - -func (state *pollState) tryReleaseIterator() error { - if state.iter == nil { - return nil - } - now := time.Now() - if !state.iter.Valid() || now.Sub(state.iterAliveTime) > state.iterMaxAliveDuration { - err := state.iter.Release() - if err != nil { - return errors.Trace(err) - } - state.metricIterRelease.Observe(time.Since(now).Seconds()) - state.iter = nil - state.iterHasRead = true - - if state.iterCh != nil { - log.Panic("there must not be iterCh", zap.Any("iter", state.iter)) - } - } - - return nil -} - -// poll receives new events and send resolved events asynchronously. -// TODO: Refactor into actor model, divide receive-send into two parts -// to reduce complexity. -func (ls *Sorter) poll(ctx context.Context, state *pollState) error { - // Wait input or output becomes available. - waitOutput := state.hasResolvedEvents() - // TODO: we should also wait state.iterCh, so that we can read and output - // resolved events ASAP. - maxCommitTs, maxResolvedTs, n, err := ls.wait(ctx, waitOutput, state.eventsBuf) - if err != nil { - return errors.Trace(err) - } - // The max commit ts and resolved ts of all received events. - state.advanceMaxTs(maxCommitTs, maxResolvedTs) - // Length of buffered resolved events. - lenResolvedEvents, _ := state.outputBuf.len() - if n == 0 && lenResolvedEvents != 0 { - // No new received events, it means output channel is available. - // output resolved events as much as possible. - ls.outputBufferedResolvedEvents(state.outputBuf, true) - lenResolvedEvents, _ = state.outputBuf.len() - } - // New received events. - newEvents := state.eventsBuf[:n] - // Build task for new events and delete sent keys. - task, err := ls.buildTask(newEvents, state.outputBuf.deleteKeys) - if err != nil { - return errors.Trace(err) - } - // Reset buffer as delete keys are scheduled. - state.outputBuf.resetDeleteKey() - // Try shrink buffer to release memory. - state.outputBuf.maybeShrink() - - // It can only read an iterator when - // 1. No buffered resolved events, they must be sent before - // sending further resolved events from iterator. - readIter := lenResolvedEvents == 0 - // 2. There are some events that can be resolved. - readIter = readIter && state.hasResolvedEvents() - if !readIter { - // No new events and no resolved events. - if !state.hasResolvedEvents() && state.maxResolvedTs != 0 { - ls.outputResolvedTs(state.maxResolvedTs) - } - // Release iterator as we does not need to read. - err := state.tryReleaseIterator() - if err != nil { - return errors.Trace(err) - } - // Send write task to leveldb. - return ls.router.SendB(ctx, ls.actorID, actormsg.SorterMessage(task)) - } - - var hasIter bool - task.IterReq, hasIter = state.tryGetIterator(ls.uid, ls.tableID) - // Send write/read task to leveldb. - err = ls.router.SendB(ctx, ls.actorID, actormsg.SorterMessage(task)) - if err != nil { - // Skip read iterator if send fails. - return errors.Trace(err) - } - if !hasIter { - // Skip read iterator if there is no iterator - return nil - } - - // Read and send resolved events from iterator. - hasReadNext, exhaustedResolvedTs, err := ls.outputIterEvents( - state.iter, state.iterHasRead, state.outputBuf, state.iterResolvedTs) - if err != nil { - return errors.Trace(err) - } - if exhaustedResolvedTs > state.exhaustedResolvedTs { - state.exhaustedResolvedTs = exhaustedResolvedTs - } - state.iterHasRead = hasReadNext - return state.tryReleaseIterator() -} - -// Run runs Sorter -func (ls *Sorter) Run(ctx context.Context) error { - state := &pollState{ - eventsBuf: make([]*model.PolymorphicEvent, batchReceiveEventSize), - outputBuf: newOutputBuffer(batchReceiveEventSize), - - maxCommitTs: uint64(0), - maxResolvedTs: uint64(0), - exhaustedResolvedTs: uint64(0), - - actorID: ls.actorID, - compact: ls.compact, - iterFirstSlowDuration: ls.iterFirstSlowDuration, - iterMaxAliveDuration: ls.iterMaxAliveDuration, - - metricIterFirst: ls.metricIterDuration.WithLabelValues("first"), - metricIterRelease: ls.metricIterDuration.WithLabelValues("release"), - } - for { - err := ls.poll(ctx, state) - if err != nil { - return errors.Trace(err) - } - } -} - -// AddEntry adds an RawKVEntry to the EntryGroup -func (ls *Sorter) AddEntry(ctx context.Context, event *model.PolymorphicEvent) { - if atomic.LoadInt32(&ls.closed) != 0 { - return - } - select { - case <-ctx.Done(): - case ls.inputCh <- event: - } -} - -// TryAddEntry tries to add an RawKVEntry to the EntryGroup -func (ls *Sorter) TryAddEntry( - ctx context.Context, event *model.PolymorphicEvent, -) (bool, error) { - if atomic.LoadInt32(&ls.closed) != 0 { - return false, nil - } - select { - case <-ctx.Done(): - return false, errors.Trace(ctx.Err()) - case ls.inputCh <- event: - return true, nil - default: - return false, nil - } -} - -// Output returns the sorted raw kv output channel -func (ls *Sorter) Output() <-chan *model.PolymorphicEvent { - return ls.outputCh -} - -// CleanupTask returns a clean up task that delete sorter's data. -func (ls *Sorter) CleanupTask() actormsg.Message { - return actormsg.SorterMessage(message.NewCleanupTask(ls.uid, ls.tableID)) -} diff --git a/cdc/cdc/sorter/leveldb/table_sorter_test.go b/cdc/cdc/sorter/leveldb/table_sorter_test.go deleted file mode 100644 index 21a2cdac..00000000 --- a/cdc/cdc/sorter/leveldb/table_sorter_test.go +++ /dev/null @@ -1,1116 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package leveldb - -import ( - "context" - "encoding/hex" - "testing" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/sorter/encoding" - "github.com/tikv/migration/cdc/cdc/sorter/leveldb/message" - "github.com/tikv/migration/cdc/pkg/actor" - actormsg "github.com/tikv/migration/cdc/pkg/actor/message" - "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/db" - "golang.org/x/sync/semaphore" -) - -func newTestSorter( - ctx context.Context, capacity int, -) (*Sorter, actor.Mailbox) { - id := actor.ID(1) - router := actor.NewRouter("test") - mb := actor.NewMailbox(1, capacity) - router.InsertMailbox4Test(id, mb) - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - compact := NewCompactScheduler(nil, cfg) - ls := NewSorter(ctx, 1, 1, router, id, compact, cfg) - return ls, mb -} - -func TestInputOutOfOrder(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // Poll twice. - capacity := 2 - require.Greater(t, batchReceiveEventSize, capacity) - ls, _ := newTestSorter(ctx, capacity) - - ls.AddEntry(ctx, model.NewResolvedPolymorphicEvent(0, 2)) - ls.AddEntry(ctx, model.NewResolvedPolymorphicEvent(0, 3)) - require.Nil(t, ls.poll(ctx, &pollState{ - eventsBuf: make([]*model.PolymorphicEvent, 1), - outputBuf: newOutputBuffer(1), - })) - require.EqualValues(t, model.NewResolvedPolymorphicEvent(0, 3), <-ls.Output()) - - ls.AddEntry(ctx, model.NewResolvedPolymorphicEvent(0, 2)) - require.Nil(t, ls.poll(ctx, &pollState{ - eventsBuf: make([]*model.PolymorphicEvent, 1), - outputBuf: newOutputBuffer(1), - })) -} - -func TestWaitInput(t *testing.T) { - t.Parallel() - // Make sure input capacity is larger than batch size in order to test - // batch behavior. - require.Greater(t, sorterInputCap, batchReceiveEventSize) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - capacity := 8 - require.Greater(t, batchReceiveEventSize, capacity) - ls, _ := newTestSorter(ctx, capacity) - // Nonbuffered channel is unavailable during the test. - ls.outputCh = make(chan *model.PolymorphicEvent) - - expectedEvents := make([]*model.PolymorphicEvent, batchReceiveEventSize) - for i := range expectedEvents { - expectedEvents[i] = model.NewPolymorphicEvent( - &model.RawKVEntry{CRTs: ls.lastSentResolvedTs, RegionID: uint64(i)}) - } - - eventsBuf := make([]*model.PolymorphicEvent, batchReceiveEventSize) - - // Test message count <= batchReceiveEventSize. - for i := 1; i <= batchReceiveEventSize; i++ { - for j := 0; j < i; j++ { - ls.inputCh <- model.NewPolymorphicEvent( - &model.RawKVEntry{CRTs: ls.lastSentResolvedTs, RegionID: uint64(j)}) - } - cts, rts, n, err := ls.wait(ctx, false, eventsBuf) - require.Nil(t, err) - require.Equal(t, i, n) - require.EqualValues(t, 0, rts) - require.EqualValues(t, ls.lastSentResolvedTs, cts) - require.EqualValues(t, expectedEvents[:n], eventsBuf[:n]) - } - - // Test message count > batchReceiveEventSize - for i := batchReceiveEventSize + 1; i <= sorterInputCap; i++ { - expectedEvents1 := make([]*model.PolymorphicEvent, i) - for j := 0; j < i; j++ { - ls.inputCh <- model.NewPolymorphicEvent( - &model.RawKVEntry{CRTs: ls.lastSentResolvedTs, RegionID: uint64(j)}) - expectedEvents1[j] = model.NewPolymorphicEvent( - &model.RawKVEntry{CRTs: ls.lastSentResolvedTs, RegionID: uint64(j)}) - } - - quotient, remainder := i/batchReceiveEventSize, i%batchReceiveEventSize - for q := 0; q < quotient; q++ { - cts, rts, n, err := ls.wait(ctx, false, eventsBuf) - require.Nil(t, err) - require.Equal(t, batchReceiveEventSize, n) - require.EqualValues(t, 0, rts) - require.EqualValues(t, ls.lastSentResolvedTs, cts) - start, end := q*batchReceiveEventSize, q*batchReceiveEventSize+n - require.EqualValues(t, expectedEvents1[start:end], eventsBuf[:n], - "%d, %d, %d, %d", i, quotient, remainder, n) - } - if remainder != 0 { - cts, rts, n, err := ls.wait(ctx, false, eventsBuf) - require.Nil(t, err) - require.Equal(t, remainder, n) - require.EqualValues(t, 0, rts) - require.EqualValues(t, ls.lastSentResolvedTs, cts) - start, end := quotient*batchReceiveEventSize, quotient*batchReceiveEventSize+n - require.EqualValues(t, expectedEvents1[start:end], eventsBuf[:n], - "%d, %d, %d, %d", i, quotient, remainder, n) - } - } - - // Test returned max resolved ts of new resolvedts events. - // Send batchReceiveEventSize/3 resolved events - for i := 1; i <= batchReceiveEventSize/3; i++ { - ls.inputCh <- model.NewResolvedPolymorphicEvent(0, uint64(i)) - } - // Send batchReceiveEventSize/3 events - for i := 0; i < batchReceiveEventSize/3; i++ { - ls.inputCh <- model.NewPolymorphicEvent( - &model.RawKVEntry{CRTs: ls.lastSentResolvedTs, RegionID: uint64(i)}) - } - _, rts, n, err := ls.wait(ctx, false, eventsBuf) - require.Nil(t, err) - require.EqualValues(t, batchReceiveEventSize/3, n) - require.EqualValues(t, batchReceiveEventSize/3, rts) - require.EqualValues(t, expectedEvents[:n], eventsBuf[:n]) - - // Test returned max commit ts of new events - // Send batchReceiveEventSize/2 events - for i := 1; i <= batchReceiveEventSize/2; i++ { - ls.inputCh <- model.NewPolymorphicEvent( - &model.RawKVEntry{CRTs: uint64(i), RegionID: uint64(i)}) - } - cts, rts, n, err := ls.wait(ctx, false, eventsBuf) - require.Nil(t, err) - require.EqualValues(t, batchReceiveEventSize/2, n) - require.EqualValues(t, batchReceiveEventSize/2, cts) - require.EqualValues(t, 0, rts) - - // Test input block on empty message. - dctx, dcancel := context.WithDeadline(ctx, time.Now().Add(100*time.Millisecond)) - defer dcancel() - cts, rts, n, err = ls.wait(dctx, false, eventsBuf) - require.Regexp(t, err, "context deadline exceeded") - require.Equal(t, 0, n) - require.EqualValues(t, 0, cts) - require.EqualValues(t, 0, rts) -} - -func TestWaitOutput(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - capacity := 4 - require.Greater(t, batchReceiveEventSize, capacity) - ls, _ := newTestSorter(ctx, capacity) - - eventsBuf := make([]*model.PolymorphicEvent, batchReceiveEventSize) - - waitOutput := true - // It sends a dummy event if there is no buffered event. - cts, rts, n, err := ls.wait(ctx, waitOutput, eventsBuf) - require.Nil(t, err) - require.EqualValues(t, 0, n) - require.EqualValues(t, 0, cts) - require.EqualValues(t, 0, rts) - require.EqualValues(t, - model.NewResolvedPolymorphicEvent(0, 0), <-ls.outputCh) - - // Test wait block when output channel is unavailable. - ls.outputCh = make(chan *model.PolymorphicEvent) - dctx, dcancel := context.WithDeadline(ctx, time.Now().Add(100*time.Millisecond)) - defer dcancel() - cts, rts, n, err = ls.wait(dctx, waitOutput, eventsBuf) - require.Regexp(t, err, "context deadline exceeded") - require.Equal(t, 0, n) - require.EqualValues(t, 0, cts) - require.EqualValues(t, 0, rts) -} - -func TestBuildTask(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - capacity := 4 - require.Greater(t, batchReceiveEventSize, capacity) - ls, _ := newTestSorter(ctx, capacity) - - cases := []struct { - events []*model.PolymorphicEvent - deleteKeys []message.Key - }{ - // Empty write and delete. - { - events: []*model.PolymorphicEvent{}, - deleteKeys: []message.Key{}, - }, - // Write one event - { - events: []*model.PolymorphicEvent{ - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 1}), - }, - deleteKeys: []message.Key{}, - }, - // Write one event and delete one key. - { - events: []*model.PolymorphicEvent{ - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 1}), - }, - deleteKeys: []message.Key{ - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 2}))), - }, - }, - // Write two events and delete one key. - { - events: []*model.PolymorphicEvent{ - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 4}), - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 6}), - }, - deleteKeys: []message.Key{ - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 1}))), - }, - }, - } - for i, cs := range cases { - events, deleteKeys := cs.events, cs.deleteKeys - task, err := ls.buildTask(events, deleteKeys) - require.Nil(t, err, "case #%d, %v", i, cs) - - expectedEvents := make(map[message.Key][]uint8) - for _, ev := range events { - value, err := ls.serde.Marshal(ev, []byte{}) - require.Nil(t, err, "case #%d, %v", i, cs) - key := message.Key(encoding.EncodeKey(ls.uid, ls.tableID, ev)) - expectedEvents[key] = value - } - for _, key := range deleteKeys { - expectedEvents[key] = []byte{} - } - require.EqualValues(t, message.Task{ - UID: ls.uid, - TableID: ls.tableID, - Events: expectedEvents, - Cleanup: false, - CleanupRatelimited: false, - }, task, "case #%d, %v", i, cs) - } -} - -func TestOutput(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - capacity := 4 - ls, _ := newTestSorter(ctx, capacity) - - ls.outputCh = make(chan *model.PolymorphicEvent, 1) - ok := ls.output(&model.PolymorphicEvent{CRTs: 1}) - require.True(t, ok) - require.EqualValues(t, &model.PolymorphicEvent{CRTs: 1}, ls.lastEvent) - ok = ls.output(&model.PolymorphicEvent{CRTs: 1}) - require.False(t, ok) - ls.outputResolvedTs(2) - require.EqualValues(t, 1, ls.lastSentResolvedTs) - - <-ls.outputCh - ls.outputResolvedTs(2) - require.EqualValues(t, 2, ls.lastSentResolvedTs) - - <-ls.outputCh - ok = ls.output(&model.PolymorphicEvent{CRTs: 3}) - require.True(t, ok) -} - -func TestOutputBufferedResolvedEvents(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - capacity := 4 - ls, _ := newTestSorter(ctx, capacity) - - buf := newOutputBuffer(capacity) - - cases := []struct { - outputChCap int - inputEvents []*model.PolymorphicEvent - inputDeleteKeys []message.Key - inputSendResolvedTsHint bool - - expectEvents []*model.PolymorphicEvent - expectDeleteKeys []message.Key - expectOutputs []*model.PolymorphicEvent - }{ - // Empty buffer. - { - outputChCap: 1, - inputEvents: []*model.PolymorphicEvent{}, - inputDeleteKeys: []message.Key{}, - inputSendResolvedTsHint: true, - - expectEvents: []*model.PolymorphicEvent{}, - expectDeleteKeys: []message.Key{}, - expectOutputs: []*model.PolymorphicEvent{}, - }, - // Output one event, delete one event. - { - outputChCap: 2, - inputEvents: []*model.PolymorphicEvent{ - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 1}), - }, - inputDeleteKeys: []message.Key{}, - inputSendResolvedTsHint: true, - - expectEvents: []*model.PolymorphicEvent{}, - expectDeleteKeys: []message.Key{ - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 1}))), - }, - expectOutputs: []*model.PolymorphicEvent{ - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 1}), - // All inputEvent are sent, it also outputs a resolved ts event. - model.NewResolvedPolymorphicEvent(0, 1), - }, - }, - // Delete one event. - { - outputChCap: 2, - inputEvents: []*model.PolymorphicEvent{}, - inputDeleteKeys: []message.Key{ - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 1}))), - }, - inputSendResolvedTsHint: true, - - expectEvents: []*model.PolymorphicEvent{}, - expectDeleteKeys: []message.Key{ - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 1}))), - }, - expectOutputs: []*model.PolymorphicEvent{}, - }, - // Output one event, delete two event. - { - outputChCap: 2, - inputEvents: []*model.PolymorphicEvent{ - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 2}), - }, - inputDeleteKeys: []message.Key{ - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 1}))), - }, - inputSendResolvedTsHint: true, - - expectEvents: []*model.PolymorphicEvent{}, - expectDeleteKeys: []message.Key{ - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 1}))), - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 2}))), - }, - expectOutputs: []*model.PolymorphicEvent{ - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 2}), - // All inputEvent are sent, it also outputs a resolved ts event. - model.NewResolvedPolymorphicEvent(0, 2), - }, - }, - // Output two events, left one event. - { - outputChCap: 2, - inputEvents: []*model.PolymorphicEvent{ - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 3, RegionID: 1}), - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 3, RegionID: 2}), - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 3, RegionID: 3}), - }, - inputDeleteKeys: []message.Key{}, - inputSendResolvedTsHint: true, - - expectEvents: []*model.PolymorphicEvent{ - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 3, RegionID: 3}), - }, - expectDeleteKeys: []message.Key{ - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 3, RegionID: 1}))), - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 3, RegionID: 2}))), - }, - expectOutputs: []*model.PolymorphicEvent{ - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 3, RegionID: 1}), - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 3, RegionID: 2}), - // No resolved ts event because not all events are sent. - }, - }, - // Output zero event, left two events. - { - outputChCap: 0, - inputEvents: []*model.PolymorphicEvent{ - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 4, RegionID: 1}), - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 4, RegionID: 2}), - }, - inputDeleteKeys: []message.Key{}, - inputSendResolvedTsHint: true, - - expectEvents: []*model.PolymorphicEvent{ - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 4, RegionID: 1}), - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 4, RegionID: 2}), - }, - expectDeleteKeys: []message.Key{}, - expectOutputs: []*model.PolymorphicEvent{}, - }, - } - - for i, cs := range cases { - ls.outputCh = make(chan *model.PolymorphicEvent, cs.outputChCap) - buf.resolvedEvents = append([]*model.PolymorphicEvent{}, cs.inputEvents...) - buf.deleteKeys = append([]message.Key{}, cs.inputDeleteKeys...) - - ls.outputBufferedResolvedEvents(buf, cs.inputSendResolvedTsHint) - require.EqualValues(t, cs.expectDeleteKeys, buf.deleteKeys, "case #%d, %v", i, cs) - require.EqualValues(t, cs.expectEvents, buf.resolvedEvents, "case #%d, %v", i, cs) - - outputEvents := []*model.PolymorphicEvent{} - RECV: - for { - select { - case ev := <-ls.outputCh: - outputEvents = append(outputEvents, ev) - default: - break RECV - } - } - require.EqualValues(t, cs.expectOutputs, outputEvents, "case #%d, %v", i, cs) - } -} - -func newTestEvent(crts, startTs uint64, key int) *model.PolymorphicEvent { - return model.NewPolymorphicEvent(&model.RawKVEntry{ - OpType: model.OpTypePut, - Key: []byte{byte(key)}, - StartTs: startTs, - CRTs: crts, - }) -} - -func prepareTxnData( - t *testing.T, ls *Sorter, txnCount, txnSize int, -) db.DB { - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - db, err := db.OpenLevelDB(context.Background(), 1, t.TempDir(), cfg) - require.Nil(t, err) - wb := db.Batch(0) - for i := 1; i < txnCount+1; i++ { // txns. - for j := 0; j < txnSize; j++ { // events. - event := newTestEvent(uint64(i)+1, uint64(i), j) - key := encoding.EncodeKey(ls.uid, ls.tableID, event) - value, err := ls.serde.Marshal(event, []byte{}) - require.Nil(t, err) - t.Logf("key: %s, value: %s\n", message.Key(key), hex.EncodeToString(value)) - wb.Put(key, value) - } - } - require.Nil(t, wb.Commit()) - return db -} - -func receiveOutputEvents( - outputCh chan *model.PolymorphicEvent, -) []*model.PolymorphicEvent { - outputEvents := []*model.PolymorphicEvent{} -RECV: - for { - select { - case ev := <-outputCh: - outputEvents = append(outputEvents, ev) - default: - break RECV - } - } - return outputEvents -} - -func TestOutputIterEvents(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - capacity := 4 - ls, _ := newTestSorter(ctx, capacity) - - // Prepare data, 3 txns, 3 events for each. - // CRTs 2, StartTs 1, keys (0|1|2) - // CRTs 3, StartTs 2, keys (0|1|2) - // CRTs 4, StartTs 3, keys (0|1|2) - // CRTs 5, StartTs 4, keys (0|1|2) - // CRTs 6, StartTs 4, keys (0|1|2) - db := prepareTxnData(t, ls, 5, 3) - - cases := []struct { - outputChCap int - maxResolvedTs uint64 - hasReadNext bool - - expectEvents []*model.PolymorphicEvent - expectDeleteKeys []message.Key - expectOutputs []*model.PolymorphicEvent - expectExhaustedRTs uint64 - expectHasReadNext bool - }{ - // Empty resolved event. - { - outputChCap: 1, - maxResolvedTs: 0, - - expectEvents: []*model.PolymorphicEvent{}, - expectDeleteKeys: []message.Key{}, - expectOutputs: []*model.PolymorphicEvent{}, - expectExhaustedRTs: 0, - expectHasReadNext: true, - }, - // Nonblocking output three events and one resolved ts. - { - outputChCap: 4, - maxResolvedTs: 2, // CRTs 2 has 3 events. - - expectEvents: []*model.PolymorphicEvent{}, - expectDeleteKeys: []message.Key{ - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, newTestEvent(2, 1, 0))), - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, newTestEvent(2, 1, 1))), - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, newTestEvent(2, 1, 2))), - }, - expectOutputs: []*model.PolymorphicEvent{ - newTestEvent(2, 1, 0), - newTestEvent(2, 1, 1), - newTestEvent(2, 1, 2), - // No buffered resolved events, it outputs a resolved ts event. - model.NewResolvedPolymorphicEvent(0, 2), - }, - expectExhaustedRTs: 2, // Iter is exhausted and no buffered resolved events. - expectHasReadNext: true, - }, - // Blocking output two events of CRTs 3. - { - outputChCap: 2, - maxResolvedTs: 3, // CRTs 3 has 3 events. - - expectEvents: []*model.PolymorphicEvent{newTestEvent(3, 2, 2)}, - expectDeleteKeys: []message.Key{ - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, newTestEvent(3, 2, 0))), - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, newTestEvent(3, 2, 1))), - }, - expectOutputs: []*model.PolymorphicEvent{ - newTestEvent(3, 2, 0), - newTestEvent(3, 2, 1), - }, - // Events of CRTs 3 have been read and buffered. - expectExhaustedRTs: 0, - expectHasReadNext: true, - }, - // Output remaining event of CRTs 3. - { - outputChCap: 3, - maxResolvedTs: 3, // CRTs 3 has 1 events. - - expectEvents: []*model.PolymorphicEvent{}, - expectDeleteKeys: []message.Key{ - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, newTestEvent(3, 2, 2))), - }, - expectOutputs: []*model.PolymorphicEvent{ - newTestEvent(3, 2, 2), - model.NewResolvedPolymorphicEvent(0, 3), - }, - expectExhaustedRTs: 3, // Iter is exhausted and no buffered resolved events. - expectHasReadNext: true, - }, - // Resolved ts covers all resolved events, - // blocking output events of CRTs 4 (3 events) and 5 (1 event). - { - outputChCap: 5, - maxResolvedTs: 7, - - expectEvents: []*model.PolymorphicEvent{ - newTestEvent(5, 4, 1), - newTestEvent(5, 4, 2), - }, - expectDeleteKeys: []message.Key{ - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, newTestEvent(4, 3, 0))), - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, newTestEvent(4, 3, 1))), - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, newTestEvent(4, 3, 2))), - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, newTestEvent(5, 4, 0))), - }, - expectOutputs: []*model.PolymorphicEvent{ - newTestEvent(4, 3, 0), - newTestEvent(4, 3, 1), - newTestEvent(4, 3, 2), - model.NewResolvedPolymorphicEvent(0, 4), - newTestEvent(5, 4, 0), - }, - expectExhaustedRTs: 0, // Iter is not exhausted. - expectHasReadNext: false, // (5, 4, 1) is neither output nor buffered. - }, - // Resolved ts covers all resolved events, nonblocking output all events. - { - outputChCap: 7, - maxResolvedTs: 7, - - expectEvents: []*model.PolymorphicEvent{}, - expectDeleteKeys: []message.Key{ - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, newTestEvent(5, 4, 1))), - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, newTestEvent(5, 4, 2))), - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, newTestEvent(6, 5, 0))), - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, newTestEvent(6, 5, 1))), - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, newTestEvent(6, 5, 2))), - }, - expectOutputs: []*model.PolymorphicEvent{ - newTestEvent(5, 4, 1), - newTestEvent(5, 4, 2), - model.NewResolvedPolymorphicEvent(0, 5), - newTestEvent(6, 5, 0), - newTestEvent(6, 5, 1), - newTestEvent(6, 5, 2), - model.NewResolvedPolymorphicEvent(0, 7), - }, - expectExhaustedRTs: 7, // Iter is exhausted and no buffered resolved events. - expectHasReadNext: true, - }, - } - - for i, cs := range cases { - ls.outputCh = make(chan *model.PolymorphicEvent, cs.outputChCap) - buf := newOutputBuffer(capacity) - - iter := db.Iterator( - encoding.EncodeTsKey(ls.uid, ls.tableID, 0), - encoding.EncodeTsKey(ls.uid, ls.tableID, cs.maxResolvedTs+1)) - iter.First() - require.Nil(t, iter.Error(), "case #%d, %v", i, cs) - hasReadLastNext, exhaustedRTs, err := - ls.outputIterEvents(iter, cs.hasReadNext, buf, cs.maxResolvedTs) - require.Nil(t, err, "case #%d, %v", i, cs) - require.EqualValues(t, cs.expectExhaustedRTs, exhaustedRTs, "case #%d, %v", i, cs) - require.EqualValues(t, cs.expectDeleteKeys, buf.deleteKeys, "case #%d, %v", i, cs) - require.EqualValues(t, cs.expectEvents, buf.resolvedEvents, "case #%d, %v", i, cs) - require.EqualValues(t, cs.expectHasReadNext, hasReadLastNext, "case #%d, %v", i, cs) - outputEvents := receiveOutputEvents(ls.outputCh) - require.EqualValues(t, cs.expectOutputs, outputEvents, "case #%d, %v", i, cs) - - wb := db.Batch(0) - for _, key := range cs.expectDeleteKeys { - wb.Delete([]byte(key)) - } - require.Nil(t, wb.Commit()) - } - - require.Nil(t, db.Close()) -} - -func TestStateIterator(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ls, _ := newTestSorter(ctx, 1) - // Prepare data, 1 txn. - db := prepareTxnData(t, ls, 1, 1) - sema := semaphore.NewWeighted(1) - metricIterDuration := sorterIterReadDurationHistogram.MustCurryWith( - prometheus.Labels{"capture": t.Name(), "id": t.Name()}) - cfg := config.GetDefaultServerConfig().Clone().Debug.DB - mb := actor.NewMailbox(1, 1) - router := actor.NewRouter(t.Name()) - router.InsertMailbox4Test(mb.ID(), mb) - state := pollState{ - actorID: mb.ID(), - iterFirstSlowDuration: 100 * time.Second, - compact: NewCompactScheduler(router, cfg), - iterMaxAliveDuration: 100 * time.Millisecond, - metricIterFirst: metricIterDuration.WithLabelValues("first"), - metricIterRelease: metricIterDuration.WithLabelValues("release"), - } - - // First get returns a request. - req, ok := state.tryGetIterator(1, 1) - require.False(t, ok) - require.NotNil(t, req) - - // Still wait for iterator response. - req1, ok := state.tryGetIterator(1, 1) - require.False(t, ok) - require.Nil(t, req1) - - // Send iterator. - require.Nil(t, sema.Acquire(ctx, 1)) - req.IterCh <- &message.LimitedIterator{ - Iterator: db.Iterator([]byte{}, []byte{}), - Sema: sema, - } - // Get iterator successfully. - req2, ok := state.tryGetIterator(1, 1) - require.True(t, ok) - require.Nil(t, req2) - // Get iterator successfully again. - req2, ok = state.tryGetIterator(1, 1) - require.True(t, ok) - require.Nil(t, req2) - - // Release an invalid iterator. - require.False(t, state.iter.Valid()) - require.Nil(t, state.tryReleaseIterator()) - require.Nil(t, state.iter) - - // Release an outdated iterator. - require.Nil(t, sema.Acquire(ctx, 1)) - state.iter = &message.LimitedIterator{ - Iterator: db.Iterator([]byte{}, []byte{0xff}), - Sema: sema, - } - require.True(t, state.iter.First()) - state.iterAliveTime = time.Now() - time.Sleep(2 * state.iterMaxAliveDuration) - require.Nil(t, state.tryReleaseIterator()) - require.Nil(t, state.iter) - - // Release empty iterator. - require.Nil(t, state.tryReleaseIterator()) - - // Slow first must send a compaction task. - req3, ok := state.tryGetIterator(1, 1) - require.False(t, ok) - require.NotNil(t, req3) - require.Nil(t, sema.Acquire(ctx, 1)) - req3.IterCh <- &message.LimitedIterator{ - Iterator: db.Iterator([]byte{}, []byte{}), - Sema: sema, - } - // No compaction task yet. - _, ok = mb.Receive() - require.False(t, ok) - // Always slow. - state.iterFirstSlowDuration = time.Duration(0) - _, ok = state.tryGetIterator(1, 1) - require.True(t, ok) - require.NotNil(t, state.iter) - // Must recv a compaction task. - _, ok = mb.Receive() - require.True(t, ok) - // Release iterator. - time.Sleep(2 * state.iterMaxAliveDuration) - require.Nil(t, state.tryReleaseIterator()) - require.Nil(t, state.iter) - - require.Nil(t, db.Close()) -} - -func TestPoll(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - capacity := 4 - ls, mb := newTestSorter(ctx, capacity) - - // Prepare data, 3 txns, 3 events for each. - // CRTs 2, StartTs 1, keys (0|1|2) - // CRTs 3, StartTs 2, keys (0|1|2) - // CRTs 4, StartTs 3, keys (0|1|2) - // CRTs 5, StartTs 4, keys (0|1|2) - // CRTs 6, StartTs 4, keys (0|1|2) - db := prepareTxnData(t, ls, 5, 3) - sema := semaphore.NewWeighted(1) - - // We need to poll twice to read resolved events, so we need a slice of - // two cases. - cases := [][2]struct { - inputEvents []*model.PolymorphicEvent - inputIter func([2][]byte) *message.LimitedIterator - state *pollState - - expectEvents []*model.PolymorphicEvent - expectDeleteKeys []message.Key - expectOutputs []*model.PolymorphicEvent - expectMaxCommitTs uint64 - expectMaxResolvedTs uint64 - expectExhaustedRTs uint64 - }{ - {{ // The first poll - inputEvents: []*model.PolymorphicEvent{ - model.NewResolvedPolymorphicEvent(0, 1), - }, - state: &pollState{ - eventsBuf: make([]*model.PolymorphicEvent, 1), - outputBuf: newOutputBuffer(1), - }, - inputIter: func([2][]byte) *message.LimitedIterator { return nil }, - - expectEvents: []*model.PolymorphicEvent{}, - expectDeleteKeys: []message.Key{}, - // It is initialized to 1 in the test. - expectOutputs: []*model.PolymorphicEvent{model.NewResolvedPolymorphicEvent(0, 1)}, - expectMaxCommitTs: 0, - expectMaxResolvedTs: 1, - expectExhaustedRTs: 0, - }, { // The second poll - inputEvents: []*model.PolymorphicEvent{ - model.NewResolvedPolymorphicEvent(0, 1), - }, - state: nil, // state is inherited from the first poll. - inputIter: nil, // no need to make an iterator. - - expectEvents: []*model.PolymorphicEvent{}, - expectDeleteKeys: []message.Key{}, - // It is initialized to 1 in the test. - expectOutputs: []*model.PolymorphicEvent{model.NewResolvedPolymorphicEvent(0, 1)}, - expectMaxCommitTs: 0, - expectMaxResolvedTs: 1, - expectExhaustedRTs: 0, - }}, - // maxCommitTs and maxResolvedTs must advance according to inputs. - // And exhaustedResolvedTs must advance if there is no resolved event. - {{ // The first poll - inputEvents: []*model.PolymorphicEvent{ - newTestEvent(3, 2, 1), // crts 3, startts 2 - model.NewResolvedPolymorphicEvent(0, 2), - }, - state: &pollState{ - eventsBuf: make([]*model.PolymorphicEvent, 2), - outputBuf: newOutputBuffer(1), - }, - // An empty iterator. - inputIter: newEmptyIterator(ctx, t, db, sema), - - expectEvents: []*model.PolymorphicEvent{}, - expectDeleteKeys: []message.Key{}, - expectOutputs: []*model.PolymorphicEvent{}, - expectMaxCommitTs: 3, - expectMaxResolvedTs: 2, - expectExhaustedRTs: 0, - }, { // The second poll - inputEvents: []*model.PolymorphicEvent{ - model.NewResolvedPolymorphicEvent(0, 2), - }, - state: nil, // state is inherited from the first poll. - inputIter: nil, // no need to make an iterator. - - expectEvents: []*model.PolymorphicEvent{}, - expectDeleteKeys: []message.Key{}, - expectOutputs: []*model.PolymorphicEvent{ - model.NewResolvedPolymorphicEvent(0, 2), - }, - expectMaxCommitTs: 3, - expectMaxResolvedTs: 2, - // exhaustedResolvedTs must advance if there is no resolved event. - expectExhaustedRTs: 2, - }}, - // exhaustedResolvedTs must advance if all resolved events are outputted. - // Output: CRTs 2, StartTs 1, keys (0|1|2) - {{ // The first poll - inputEvents: []*model.PolymorphicEvent{ - newTestEvent(3, 2, 1), // crts 3, startts 2 - model.NewResolvedPolymorphicEvent(0, 2), - }, - state: &pollState{ - eventsBuf: make([]*model.PolymorphicEvent, 2), - outputBuf: newOutputBuffer(1), - }, - inputIter: newSnapshot(ctx, t, db, sema), - - expectEvents: []*model.PolymorphicEvent{}, - expectDeleteKeys: []message.Key{}, - expectOutputs: []*model.PolymorphicEvent{}, - expectMaxCommitTs: 3, - expectMaxResolvedTs: 2, - expectExhaustedRTs: 0, - }, { // The second poll - inputEvents: []*model.PolymorphicEvent{ - model.NewResolvedPolymorphicEvent(0, 2), - }, - state: nil, // state is inherited from the first poll. - inputIter: nil, // no need to make an iterator. - - expectEvents: []*model.PolymorphicEvent{}, - expectDeleteKeys: []message.Key{ - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, newTestEvent(2, 1, 0))), - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, newTestEvent(2, 1, 1))), - message.Key(encoding.EncodeKey(ls.uid, ls.tableID, newTestEvent(2, 1, 2))), - }, - expectOutputs: []*model.PolymorphicEvent{ - newTestEvent(2, 1, 0), - newTestEvent(2, 1, 1), - newTestEvent(2, 1, 2), - model.NewResolvedPolymorphicEvent(0, 2), - }, - expectMaxCommitTs: 3, - expectMaxResolvedTs: 2, - // exhaustedResolvedTs must advance if there is no resolved event. - expectExhaustedRTs: 2, - }}, - // maxResolvedTs must advance even if there is only resolved ts event. - {{ // The first poll - inputEvents: []*model.PolymorphicEvent{ - model.NewResolvedPolymorphicEvent(0, 3), - }, - state: &pollState{ - eventsBuf: make([]*model.PolymorphicEvent, 2), - outputBuf: newOutputBuffer(1), - maxCommitTs: 2, - exhaustedResolvedTs: 2, - }, - inputIter: func([2][]byte) *message.LimitedIterator { return nil }, - - expectEvents: []*model.PolymorphicEvent{}, - expectDeleteKeys: []message.Key{}, - expectOutputs: []*model.PolymorphicEvent{ - model.NewResolvedPolymorphicEvent(0, 3), - }, - expectMaxCommitTs: 2, - expectMaxResolvedTs: 3, - expectExhaustedRTs: 2, - }, { // The second poll - inputEvents: []*model.PolymorphicEvent{ - model.NewResolvedPolymorphicEvent(0, 3), - }, - state: nil, // state is inherited from the first poll. - inputIter: nil, // no need to make an iterator. - - expectEvents: []*model.PolymorphicEvent{}, - expectDeleteKeys: []message.Key{}, - expectOutputs: []*model.PolymorphicEvent{ - model.NewResolvedPolymorphicEvent(0, 3), - }, - expectMaxCommitTs: 2, - expectMaxResolvedTs: 3, - expectExhaustedRTs: 2, - }}, - // Batch output buffered resolved events - {{ // The first poll - inputEvents: []*model.PolymorphicEvent{}, - state: &pollState{ - eventsBuf: make([]*model.PolymorphicEvent, 2), - outputBuf: &outputBuffer{ - deleteKeys: make([]message.Key, 0, 2), - resolvedEvents: []*model.PolymorphicEvent{ - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 4}), - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 4}), - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 4}), - }, - advisedCapacity: 2, - }, - }, - inputIter: func([2][]byte) *message.LimitedIterator { return nil }, - - expectEvents: []*model.PolymorphicEvent{}, - expectDeleteKeys: []message.Key{}, - expectOutputs: []*model.PolymorphicEvent{ - model.NewResolvedPolymorphicEvent(0, 0), // A dummy events. - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 4}), - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 4}), - model.NewPolymorphicEvent(&model.RawKVEntry{CRTs: 4}), - model.NewResolvedPolymorphicEvent(0, 4), - }, - expectMaxCommitTs: 0, - expectMaxResolvedTs: 0, - expectExhaustedRTs: 0, - }, { // The second poll - inputEvents: []*model.PolymorphicEvent{ - model.NewResolvedPolymorphicEvent(0, 4), - }, - state: nil, // state is inherited from the first poll. - inputIter: nil, // no need to make an iterator. - - expectEvents: []*model.PolymorphicEvent{}, - expectDeleteKeys: []message.Key{}, - expectOutputs: []*model.PolymorphicEvent{ - model.NewResolvedPolymorphicEvent(0, 4), - }, - expectMaxCommitTs: 0, - expectMaxResolvedTs: 4, - expectExhaustedRTs: 0, - }}, - } - - metricIterDuration := sorterIterReadDurationHistogram.MustCurryWith( - prometheus.Labels{"capture": t.Name(), "id": t.Name()}) - for i, css := range cases { - state := css[0].state - state.iterFirstSlowDuration = 100 * time.Second - state.iterMaxAliveDuration = 100 * time.Second - state.metricIterFirst = metricIterDuration.WithLabelValues("first") - state.metricIterRelease = metricIterDuration.WithLabelValues("release") - for j, cs := range css { - for i := range cs.inputEvents { - ls.AddEntry(ctx, cs.inputEvents[i]) - } - t.Logf("test case #%d[%d], %v", i, j, cs) - require.Nil(t, ls.poll(ctx, state)) - require.EqualValues(t, cs.expectEvents, state.outputBuf.resolvedEvents, "case #%d[%d], %v", i, j, cs) - require.EqualValues(t, cs.expectDeleteKeys, state.outputBuf.deleteKeys, "case #%d[%d], %v", i, j, cs) - require.EqualValues(t, cs.expectMaxCommitTs, state.maxCommitTs, "case #%d[%d], %v", i, j, cs) - require.EqualValues(t, cs.expectMaxResolvedTs, state.maxResolvedTs, "case #%d[%d], %v", i, j, cs) - require.EqualValues(t, cs.expectExhaustedRTs, state.exhaustedResolvedTs, "case #%d[%d], %v", i, j, cs) - outputEvents := receiveOutputEvents(ls.outputCh) - require.EqualValues(t, cs.expectOutputs, outputEvents, "case #%d[%d], %v", i, j, cs) - - task, ok := mb.Receive() - if !ok { - // No task, so there must be nil inputIter. - require.Nil(t, cs.inputIter, "case #%d[%d], %v", i, j, cs) - continue - } - handleTask(task, cs.inputIter) - } - if state.iter != nil { - require.Nil(t, state.iter.Release()) - } - } - - require.Nil(t, db.Close()) -} - -func handleTask( - task actormsg.Message, iterFn func(rg [2][]byte) *message.LimitedIterator, -) { - if task.SorterTask.IterReq == nil || iterFn == nil { - return - } - iter := iterFn(task.SorterTask.IterReq.Range) - if iter != nil { - iter.ResolvedTs = task.SorterTask.IterReq.ResolvedTs - task.SorterTask.IterReq.IterCh <- iter - } - close(task.SorterTask.IterReq.IterCh) -} - -func newSnapshot( - ctx context.Context, t *testing.T, db db.DB, sema *semaphore.Weighted, -) func(rg [2][]byte) *message.LimitedIterator { - return func(rg [2][]byte) *message.LimitedIterator { - require.Nil(t, sema.Acquire(ctx, 1)) - return &message.LimitedIterator{ - Iterator: db.Iterator(rg[0], rg[1]), - Sema: sema, - } - } -} - -func newEmptyIterator( - ctx context.Context, t *testing.T, db db.DB, sema *semaphore.Weighted, -) func(rg [2][]byte) *message.LimitedIterator { - return func(rg [2][]byte) *message.LimitedIterator { - require.Nil(t, sema.Acquire(ctx, 1)) - return &message.LimitedIterator{ - Iterator: db.Iterator([]byte{}, []byte{}), - Sema: sema, - } - } -} - -func TestTryAddEntry(t *testing.T) { - t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - capacity := 1 - ls, _ := newTestSorter(ctx, capacity) - - resolvedTs1 := model.NewResolvedPolymorphicEvent(0, 1) - sent, err := ls.TryAddEntry(ctx, resolvedTs1) - require.True(t, sent) - require.Nil(t, err) - require.EqualValues(t, resolvedTs1, <-ls.inputCh) - - ls.inputCh = make(chan *model.PolymorphicEvent) - sent, err = ls.TryAddEntry(ctx, resolvedTs1) - require.False(t, sent) - require.Nil(t, err) -} diff --git a/cdc/cdc/sorter/memory/doc.go b/cdc/cdc/sorter/memory/doc.go deleted file mode 100644 index 13cbaf3f..00000000 --- a/cdc/cdc/sorter/memory/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package memory is an in-memory EventSorter implementation. -package memory diff --git a/cdc/cdc/sorter/memory/entry_sorter.go b/cdc/cdc/sorter/memory/entry_sorter.go deleted file mode 100644 index b8fb413f..00000000 --- a/cdc/cdc/sorter/memory/entry_sorter.go +++ /dev/null @@ -1,237 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package memory - -import ( - "context" - "sort" - "sync" - "sync/atomic" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/notify" - "github.com/tikv/migration/cdc/pkg/util" - "go.uber.org/zap" - "golang.org/x/sync/errgroup" -) - -// EntrySorter accepts out-of-order raw kv entries and output sorted entries -type EntrySorter struct { - unsorted []*model.PolymorphicEvent - lock sync.Mutex - resolvedTsGroup []uint64 - closed int32 - - outputCh chan *model.PolymorphicEvent - resolvedNotifier *notify.Notifier -} - -// NewEntrySorter creates a new EntrySorter -func NewEntrySorter() *EntrySorter { - return &EntrySorter{ - resolvedNotifier: new(notify.Notifier), - outputCh: make(chan *model.PolymorphicEvent, 128000), - } -} - -// Run runs EntrySorter -func (es *EntrySorter) Run(ctx context.Context) error { - captureAddr := util.CaptureAddrFromCtx(ctx) - changefeedID := util.ChangefeedIDFromCtx(ctx) - _, tableName := util.TableIDFromCtx(ctx) - metricEntrySorterResolvedChanSizeGuage := entrySorterResolvedChanSizeGauge.WithLabelValues(captureAddr, changefeedID, tableName) - metricEntrySorterOutputChanSizeGauge := entrySorterOutputChanSizeGauge.WithLabelValues(captureAddr, changefeedID, tableName) - metricEntryUnsortedSizeGauge := entrySorterUnsortedSizeGauge.WithLabelValues(captureAddr, changefeedID, tableName) - metricEntrySorterSortDuration := entrySorterSortDuration.WithLabelValues(captureAddr, changefeedID, tableName) - metricEntrySorterMergeDuration := entrySorterMergeDuration.WithLabelValues(captureAddr, changefeedID, tableName) - - output := func(ctx context.Context, entry *model.PolymorphicEvent) { - select { - case <-ctx.Done(): - return - case es.outputCh <- entry: - } - } - - errg, ctx := errgroup.WithContext(ctx) - receiver, err := es.resolvedNotifier.NewReceiver(1000 * time.Millisecond) - if err != nil { - return err - } - defer es.resolvedNotifier.Close() - errg.Go(func() error { - var sorted []*model.PolymorphicEvent - for { - select { - case <-ctx.Done(): - atomic.StoreInt32(&es.closed, 1) - close(es.outputCh) - return errors.Trace(ctx.Err()) - case <-time.After(defaultMetricInterval): - metricEntrySorterOutputChanSizeGauge.Set(float64(len(es.outputCh))) - es.lock.Lock() - metricEntrySorterResolvedChanSizeGuage.Set(float64(len(es.resolvedTsGroup))) - metricEntryUnsortedSizeGauge.Set(float64(len(es.unsorted))) - es.lock.Unlock() - case <-receiver.C: - es.lock.Lock() - if len(es.resolvedTsGroup) == 0 { - es.lock.Unlock() - continue - } - resolvedTsGroup := es.resolvedTsGroup - es.resolvedTsGroup = nil - toSort := es.unsorted - es.unsorted = nil - es.lock.Unlock() - - resEvents := make([]*model.PolymorphicEvent, len(resolvedTsGroup)) - for i, rts := range resolvedTsGroup { - // regionID = 0 means the event is produced by TiCDC - resEvents[i] = model.NewResolvedPolymorphicEvent(0, rts) - } - toSort = append(toSort, resEvents...) - startTime := time.Now() - sort.Slice(toSort, func(i, j int) bool { - return eventLess(toSort[i], toSort[j]) - }) - metricEntrySorterSortDuration.Observe(time.Since(startTime).Seconds()) - maxResolvedTs := resolvedTsGroup[len(resolvedTsGroup)-1] - - startTime = time.Now() - var merged []*model.PolymorphicEvent - mergeEvents(toSort, sorted, func(entry *model.PolymorphicEvent) { - if entry.CRTs <= maxResolvedTs { - output(ctx, entry) - } else { - merged = append(merged, entry) - } - }) - metricEntrySorterMergeDuration.Observe(time.Since(startTime).Seconds()) - sorted = merged - } - } - }) - return errg.Wait() -} - -// AddEntry adds an RawKVEntry to the EntryGroup -func (es *EntrySorter) AddEntry(_ context.Context, entry *model.PolymorphicEvent) { - if atomic.LoadInt32(&es.closed) != 0 { - return - } - es.lock.Lock() - defer es.lock.Unlock() - if entry.RawKV.OpType == model.OpTypeResolved { - es.resolvedTsGroup = append(es.resolvedTsGroup, entry.CRTs) - es.resolvedNotifier.Notify() - } else { - es.unsorted = append(es.unsorted, entry) - } -} - -func (es *EntrySorter) TryAddEntry(ctx context.Context, entry *model.PolymorphicEvent) (bool, error) { - if atomic.LoadInt32(&es.closed) != 0 { - return false, cerror.ErrSorterClosed.GenWithStackByArgs() - } - es.AddEntry(ctx, entry) - return true, nil -} - -// Output returns the sorted raw kv output channel -func (es *EntrySorter) Output() <-chan *model.PolymorphicEvent { - return es.outputCh -} - -func eventLess(i *model.PolymorphicEvent, j *model.PolymorphicEvent) bool { - if i.CRTs == j.CRTs { - if i.RawKV.OpType == model.OpTypeDelete { - return true - } - - if j.RawKV.OpType == model.OpTypeResolved { - return true - } - } - return i.CRTs < j.CRTs -} - -func mergeEvents(kvsA []*model.PolymorphicEvent, kvsB []*model.PolymorphicEvent, output func(*model.PolymorphicEvent)) { - var i, j int - for i < len(kvsA) && j < len(kvsB) { - if eventLess(kvsA[i], kvsB[j]) { - output(kvsA[i]) - i++ - } else { - output(kvsB[j]) - j++ - } - } - for ; i < len(kvsA); i++ { - output(kvsA[i]) - } - for ; j < len(kvsB); j++ { - output(kvsB[j]) - } -} - -// SortOutput receives a channel from a puller, then sort event and output to the channel returned. -func SortOutput(ctx context.Context, input <-chan *model.RawKVEntry) <-chan *model.RawKVEntry { - ctx, cancel := context.WithCancel(ctx) - sorter := NewEntrySorter() - outputCh := make(chan *model.RawKVEntry, 128) - output := func(rawKV *model.RawKVEntry) { - select { - case <-ctx.Done(): - if errors.Cause(ctx.Err()) != context.Canceled { - log.Error("sorter exited with error", zap.Error(ctx.Err())) - } - return - case outputCh <- rawKV: - } - } - go func() { - for { - select { - case <-ctx.Done(): - if errors.Cause(ctx.Err()) != context.Canceled { - log.Error("sorter exited with error", zap.Error(ctx.Err())) - } - return - case rawKV := <-input: - if rawKV == nil { - continue - } - sorter.AddEntry(ctx, model.NewPolymorphicEvent(rawKV)) - case sorted := <-sorter.Output(): - if sorted != nil { - output(sorted.RawKV) - } - } - } - }() - go func() { - if err := sorter.Run(ctx); err != nil { - if errors.Cause(ctx.Err()) != context.Canceled { - log.Error("sorter exited with error", zap.Error(ctx.Err())) - } - } - cancel() - }() - return outputCh -} diff --git a/cdc/cdc/sorter/memory/entry_sorter_test.go b/cdc/cdc/sorter/memory/entry_sorter_test.go deleted file mode 100644 index 680b4854..00000000 --- a/cdc/cdc/sorter/memory/entry_sorter_test.go +++ /dev/null @@ -1,518 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package memory - -import ( - "context" - "math/rand" - "sort" - "sync" - "sync/atomic" - "testing" - - "github.com/pingcap/check" - "github.com/pingcap/errors" - "github.com/tikv/migration/cdc/cdc/model" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/util/testleak" -) - -type mockEntrySorterSuite struct{} - -var _ = check.Suite(&mockEntrySorterSuite{}) - -func TestSuite(t *testing.T) { - check.TestingT(t) -} - -func (s *mockEntrySorterSuite) TestEntrySorter(c *check.C) { - defer testleak.AfterTest(c)() - testCases := []struct { - input []*model.RawKVEntry - resolvedTs uint64 - expect []*model.RawKVEntry - }{ - { - input: []*model.RawKVEntry{ - {CRTs: 1, OpType: model.OpTypePut}, - {CRTs: 2, OpType: model.OpTypePut}, - {CRTs: 4, OpType: model.OpTypeDelete}, - {CRTs: 2, OpType: model.OpTypeDelete}, - }, - resolvedTs: 0, - expect: []*model.RawKVEntry{ - {CRTs: 0, OpType: model.OpTypeResolved}, - }, - }, - { - input: []*model.RawKVEntry{ - {CRTs: 3, OpType: model.OpTypePut}, - {CRTs: 2, OpType: model.OpTypePut}, - {CRTs: 5, OpType: model.OpTypePut}, - }, - resolvedTs: 3, - expect: []*model.RawKVEntry{ - {CRTs: 1, OpType: model.OpTypePut}, - {CRTs: 2, OpType: model.OpTypeDelete}, - {CRTs: 2, OpType: model.OpTypePut}, - {CRTs: 2, OpType: model.OpTypePut}, - {CRTs: 3, OpType: model.OpTypePut}, - {CRTs: 3, OpType: model.OpTypeResolved}, - }, - }, - { - input: []*model.RawKVEntry{}, - resolvedTs: 3, - expect: []*model.RawKVEntry{{CRTs: 3, OpType: model.OpTypeResolved}}, - }, - { - input: []*model.RawKVEntry{ - {CRTs: 7, OpType: model.OpTypePut}, - }, - resolvedTs: 6, - expect: []*model.RawKVEntry{ - {CRTs: 4, OpType: model.OpTypeDelete}, - {CRTs: 5, OpType: model.OpTypePut}, - {CRTs: 6, OpType: model.OpTypeResolved}, - }, - }, - { - input: []*model.RawKVEntry{{CRTs: 7, OpType: model.OpTypeDelete}}, - resolvedTs: 6, - expect: []*model.RawKVEntry{ - {CRTs: 6, OpType: model.OpTypeResolved}, - }, - }, - { - input: []*model.RawKVEntry{{CRTs: 7, OpType: model.OpTypeDelete}}, - resolvedTs: 8, - expect: []*model.RawKVEntry{ - {CRTs: 7, OpType: model.OpTypeDelete}, - {CRTs: 7, OpType: model.OpTypeDelete}, - {CRTs: 7, OpType: model.OpTypePut}, - {CRTs: 8, OpType: model.OpTypeResolved}, - }, - }, - { - input: []*model.RawKVEntry{}, - resolvedTs: 15, - expect: []*model.RawKVEntry{ - {CRTs: 15, OpType: model.OpTypeResolved}, - }, - }, - } - es := NewEntrySorter() - ctx, cancel := context.WithCancel(context.Background()) - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - err := es.Run(ctx) - c.Assert(errors.Cause(err), check.Equals, context.Canceled) - }() - for _, tc := range testCases { - for _, entry := range tc.input { - es.AddEntry(ctx, model.NewPolymorphicEvent(entry)) - } - es.AddEntry(ctx, model.NewResolvedPolymorphicEvent(0, tc.resolvedTs)) - for i := 0; i < len(tc.expect); i++ { - e := <-es.Output() - c.Check(e.RawKV, check.DeepEquals, tc.expect[i]) - } - } - cancel() - wg.Wait() -} - -func (s *mockEntrySorterSuite) TestEntrySorterNonBlocking(c *check.C) { - defer testleak.AfterTest(c)() - testCases := []struct { - input []*model.RawKVEntry - resolvedTs uint64 - expect []*model.RawKVEntry - }{ - { - input: []*model.RawKVEntry{ - {CRTs: 1, OpType: model.OpTypePut}, - {CRTs: 2, OpType: model.OpTypePut}, - {CRTs: 4, OpType: model.OpTypeDelete}, - {CRTs: 2, OpType: model.OpTypeDelete}, - }, - resolvedTs: 0, - expect: []*model.RawKVEntry{ - {CRTs: 0, OpType: model.OpTypeResolved}, - }, - }, - { - input: []*model.RawKVEntry{ - {CRTs: 3, OpType: model.OpTypePut}, - {CRTs: 2, OpType: model.OpTypePut}, - {CRTs: 5, OpType: model.OpTypePut}, - }, - resolvedTs: 3, - expect: []*model.RawKVEntry{ - {CRTs: 1, OpType: model.OpTypePut}, - {CRTs: 2, OpType: model.OpTypeDelete}, - {CRTs: 2, OpType: model.OpTypePut}, - {CRTs: 2, OpType: model.OpTypePut}, - {CRTs: 3, OpType: model.OpTypePut}, - {CRTs: 3, OpType: model.OpTypeResolved}, - }, - }, - { - input: []*model.RawKVEntry{}, - resolvedTs: 3, - expect: []*model.RawKVEntry{{CRTs: 3, OpType: model.OpTypeResolved}}, - }, - { - input: []*model.RawKVEntry{ - {CRTs: 7, OpType: model.OpTypePut}, - }, - resolvedTs: 6, - expect: []*model.RawKVEntry{ - {CRTs: 4, OpType: model.OpTypeDelete}, - {CRTs: 5, OpType: model.OpTypePut}, - {CRTs: 6, OpType: model.OpTypeResolved}, - }, - }, - { - input: []*model.RawKVEntry{{CRTs: 7, OpType: model.OpTypeDelete}}, - resolvedTs: 6, - expect: []*model.RawKVEntry{ - {CRTs: 6, OpType: model.OpTypeResolved}, - }, - }, - { - input: []*model.RawKVEntry{{CRTs: 7, OpType: model.OpTypeDelete}}, - resolvedTs: 8, - expect: []*model.RawKVEntry{ - {CRTs: 7, OpType: model.OpTypeDelete}, - {CRTs: 7, OpType: model.OpTypeDelete}, - {CRTs: 7, OpType: model.OpTypePut}, - {CRTs: 8, OpType: model.OpTypeResolved}, - }, - }, - { - input: []*model.RawKVEntry{}, - resolvedTs: 15, - expect: []*model.RawKVEntry{ - {CRTs: 15, OpType: model.OpTypeResolved}, - }, - }, - } - es := NewEntrySorter() - ctx, cancel := context.WithCancel(context.Background()) - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - err := es.Run(ctx) - c.Assert(errors.Cause(err), check.Equals, context.Canceled) - }() - for _, tc := range testCases { - for _, entry := range tc.input { - added, err := es.TryAddEntry(ctx, model.NewPolymorphicEvent(entry)) - c.Assert(added, check.IsTrue) - c.Assert(err, check.IsNil) - } - added, err := es.TryAddEntry(ctx, model.NewResolvedPolymorphicEvent(0, tc.resolvedTs)) - c.Assert(added, check.IsTrue) - c.Assert(err, check.IsNil) - for i := 0; i < len(tc.expect); i++ { - e := <-es.Output() - c.Check(e.RawKV, check.DeepEquals, tc.expect[i]) - } - } - cancel() - wg.Wait() -} - -func (s *mockEntrySorterSuite) TestEntrySorterRandomly(c *check.C) { - defer testleak.AfterTest(c)() - es := NewEntrySorter() - ctx, cancel := context.WithCancel(context.Background()) - - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - err := es.Run(ctx) - c.Assert(errors.Cause(err), check.Equals, context.Canceled) - }() - - maxTs := uint64(1000000) - wg.Add(1) - go func() { - defer wg.Done() - for resolvedTs := uint64(1); resolvedTs <= maxTs; resolvedTs += 400 { - var opType model.OpType - if rand.Intn(2) == 0 { - opType = model.OpTypePut - } else { - opType = model.OpTypeDelete - } - for i := 0; i < 1000; i++ { - entry := &model.RawKVEntry{ - CRTs: uint64(int64(resolvedTs) + rand.Int63n(int64(maxTs-resolvedTs))), - OpType: opType, - } - es.AddEntry(ctx, model.NewPolymorphicEvent(entry)) - } - es.AddEntry(ctx, model.NewResolvedPolymorphicEvent(0, resolvedTs)) - } - es.AddEntry(ctx, model.NewResolvedPolymorphicEvent(0, maxTs)) - }() - var lastTs uint64 - var resolvedTs uint64 - lastOpType := model.OpTypePut - for entry := range es.Output() { - c.Assert(entry.CRTs, check.GreaterEqual, lastTs) - c.Assert(entry.CRTs, check.Greater, resolvedTs) - if lastOpType == model.OpTypePut && entry.RawKV.OpType == model.OpTypeDelete { - c.Assert(entry.CRTs, check.Greater, lastTs) - } - lastTs = entry.CRTs - lastOpType = entry.RawKV.OpType - if entry.RawKV.OpType == model.OpTypeResolved { - resolvedTs = entry.CRTs - } - if resolvedTs == maxTs { - break - } - } - cancel() - wg.Wait() -} - -func (s *mockEntrySorterSuite) TestEventLess(c *check.C) { - defer testleak.AfterTest(c)() - testCases := []struct { - i *model.PolymorphicEvent - j *model.PolymorphicEvent - expected bool - }{ - { - &model.PolymorphicEvent{ - CRTs: 1, - }, - &model.PolymorphicEvent{ - CRTs: 2, - }, - true, - }, - { - &model.PolymorphicEvent{ - CRTs: 2, - RawKV: &model.RawKVEntry{ - OpType: model.OpTypeDelete, - }, - }, - &model.PolymorphicEvent{ - CRTs: 2, - RawKV: &model.RawKVEntry{ - OpType: model.OpTypeDelete, - }, - }, - true, - }, - { - &model.PolymorphicEvent{ - CRTs: 2, - RawKV: &model.RawKVEntry{ - OpType: model.OpTypeResolved, - }, - }, - &model.PolymorphicEvent{ - CRTs: 2, - RawKV: &model.RawKVEntry{ - OpType: model.OpTypeResolved, - }, - }, - true, - }, - { - &model.PolymorphicEvent{ - CRTs: 2, - RawKV: &model.RawKVEntry{ - OpType: model.OpTypeResolved, - }, - }, - &model.PolymorphicEvent{ - CRTs: 2, - RawKV: &model.RawKVEntry{ - OpType: model.OpTypeDelete, - }, - }, - false, - }, - { - &model.PolymorphicEvent{ - CRTs: 3, - RawKV: &model.RawKVEntry{ - OpType: model.OpTypeDelete, - }, - }, - &model.PolymorphicEvent{ - CRTs: 2, - RawKV: &model.RawKVEntry{ - OpType: model.OpTypeResolved, - }, - }, - false, - }, - } - - for _, tc := range testCases { - c.Assert(eventLess(tc.i, tc.j), check.Equals, tc.expected) - } -} - -func (s *mockEntrySorterSuite) TestMergeEvents(c *check.C) { - defer testleak.AfterTest(c)() - events1 := []*model.PolymorphicEvent{ - { - CRTs: 1, - RawKV: &model.RawKVEntry{ - OpType: model.OpTypeDelete, - }, - }, - { - CRTs: 2, - RawKV: &model.RawKVEntry{ - OpType: model.OpTypePut, - }, - }, - { - CRTs: 3, - RawKV: &model.RawKVEntry{ - OpType: model.OpTypePut, - }, - }, - { - CRTs: 4, - RawKV: &model.RawKVEntry{ - OpType: model.OpTypePut, - }, - }, - { - CRTs: 5, - RawKV: &model.RawKVEntry{ - OpType: model.OpTypeDelete, - }, - }, - } - events2 := []*model.PolymorphicEvent{ - { - CRTs: 3, - RawKV: &model.RawKVEntry{ - OpType: model.OpTypeResolved, - }, - }, - { - CRTs: 4, - RawKV: &model.RawKVEntry{ - OpType: model.OpTypePut, - }, - }, - { - CRTs: 4, - RawKV: &model.RawKVEntry{ - OpType: model.OpTypeResolved, - }, - }, - { - CRTs: 7, - RawKV: &model.RawKVEntry{ - OpType: model.OpTypePut, - }, - }, - { - CRTs: 9, - RawKV: &model.RawKVEntry{ - OpType: model.OpTypeDelete, - }, - }, - } - - var outputResults []*model.PolymorphicEvent - output := func(event *model.PolymorphicEvent) { - outputResults = append(outputResults, event) - } - - expectedResults := append(events1, events2...) - sort.Slice(expectedResults, func(i, j int) bool { - return eventLess(expectedResults[i], expectedResults[j]) - }) - - mergeEvents(events1, events2, output) - c.Assert(outputResults, check.DeepEquals, expectedResults) -} - -func (s *mockEntrySorterSuite) TestEntrySorterClosed(c *check.C) { - defer testleak.AfterTest(c)() - es := NewEntrySorter() - atomic.StoreInt32(&es.closed, 1) - added, err := es.TryAddEntry(context.TODO(), model.NewResolvedPolymorphicEvent(0, 1)) - c.Assert(added, check.IsFalse) - c.Assert(cerror.ErrSorterClosed.Equal(err), check.IsTrue) -} - -func BenchmarkSorter(b *testing.B) { - es := NewEntrySorter() - ctx, cancel := context.WithCancel(context.Background()) - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - err := es.Run(ctx) - if errors.Cause(err) != context.Canceled { - panic(errors.Annotate(err, "unexpected error")) - } - }() - - maxTs := uint64(10000000) - b.ResetTimer() - wg.Add(1) - go func() { - defer wg.Done() - for resolvedTs := uint64(1); resolvedTs <= maxTs; resolvedTs += 400 { - var opType model.OpType - if rand.Intn(2) == 0 { - opType = model.OpTypePut - } else { - opType = model.OpTypeDelete - } - for i := 0; i < 100000; i++ { - entry := &model.RawKVEntry{ - CRTs: uint64(int64(resolvedTs) + rand.Int63n(1000)), - OpType: opType, - } - es.AddEntry(ctx, model.NewPolymorphicEvent(entry)) - } - es.AddEntry(ctx, model.NewResolvedPolymorphicEvent(0, resolvedTs)) - } - es.AddEntry(ctx, model.NewResolvedPolymorphicEvent(0, maxTs)) - }() - var resolvedTs uint64 - for entry := range es.Output() { - if entry.RawKV.OpType == model.OpTypeResolved { - resolvedTs = entry.CRTs - } - if resolvedTs == maxTs { - break - } - } - cancel() - wg.Wait() -} diff --git a/cdc/cdc/sorter/memory/metrics.go b/cdc/cdc/sorter/memory/metrics.go deleted file mode 100644 index 5995f4f9..00000000 --- a/cdc/cdc/sorter/memory/metrics.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package memory - -import ( - "time" - - "github.com/prometheus/client_golang/prometheus" -) - -const ( - defaultMetricInterval = time.Second * 15 -) - -var ( - entrySorterResolvedChanSizeGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "ticdc", - Subsystem: "puller", - Name: "entry_sorter_resolved_chan_size", - Help: "Puller entry sorter resolved channel size", - }, []string{"capture", "changefeed", "table"}) - entrySorterOutputChanSizeGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "ticdc", - Subsystem: "puller", - Name: "entry_sorter_output_chan_size", - Help: "Puller entry sorter output channel size", - }, []string{"capture", "changefeed", "table"}) - entrySorterUnsortedSizeGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "ticdc", - Subsystem: "puller", - Name: "entry_sorter_unsorted_size", - Help: "Puller entry sorter unsorted items size", - }, []string{"capture", "changefeed", "table"}) - entrySorterSortDuration = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: "ticdc", - Subsystem: "puller", - Name: "entry_sorter_sort", - Help: "Bucketed histogram of processing time (s) of sort in entry sorter.", - Buckets: prometheus.ExponentialBuckets(0.000001, 10, 10), - }, []string{"capture", "changefeed", "table"}) - entrySorterMergeDuration = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: "ticdc", - Subsystem: "puller", - Name: "entry_sorter_merge", - Help: "Bucketed histogram of processing time (s) of merge in entry sorter.", - Buckets: prometheus.ExponentialBuckets(0.000001, 10, 10), - }, []string{"capture", "changefeed", "table"}) -) - -// InitMetrics registers all metrics in this file -func InitMetrics(registry *prometheus.Registry) { - registry.MustRegister(entrySorterResolvedChanSizeGauge) - registry.MustRegister(entrySorterOutputChanSizeGauge) - registry.MustRegister(entrySorterUnsortedSizeGauge) - registry.MustRegister(entrySorterSortDuration) - registry.MustRegister(entrySorterMergeDuration) -} diff --git a/cdc/cdc/sorter/metrics.go b/cdc/cdc/sorter/metrics.go deleted file mode 100644 index 89594f64..00000000 --- a/cdc/cdc/sorter/metrics.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package sorter - -import ( - "github.com/prometheus/client_golang/prometheus" -) - -var ( - // EventCount is the metric that counts events output by the sorter. - EventCount = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "ticdc", - Subsystem: "sorter", - Name: "event_count", - Help: "The number of events output by the sorter", - }, []string{"capture", "changefeed", "type"}) - - // ResolvedTsGauge is the metric that records sorter resolved ts. - ResolvedTsGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "ticdc", - Subsystem: "sorter", - Name: "resolved_ts_gauge", - Help: "the resolved ts of the sorter", - }, []string{"capture", "changefeed"}) - - // InMemoryDataSizeGauge is the metric that records sorter memory usage. - InMemoryDataSizeGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "ticdc", - Subsystem: "sorter", - Name: "in_memory_data_size_gauge", - Help: "The amount of pending data stored in-memory by the sorter", - }, []string{"capture", "id"}) - - // OnDiskDataSizeGauge is the metric that records sorter disk usage. - OnDiskDataSizeGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "ticdc", - Subsystem: "sorter", - Name: "on_disk_data_size_gauge", - Help: "The amount of pending data stored on-disk by the sorter", - }, []string{"capture", "id"}) - - // OpenFileCountGauge is the metric that records sorter open files. - OpenFileCountGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "ticdc", - Subsystem: "sorter", - Name: "open_file_count_gauge", - Help: "The number of open file descriptors held by the sorter", - }, []string{"capture", "id"}) -) - -// InitMetrics registers all metrics in this file -func InitMetrics(registry *prometheus.Registry) { - registry.MustRegister(EventCount) - registry.MustRegister(ResolvedTsGauge) - registry.MustRegister(InMemoryDataSizeGauge) - registry.MustRegister(OnDiskDataSizeGauge) - registry.MustRegister(OpenFileCountGauge) -} diff --git a/cdc/cdc/sorter/sorter.go b/cdc/cdc/sorter/sorter.go deleted file mode 100644 index 7169ae3a..00000000 --- a/cdc/cdc/sorter/sorter.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package sorter - -import ( - "context" - - "github.com/tikv/migration/cdc/cdc/model" -) - -// EventSorter accepts unsorted PolymorphicEvents, sort them in background and returns -// sorted PolymorphicEvents in Output channel -type EventSorter interface { - Run(ctx context.Context) error - // TODO add constraints to entries, e.g., order and duplication guarantees. - AddEntry(ctx context.Context, entry *model.PolymorphicEvent) - // TryAddEntry tries to add and entry to the sorter. - // Returns false if the entry can not be added; otherwise it returns true - // Returns error if the sorter is closed or context is done - TryAddEntry(ctx context.Context, entry *model.PolymorphicEvent) (bool, error) - // Output sorted events, orderd by commit ts. - // It may output a dummy event, a zero resolved ts event, to detect whether - // output is available. - Output() <-chan *model.PolymorphicEvent -} diff --git a/cdc/cdc/sorter/unified/backend.go b/cdc/cdc/sorter/unified/backend.go deleted file mode 100644 index a7f610f7..00000000 --- a/cdc/cdc/sorter/unified/backend.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package unified - -import "github.com/tikv/migration/cdc/cdc/model" - -type backEnd interface { - reader() (backEndReader, error) - writer() (backEndWriter, error) - free() error -} - -type backEndReader interface { - readNext() (*model.PolymorphicEvent, error) - resetAndClose() error -} - -type backEndWriter interface { - writeNext(event *model.PolymorphicEvent) error - writtenCount() int - dataSize() uint64 - flushAndClose() error -} diff --git a/cdc/cdc/sorter/unified/backend_pool.go b/cdc/cdc/sorter/unified/backend_pool.go deleted file mode 100644 index af7f32ca..00000000 --- a/cdc/cdc/sorter/unified/backend_pool.go +++ /dev/null @@ -1,404 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package unified - -import ( - "context" - "fmt" - "os" - "path/filepath" - "reflect" - "sync" - "sync/atomic" - "time" - "unsafe" - - "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/pingcap/log" - "github.com/pingcap/tidb/util/memory" - "github.com/tikv/migration/cdc/cdc/sorter" - sorterencoding "github.com/tikv/migration/cdc/cdc/sorter/encoding" - "github.com/tikv/migration/cdc/pkg/config" - cerrors "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/fsutil" - "github.com/tikv/migration/cdc/pkg/util" - "go.uber.org/zap" -) - -const ( - backgroundJobInterval = time.Second * 15 - sortDirLockFileName = "ticdc_lock" - sortDirDataFileMagicPrefix = "sort" -) - -var ( - pool *backEndPool // this is the singleton instance of backEndPool - poolMu sync.Mutex // this mutex is for delayed initialization of `pool` only -) - -type backEndPool struct { - memoryUseEstimate int64 - onDiskDataSize int64 - fileNameCounter uint64 - memPressure int32 - cache [256]unsafe.Pointer - dir string - filePrefix string - - // to prevent `dir` from being accidentally used by another TiCDC server process. - fileLock *fsutil.FileLock - - // cancelCh needs to be unbuffered to prevent races - cancelCh chan struct{} - // cancelRWLock protects cache against races when the backEnd is exiting - cancelRWLock sync.RWMutex - isTerminating bool -} - -func newBackEndPool(dir string, captureAddr string) (*backEndPool, error) { - ret := &backEndPool{ - memoryUseEstimate: 0, - fileNameCounter: 0, - dir: dir, - cancelCh: make(chan struct{}), - filePrefix: fmt.Sprintf("%s/%s-%d-", dir, sortDirDataFileMagicPrefix, os.Getpid()), - } - - err := ret.lockSortDir() - if err != nil { - log.Warn("failed to lock file prefix", - zap.String("prefix", ret.filePrefix), - zap.Error(err)) - return nil, errors.Trace(err) - } - - err = ret.cleanUpStaleFiles() - if err != nil { - log.Warn("Unified Sorter: failed to clean up stale temporary files. Report a bug if you believe this is unexpected", zap.Error(err)) - return nil, errors.Trace(err) - } - - go func() { - ticker := time.NewTicker(backgroundJobInterval) - defer ticker.Stop() - - id := "0" // A placeholder for ID label in metrics. - metricSorterInMemoryDataSizeGauge := sorter.InMemoryDataSizeGauge.WithLabelValues(captureAddr, id) - metricSorterOnDiskDataSizeGauge := sorter.OnDiskDataSizeGauge.WithLabelValues(captureAddr, id) - metricSorterOpenFileCountGauge := sorter.OpenFileCountGauge.WithLabelValues(captureAddr, id) - - // TODO: The underlaying implementation only recognizes cgroups set by - // containers, we need to support cgroups set by systemd or manually. - // See https://github.com/pingcap/tidb/issues/22132 - totalMemory, err := memory.MemTotal() - if err != nil { - log.Panic("read memory stat failed", zap.Error(err)) - } - for { - select { - case <-ret.cancelCh: - log.Info("Unified Sorter backEnd is being cancelled") - return - case <-ticker.C: - } - - metricSorterInMemoryDataSizeGauge.Set(float64(atomic.LoadInt64(&ret.memoryUseEstimate))) - metricSorterOnDiskDataSizeGauge.Set(float64(atomic.LoadInt64(&ret.onDiskDataSize))) - metricSorterOpenFileCountGauge.Set(float64(atomic.LoadInt64(&openFDCount))) - - // update memPressure - usedMemory, err := memory.MemUsed() - if err != nil || totalMemory == 0 { - failpoint.Inject("sorterDebug", func() { - log.Panic("unified sorter: getting system memory usage failed", zap.Error(err)) - }) - - log.Warn("unified sorter: getting system memory usage failed", zap.Error(err)) - // Reports a 100% memory pressure, so that the backEndPool will allocate fileBackEnds. - // We default to fileBackEnds because they are unlikely to cause OOMs. If IO errors are - // encountered, we can fail gracefully. - atomic.StoreInt32(&ret.memPressure, 100) - } else { - memPressure := usedMemory * 100 / totalMemory - atomic.StoreInt32(&ret.memPressure, int32(memPressure)) - } - - // garbage collect temporary files in batches - freedCount := 0 - for i := range ret.cache { - ptr := &ret.cache[i] - innerPtr := atomic.SwapPointer(ptr, nil) - if innerPtr == nil { - continue - } - backEnd := (*fileBackEnd)(innerPtr) - err := backEnd.free() - if err != nil { - log.Warn("Cannot remove temporary file for sorting", zap.String("file", backEnd.fileName), zap.Error(err)) - } else { - log.Debug("Temporary file removed", zap.String("file", backEnd.fileName)) - freedCount += 1 - } - if freedCount >= 16 { - freedCount = 0 - break - } - } - } - }() - - return ret, nil -} - -func (p *backEndPool) alloc(ctx context.Context) (backEnd, error) { - sorterConfig := config.GetGlobalServerConfig().Sorter - if p.sorterMemoryUsage() < int64(sorterConfig.MaxMemoryConsumption) && - p.memoryPressure() < int32(sorterConfig.MaxMemoryPressure) { - - ret := newMemoryBackEnd() - return ret, nil - } - - p.cancelRWLock.RLock() - defer p.cancelRWLock.RUnlock() - - if p.isTerminating { - return nil, cerrors.ErrUnifiedSorterBackendTerminating.GenWithStackByArgs() - } - - for i := range p.cache { - ptr := &p.cache[i] - ret := atomic.SwapPointer(ptr, nil) - if ret != nil { - return (*fileBackEnd)(ret), nil - } - } - - fname := fmt.Sprintf("%s%d.tmp", p.filePrefix, atomic.AddUint64(&p.fileNameCounter, 1)) - tableID, tableName := util.TableIDFromCtx(ctx) - log.Debug("Unified Sorter: trying to create file backEnd", - zap.String("filename", fname), - zap.Int64("table-id", tableID), - zap.String("table-name", tableName)) - - if err := checkDataDirSatisfied(); err != nil { - return nil, errors.Trace(err) - } - - ret, err := newFileBackEnd(fname, &sorterencoding.MsgPackGenSerde{}) - if err != nil { - return nil, errors.Trace(err) - } - - return ret, nil -} - -func (p *backEndPool) dealloc(backEnd backEnd) error { - switch b := backEnd.(type) { - case *memoryBackEnd: - err := b.free() - if err != nil { - log.Warn("error freeing memory backend", zap.Error(err)) - } - // Let GC do its job - return nil - case *fileBackEnd: - failpoint.Inject("sorterDebug", func() { - if atomic.LoadInt32(&b.borrowed) != 0 { - log.Warn("Deallocating a fileBackEnd in use", zap.String("filename", b.fileName)) - failpoint.Return(nil) - } - }) - - b.cleanStats() - - p.cancelRWLock.RLock() - defer p.cancelRWLock.RUnlock() - - if p.isTerminating { - return cerrors.ErrUnifiedSorterBackendTerminating.GenWithStackByArgs() - } - - for i := range p.cache { - ptr := &p.cache[i] - if atomic.CompareAndSwapPointer(ptr, nil, unsafe.Pointer(b)) { - return nil - } - } - // Cache is full. - err := b.free() - if err != nil { - return errors.Trace(err) - } - - return nil - default: - log.Panic("backEndPool: unexpected backEnd type to be deallocated", zap.Reflect("type", reflect.TypeOf(backEnd))) - } - return nil -} - -func (p *backEndPool) terminate() { - defer func() { - if p.fileLock == nil { - return - } - err := p.unlockSortDir() - if err != nil { - log.Warn("failed to unlock file prefix", zap.String("prefix", p.filePrefix)) - } - }() - - p.cancelCh <- struct{}{} - defer close(p.cancelCh) - // the background goroutine can be considered terminated here - - log.Debug("Unified Sorter terminating...") - p.cancelRWLock.Lock() - defer p.cancelRWLock.Unlock() - p.isTerminating = true - - log.Debug("Unified Sorter cleaning up before exiting") - // any new allocs and deallocs will not succeed from this point - // accessing p.cache without atomics is safe from now - - for i := range p.cache { - ptr := &p.cache[i] - backend := (*fileBackEnd)(*ptr) - if backend == nil { - continue - } - _ = backend.free() - } - - if p.filePrefix == "" { - // This should not happen. But to prevent accidents in production, we add this anyway. - log.Panic("Empty filePrefix, please report a bug") - } - - files, err := filepath.Glob(p.filePrefix + "*") - if err != nil { - log.Warn("Unified Sorter clean-up failed", zap.Error(err)) - } - for _, file := range files { - log.Debug("Unified Sorter backEnd removing file", zap.String("file", file)) - err = os.RemoveAll(file) - if err != nil { - log.Warn("Unified Sorter clean-up failed: failed to remove", zap.String("file-name", file), zap.Error(err)) - } - } - - log.Debug("Unified Sorter backEnd terminated") -} - -func (p *backEndPool) sorterMemoryUsage() int64 { - failpoint.Inject("memoryUsageInjectPoint", func(val failpoint.Value) { - failpoint.Return(int64(val.(int))) - }) - return atomic.LoadInt64(&p.memoryUseEstimate) -} - -func (p *backEndPool) memoryPressure() int32 { - failpoint.Inject("memoryPressureInjectPoint", func(val failpoint.Value) { - failpoint.Return(int32(val.(int))) - }) - return atomic.LoadInt32(&p.memPressure) -} - -func (p *backEndPool) lockSortDir() error { - lockFileName := fmt.Sprintf("%s/%s", p.dir, sortDirLockFileName) - fileLock, err := fsutil.NewFileLock(lockFileName) - if err != nil { - return cerrors.ErrSortDirLockError.Wrap(err).GenWithStackByCause() - } - - err = fileLock.Lock() - if err != nil { - if cerrors.ErrConflictingFileLocks.Equal(err) { - log.Warn("TiCDC failed to lock sorter temporary file directory. "+ - "Make sure that another instance of TiCDC, or any other program, is not using the directory. "+ - "If you believe you should not see this error, try deleting the lock file and resume the changefeed. "+ - "Report a bug or contact support if the problem persists.", - zap.String("lock-file", lockFileName)) - return errors.Trace(err) - } - return cerrors.ErrSortDirLockError.Wrap(err).GenWithStackByCause() - } - - p.fileLock = fileLock - return nil -} - -func (p *backEndPool) unlockSortDir() error { - err := p.fileLock.Unlock() - if err != nil { - return cerrors.ErrSortDirLockError.Wrap(err).FastGenWithCause() - } - return nil -} - -func (p *backEndPool) cleanUpStaleFiles() error { - if p.dir == "" { - // guard against programmer error. Must be careful when we are deleting user files. - log.Panic("unexpected sort-dir", zap.String("sort-dir", p.dir)) - } - - files, err := filepath.Glob(filepath.Join(p.dir, fmt.Sprintf("%s-*", sortDirDataFileMagicPrefix))) - if err != nil { - return errors.Trace(err) - } - - for _, toRemoveFilePath := range files { - log.Debug("Removing stale sorter temporary file", zap.String("file", toRemoveFilePath)) - err := os.Remove(toRemoveFilePath) - if err != nil { - // In production, we do not want an error here to interfere with normal operation, - // because in most situations, failure to remove files only indicates non-fatal misconfigurations - // such as permission problems, rather than fatal errors. - // If the directory is truly unusable, other errors would be raised when we try to write to it. - log.Warn("failed to remove file", - zap.String("file", toRemoveFilePath), - zap.Error(err)) - // For fail-fast in integration tests - failpoint.Inject("sorterDebug", func() { - log.Panic("panicking", zap.Error(err)) - }) - } - } - - return nil -} - -// checkDataDirSatisfied check if the data-dir meet the requirement during server running -// the caller should guarantee that dir exist -func checkDataDirSatisfied() error { - const dataDirAvailLowThreshold = 10 // percentage - - conf := config.GetGlobalServerConfig() - diskInfo, err := fsutil.GetDiskInfo(conf.DataDir) - if err != nil { - return cerrors.WrapError(cerrors.ErrCheckDataDirSatisfied, err) - } - if diskInfo.AvailPercentage < dataDirAvailLowThreshold { - failpoint.Inject("InjectCheckDataDirSatisfied", func() { - log.Info("inject check data dir satisfied error") - failpoint.Return(nil) - }) - return cerrors.WrapError(cerrors.ErrCheckDataDirSatisfied, errors.Errorf("disk is almost full, TiCDC require that the disk mount data-dir "+ - "have 10%% available space, and the total amount has at least 500GB is preferred. disk info: %+v", diskInfo)) - } - - return nil -} diff --git a/cdc/cdc/sorter/unified/backend_pool_test.go b/cdc/cdc/sorter/unified/backend_pool_test.go deleted file mode 100644 index 22240c98..00000000 --- a/cdc/cdc/sorter/unified/backend_pool_test.go +++ /dev/null @@ -1,366 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package unified - -import ( - "context" - "fmt" - "os" - "path/filepath" - "strconv" - "time" - - "github.com/pingcap/check" - "github.com/pingcap/failpoint" - "github.com/pingcap/tidb/util/memory" - "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/fsutil" - "github.com/tikv/migration/cdc/pkg/util/testleak" -) - -type backendPoolSuite struct{} - -var _ = check.SerialSuites(&backendPoolSuite{}) - -func (s *backendPoolSuite) TestBasicFunction(c *check.C) { - defer testleak.AfterTest(c)() - - dataDir := c.MkDir() - err := os.MkdirAll(dataDir, 0o755) - c.Assert(err, check.IsNil) - - sortDir := filepath.Join(dataDir, config.DefaultSortDir) - err = os.MkdirAll(sortDir, 0o755) - c.Assert(err, check.IsNil) - - conf := config.GetDefaultServerConfig() - conf.DataDir = dataDir - conf.Sorter.SortDir = sortDir - conf.Sorter.MaxMemoryPressure = 90 // 90% - conf.Sorter.MaxMemoryConsumption = 16 * 1024 * 1024 * 1024 // 16G - config.StoreGlobalServerConfig(conf) - - err = failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/memoryPressureInjectPoint", "return(100)") - c.Assert(err, check.IsNil) - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*20) - defer cancel() - - backEndPool, err := newBackEndPool(sortDir, "") - c.Assert(err, check.IsNil) - c.Assert(backEndPool, check.NotNil) - defer backEndPool.terminate() - - backEnd, err := backEndPool.alloc(ctx) - c.Assert(err, check.IsNil) - c.Assert(backEnd, check.FitsTypeOf, &fileBackEnd{}) - fileName := backEnd.(*fileBackEnd).fileName - c.Assert(fileName, check.Not(check.Equals), "") - - err = failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/memoryPressureInjectPoint", "return(0)") - c.Assert(err, check.IsNil) - err = failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/memoryUsageInjectPoint", "return(34359738368)") - c.Assert(err, check.IsNil) - - backEnd1, err := backEndPool.alloc(ctx) - c.Assert(err, check.IsNil) - c.Assert(backEnd1, check.FitsTypeOf, &fileBackEnd{}) - fileName1 := backEnd1.(*fileBackEnd).fileName - c.Assert(fileName1, check.Not(check.Equals), "") - c.Assert(fileName1, check.Not(check.Equals), fileName) - - err = failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/memoryPressureInjectPoint", "return(0)") - c.Assert(err, check.IsNil) - err = failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/memoryUsageInjectPoint", "return(0)") - c.Assert(err, check.IsNil) - - backEnd2, err := backEndPool.alloc(ctx) - c.Assert(err, check.IsNil) - c.Assert(backEnd2, check.FitsTypeOf, &memoryBackEnd{}) - - err = backEndPool.dealloc(backEnd) - c.Assert(err, check.IsNil) - - err = backEndPool.dealloc(backEnd1) - c.Assert(err, check.IsNil) - - err = backEndPool.dealloc(backEnd2) - c.Assert(err, check.IsNil) - - time.Sleep(backgroundJobInterval * 3 / 2) - - _, err = os.Stat(fileName) - c.Assert(os.IsNotExist(err), check.IsTrue) - - _, err = os.Stat(fileName1) - c.Assert(os.IsNotExist(err), check.IsTrue) -} - -// TestDirectoryBadPermission verifies that no permission to ls the directory does not prevent using it -// as a temporary file directory. -func (s *backendPoolSuite) TestDirectoryBadPermission(c *check.C) { - defer testleak.AfterTest(c)() - - dataDir := c.MkDir() - sortDir := filepath.Join(dataDir, config.DefaultSortDir) - err := os.MkdirAll(sortDir, 0o755) - c.Assert(err, check.IsNil) - - err = os.Chmod(sortDir, 0o311) // no permission to `ls` - c.Assert(err, check.IsNil) - - conf := config.GetGlobalServerConfig() - conf.DataDir = dataDir - conf.Sorter.SortDir = sortDir - conf.Sorter.MaxMemoryPressure = 0 // force using files - - backEndPool, err := newBackEndPool(sortDir, "") - c.Assert(err, check.IsNil) - c.Assert(backEndPool, check.NotNil) - defer backEndPool.terminate() - - backEnd, err := backEndPool.alloc(context.Background()) - c.Assert(err, check.IsNil) - defer backEnd.free() //nolint:errcheck - - fileName := backEnd.(*fileBackEnd).fileName - _, err = os.Stat(fileName) - c.Assert(err, check.IsNil) // assert that the file exists - - err = backEndPool.dealloc(backEnd) - c.Assert(err, check.IsNil) -} - -// TestCleanUpSelf verifies that the backendPool correctly cleans up files used by itself on exit. -func (s *backendPoolSuite) TestCleanUpSelf(c *check.C) { - defer testleak.AfterTest(c)() - - dataDir := c.MkDir() - err := os.Chmod(dataDir, 0o755) - c.Assert(err, check.IsNil) - - sorterDir := filepath.Join(dataDir, config.DefaultSortDir) - err = os.MkdirAll(sorterDir, 0o755) - c.Assert(err, check.IsNil) - - conf := config.GetDefaultServerConfig() - conf.DataDir = dataDir - conf.Sorter.SortDir = sorterDir - conf.Sorter.MaxMemoryPressure = 90 // 90% - conf.Sorter.MaxMemoryConsumption = 16 * 1024 * 1024 * 1024 // 16G - config.StoreGlobalServerConfig(conf) - - err = failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/memoryPressureInjectPoint", "return(100)") - c.Assert(err, check.IsNil) - defer failpoint.Disable("github.com/tikv/migration/cdc/cdc/sorter/unified/memoryPressureInjectPoint") //nolint:errcheck - - backEndPool, err := newBackEndPool(sorterDir, "") - c.Assert(err, check.IsNil) - c.Assert(backEndPool, check.NotNil) - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*20) - defer cancel() - - var fileNames []string - for i := 0; i < 20; i++ { - backEnd, err := backEndPool.alloc(ctx) - c.Assert(err, check.IsNil) - c.Assert(backEnd, check.FitsTypeOf, &fileBackEnd{}) - - fileName := backEnd.(*fileBackEnd).fileName - _, err = os.Stat(fileName) - c.Assert(err, check.IsNil) - - fileNames = append(fileNames, fileName) - } - - prefix := backEndPool.filePrefix - c.Assert(prefix, check.Not(check.Equals), "") - - for j := 100; j < 120; j++ { - fileName := prefix + strconv.Itoa(j) + ".tmp" - f, err := os.Create(fileName) - c.Assert(err, check.IsNil) - err = f.Close() - c.Assert(err, check.IsNil) - - fileNames = append(fileNames, fileName) - } - - backEndPool.terminate() - - for _, fileName := range fileNames { - _, err = os.Stat(fileName) - c.Assert(os.IsNotExist(err), check.IsTrue) - } -} - -type mockOtherProcess struct { - dir string - prefix string - flock *fsutil.FileLock - files []string -} - -func newMockOtherProcess(c *check.C, dir string, prefix string) *mockOtherProcess { - prefixLockPath := fmt.Sprintf("%s/%s", dir, sortDirLockFileName) - flock, err := fsutil.NewFileLock(prefixLockPath) - c.Assert(err, check.IsNil) - - err = flock.Lock() - c.Assert(err, check.IsNil) - - return &mockOtherProcess{ - dir: dir, - prefix: prefix, - flock: flock, - } -} - -func (p *mockOtherProcess) writeMockFiles(c *check.C, num int) { - for i := 0; i < num; i++ { - fileName := fmt.Sprintf("%s%d", p.prefix, i) - f, err := os.Create(fileName) - c.Assert(err, check.IsNil) - _ = f.Close() - p.files = append(p.files, fileName) - } -} - -func (p *mockOtherProcess) changeLockPermission(c *check.C, mode os.FileMode) { - prefixLockPath := fmt.Sprintf("%s/%s", p.dir, sortDirLockFileName) - err := os.Chmod(prefixLockPath, mode) - c.Assert(err, check.IsNil) -} - -func (p *mockOtherProcess) unlock(c *check.C) { - err := p.flock.Unlock() - c.Assert(err, check.IsNil) -} - -func (p *mockOtherProcess) assertFilesExist(c *check.C) { - for _, file := range p.files { - _, err := os.Stat(file) - c.Assert(err, check.IsNil) - } -} - -func (p *mockOtherProcess) assertFilesNotExist(c *check.C) { - for _, file := range p.files { - _, err := os.Stat(file) - c.Assert(os.IsNotExist(err), check.IsTrue) - } -} - -// TestCleanUpStaleBasic verifies that the backendPool correctly cleans up stale temporary files -// left by other CDC processes that have exited abnormally. -func (s *backendPoolSuite) TestCleanUpStaleBasic(c *check.C) { - defer testleak.AfterTest(c)() - - dir := c.MkDir() - prefix := dir + "/sort-1-" - - mockP := newMockOtherProcess(c, dir, prefix) - mockP.writeMockFiles(c, 100) - mockP.unlock(c) - mockP.assertFilesExist(c) - - backEndPool, err := newBackEndPool(dir, "") - c.Assert(err, check.IsNil) - c.Assert(backEndPool, check.NotNil) - defer backEndPool.terminate() - - mockP.assertFilesNotExist(c) -} - -// TestFileLockConflict tests that if two backEndPools were to use the same sort-dir, -// and error would be returned by one of them. -func (s *backendPoolSuite) TestFileLockConflict(c *check.C) { - defer testleak.AfterTest(c)() - dir := c.MkDir() - - backEndPool1, err := newBackEndPool(dir, "") - c.Assert(err, check.IsNil) - c.Assert(backEndPool1, check.NotNil) - defer backEndPool1.terminate() - - backEndPool2, err := newBackEndPool(dir, "") - c.Assert(err, check.ErrorMatches, ".*file lock conflict.*") - c.Assert(backEndPool2, check.IsNil) -} - -// TestCleanUpStaleBasic verifies that the backendPool correctly cleans up stale temporary files -// left by other CDC processes that have exited abnormally. -func (s *backendPoolSuite) TestCleanUpStaleLockNoPermission(c *check.C) { - defer testleak.AfterTest(c)() - - dir := c.MkDir() - prefix := dir + "/sort-1-" - - mockP := newMockOtherProcess(c, dir, prefix) - mockP.writeMockFiles(c, 100) - // set a bad permission - mockP.changeLockPermission(c, 0o000) - - backEndPool, err := newBackEndPool(dir, "") - c.Assert(err, check.ErrorMatches, ".*permission denied.*") - c.Assert(backEndPool, check.IsNil) - - mockP.assertFilesExist(c) -} - -// TestGetMemoryPressureFailure verifies that the backendPool can handle gracefully failures that happen when -// getting the current system memory pressure. Such a failure is usually caused by a lack of file descriptor quota -// set by the operating system. -func (s *backendPoolSuite) TestGetMemoryPressureFailure(c *check.C) { - defer testleak.AfterTest(c)() - - origin := memory.MemTotal - defer func() { - memory.MemTotal = origin - }() - memory.MemTotal = func() (uint64, error) { return 0, nil } - - dir := c.MkDir() - backEndPool, err := newBackEndPool(dir, "") - c.Assert(err, check.IsNil) - c.Assert(backEndPool, check.NotNil) - defer backEndPool.terminate() - - after := time.After(time.Second * 20) - tick := time.Tick(time.Millisecond * 100) - for { - select { - case <-after: - c.Fatal("TestGetMemoryPressureFailure timed out") - case <-tick: - if backEndPool.memoryPressure() == 100 { - return - } - } - } -} - -func (s *backendPoolSuite) TestCheckDataDirSatisfied(c *check.C) { - defer testleak.AfterTest(c)() - dir := c.MkDir() - conf := config.GetGlobalServerConfig() - conf.DataDir = dir - config.StoreGlobalServerConfig(conf) - - c.Assert(failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/InjectCheckDataDirSatisfied", ""), check.IsNil) - err := checkDataDirSatisfied() - c.Assert(err, check.IsNil) - c.Assert(failpoint.Disable("github.com/tikv/migration/cdc/cdc/sorter/unified/InjectCheckDataDirSatisfied"), check.IsNil) -} diff --git a/cdc/cdc/sorter/unified/file_backend.go b/cdc/cdc/sorter/unified/file_backend.go deleted file mode 100644 index 3e947e80..00000000 --- a/cdc/cdc/sorter/unified/file_backend.go +++ /dev/null @@ -1,449 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package unified - -import ( - "bufio" - "encoding/binary" - "io" - "os" - "sync/atomic" - - "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/sorter/encoding" - cerrors "github.com/tikv/migration/cdc/pkg/errors" - "go.uber.org/zap" -) - -const ( - fileBufferSize = 4 * 1024 // 4KB - fileMagic = 0x12345678 - numFileEntriesOffset = 4 - blockMagic = 0xbeefbeef -) - -var openFDCount int64 - -type fileBackEnd struct { - fileName string - serde encoding.SerializerDeserializer - borrowed int32 - size int64 -} - -func newFileBackEnd(fileName string, serde encoding.SerializerDeserializer) (*fileBackEnd, error) { - f, err := os.Create(fileName) - if err != nil { - return nil, errors.Trace(wrapIOError(err)) - } - - err = f.Close() - if err != nil { - return nil, errors.Trace(wrapIOError(err)) - } - - log.Debug("new FileSorterBackEnd created", zap.String("filename", fileName)) - return &fileBackEnd{ - fileName: fileName, - serde: serde, - borrowed: 0, - }, nil -} - -func (f *fileBackEnd) reader() (backEndReader, error) { - fd, err := os.OpenFile(f.fileName, os.O_RDWR, 0o600) - if err != nil { - return nil, errors.Trace(wrapIOError(err)) - } - - atomic.AddInt64(&openFDCount, 1) - - var totalSize int64 - failpoint.Inject("sorterDebug", func() { - info, err := fd.Stat() - if err != nil { - failpoint.Return(nil, errors.Trace(wrapIOError(err))) - } - totalSize = info.Size() - }) - - failpoint.Inject("sorterDebug", func() { - if atomic.SwapInt32(&f.borrowed, 1) != 0 { - log.Panic("fileBackEnd: already borrowed", zap.String("fileName", f.fileName)) - } - }) - - ret := &fileBackEndReader{ - backEnd: f, - f: fd, - reader: bufio.NewReaderSize(fd, fileBufferSize), - totalSize: totalSize, - } - - err = ret.readHeader() - if err != nil { - return nil, errors.Trace(wrapIOError(err)) - } - - return ret, nil -} - -func (f *fileBackEnd) writer() (backEndWriter, error) { - fd, err := os.OpenFile(f.fileName, os.O_TRUNC|os.O_RDWR, 0o600) - if err != nil { - return nil, errors.Trace(wrapIOError(err)) - } - - atomic.AddInt64(&openFDCount, 1) - - failpoint.Inject("sorterDebug", func() { - if atomic.SwapInt32(&f.borrowed, 1) != 0 { - log.Panic("fileBackEnd: already borrowed", zap.String("fileName", f.fileName)) - } - }) - - ret := &fileBackEndWriter{ - backEnd: f, - f: fd, - writer: bufio.NewWriterSize(fd, fileBufferSize), - } - - err = ret.writeFileHeader() - if err != nil { - return nil, errors.Trace(wrapIOError(err)) - } - - return ret, nil -} - -func (f *fileBackEnd) free() error { - failpoint.Inject("sorterDebug", func() { - if atomic.LoadInt32(&f.borrowed) != 0 { - log.Panic("fileBackEnd: trying to free borrowed file", zap.String("fileName", f.fileName)) - } - }) - - log.Debug("Removing file", zap.String("file", f.fileName)) - - f.cleanStats() - - err := os.Remove(f.fileName) - if err != nil { - failpoint.Inject("sorterDebug", func() { - failpoint.Return(errors.Trace(wrapIOError(err))) - }) - // ignore this error in production to provide some resilience - log.Warn("fileBackEnd: failed to remove file", zap.Error(wrapIOError(err))) - } - - return nil -} - -func (f *fileBackEnd) cleanStats() { - if pool != nil { - atomic.AddInt64(&pool.onDiskDataSize, -f.size) - } - f.size = 0 -} - -type fileBackEndReader struct { - backEnd *fileBackEnd - f *os.File - reader *bufio.Reader - isEOF bool - - // to prevent truncation-like corruption - totalEvents uint64 - readEvents uint64 - - // debug only fields - readBytes int64 - totalSize int64 -} - -func (r *fileBackEndReader) readHeader() error { - failpoint.Inject("sorterDebug", func() { - pos, err := r.f.Seek(0, 1 /* relative to the current position */) - if err != nil { - failpoint.Return(errors.Trace(err)) - } - // verify that we are reading from the beginning of the file - if pos != 0 { - log.Panic("unexpected file descriptor cursor position", zap.Int64("pos", pos)) - } - }) - - var m uint32 - err := binary.Read(r.reader, binary.LittleEndian, &m) - if err != nil { - return errors.Trace(err) - } - if m != fileMagic { - log.Panic("fileSorterBackEnd: wrong fileMagic. Damaged file or bug?", zap.Uint32("actual", m)) - } - - err = binary.Read(r.reader, binary.LittleEndian, &r.totalEvents) - if err != nil { - return errors.Trace(err) - } - - return nil -} - -func (r *fileBackEndReader) readNext() (*model.PolymorphicEvent, error) { - if r.isEOF { - // guaranteed EOF idempotency - return nil, nil - } - - var m uint32 - err := binary.Read(r.reader, binary.LittleEndian, &m) - if err != nil { - if err == io.EOF { - r.isEOF = true - // verifies that the file has not been truncated unexpectedly. - if r.totalEvents != r.readEvents { - log.Panic("unexpected EOF", - zap.String("file", r.backEnd.fileName), - zap.Uint64("expected-num-events", r.totalEvents), - zap.Uint64("actual-num-events", r.readEvents)) - } - return nil, nil - } - return nil, errors.Trace(wrapIOError(err)) - } - - if m != blockMagic { - log.Panic("fileSorterBackEnd: wrong blockMagic. Damaged file or bug?", zap.Uint32("actual", m)) - } - - var size uint32 - err = binary.Read(r.reader, binary.LittleEndian, &size) - if err != nil { - return nil, errors.Trace(wrapIOError(err)) - } - - // Note, do not hold the buffer in reader to avoid hogging memory. - rawBytesBuf := make([]byte, size) - - // short reads are possible with bufio, hence the need for io.ReadFull - n, err := io.ReadFull(r.reader, rawBytesBuf) - if err != nil { - return nil, errors.Trace(wrapIOError(err)) - } - - if n != int(size) { - return nil, errors.Errorf("fileSorterBackEnd: expected %d bytes, actually read %d bytes", size, n) - } - - event := new(model.PolymorphicEvent) - _, err = r.backEnd.serde.Unmarshal(event, rawBytesBuf) - if err != nil { - return nil, errors.Trace(err) - } - - r.readEvents++ - - failpoint.Inject("sorterDebug", func() { - r.readBytes += int64(4 + 4 + int(size)) - if r.readBytes > r.totalSize { - log.Panic("fileSorterBackEnd: read more bytes than expected, check concurrent use of file", - zap.String("fileName", r.backEnd.fileName)) - } - }) - - return event, nil -} - -func (r *fileBackEndReader) resetAndClose() error { - defer func() { - // fail-fast for double-close - r.f = nil - - r.backEnd.cleanStats() - - failpoint.Inject("sorterDebug", func() { - atomic.StoreInt32(&r.backEnd.borrowed, 0) - }) - }() - - if r.f == nil { - failpoint.Inject("sorterDebug", func() { - log.Panic("Double closing of file", zap.String("filename", r.backEnd.fileName)) - }) - log.Warn("Double closing of file", zap.String("filename", r.backEnd.fileName)) - return nil - } - - err := r.f.Truncate(0) - if err != nil { - failpoint.Inject("sorterDebug", func() { - info, err1 := r.f.Stat() - if err1 != nil { - failpoint.Return(errors.Trace(wrapIOError(err))) - } - - log.Info("file debug info", zap.String("filename", info.Name()), - zap.Int64("size", info.Size())) - - failpoint.Return(nil) - }) - log.Warn("fileBackEndReader: could not truncate file", zap.Error(err)) - } - - err = r.f.Close() - if err != nil { - failpoint.Inject("sorterDebug", func() { - failpoint.Return(errors.Trace(err)) - }) - log.Warn("fileBackEndReader: could not close file", zap.Error(err)) - return nil - } - - atomic.AddInt64(&openFDCount, -1) - - return nil -} - -type fileBackEndWriter struct { - backEnd *fileBackEnd - f *os.File - writer *bufio.Writer - - bytesWritten int64 - eventsWritten int64 -} - -func (w *fileBackEndWriter) writeFileHeader() error { - err := binary.Write(w.writer, binary.LittleEndian, uint32(fileMagic)) - if err != nil { - return errors.Trace(err) - } - - // reserves the space for writing the total number of entries in this file - err = binary.Write(w.writer, binary.LittleEndian, uint64(0)) - if err != nil { - return errors.Trace(err) - } - - return nil -} - -func (w *fileBackEndWriter) writeNext(event *model.PolymorphicEvent) error { - var err error - // Note, do not hold the buffer in writer to avoid hogging memory. - var rawBytesBuf []byte - rawBytesBuf, err = w.backEnd.serde.Marshal(event, rawBytesBuf) - if err != nil { - return errors.Trace(wrapIOError(err)) - } - - size := len(rawBytesBuf) - if size == 0 { - log.Panic("fileSorterBackEnd: serialized to empty byte array. Bug?") - } - - err = binary.Write(w.writer, binary.LittleEndian, uint32(blockMagic)) - if err != nil { - return errors.Trace(wrapIOError(err)) - } - - err = binary.Write(w.writer, binary.LittleEndian, uint32(size)) - if err != nil { - return errors.Trace(wrapIOError(err)) - } - - // short writes are possible with bufio - offset := 0 - for offset < size { - n, err := w.writer.Write(rawBytesBuf[offset:]) - if err != nil { - return errors.Trace(wrapIOError(err)) - } - offset += n - } - if offset != size { - return errors.Errorf("fileSorterBackEnd: expected to write %d bytes, actually wrote %d bytes", size, offset) - } - - w.eventsWritten++ - w.bytesWritten += int64(size) - return nil -} - -func (w *fileBackEndWriter) writtenCount() int { - return int(w.bytesWritten) -} - -func (w *fileBackEndWriter) dataSize() uint64 { - return uint64(w.eventsWritten) -} - -func (w *fileBackEndWriter) flushAndClose() error { - defer func() { - // fail-fast for double-close - w.f = nil - }() - - err := w.writer.Flush() - if err != nil { - return errors.Trace(wrapIOError(err)) - } - - _, err = w.f.Seek(numFileEntriesOffset, 0 /* relative to the beginning of the file */) - if err != nil { - return errors.Trace(wrapIOError(err)) - } - - // write the total number of entries in the file to the header - err = binary.Write(w.f, binary.LittleEndian, uint64(w.eventsWritten)) - if err != nil { - return errors.Trace(wrapIOError(err)) - } - - err = w.f.Close() - if err != nil { - failpoint.Inject("sorterDebug", func() { - failpoint.Return(errors.Trace(wrapIOError(err))) - }) - log.Warn("fileBackEndReader: could not close file", zap.Error(err)) - return nil - } - - atomic.AddInt64(&openFDCount, -1) - w.backEnd.size = w.bytesWritten - atomic.AddInt64(&pool.onDiskDataSize, w.bytesWritten) - - failpoint.Inject("sorterDebug", func() { - atomic.StoreInt32(&w.backEnd.borrowed, 0) - }) - - return nil -} - -// wrapIOError should be called when the error is to be returned to an caller outside this file and -// if the error could be caused by a filesystem-related error. -func wrapIOError(err error) error { - cause := errors.Cause(err) - switch cause.(type) { - case *os.PathError: - // We don't generate stack in this helper function to avoid confusion. - return cerrors.ErrUnifiedSorterIOError.FastGenByArgs(err.Error()) - default: - return err - } -} diff --git a/cdc/cdc/sorter/unified/file_backend_test.go b/cdc/cdc/sorter/unified/file_backend_test.go deleted file mode 100644 index 1153baf9..00000000 --- a/cdc/cdc/sorter/unified/file_backend_test.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package unified - -import ( - "io" - "os" - - "github.com/pingcap/check" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/sorter/encoding" - cerrors "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/util/testleak" -) - -type fileBackendSuite struct{} - -var _ = check.SerialSuites(&fileBackendSuite{}) - -func (s *fileBackendSuite) TestWrapIOError(c *check.C) { - defer testleak.AfterTest(c)() - - fullFile, err := os.OpenFile("/dev/full", os.O_RDWR, 0) - c.Assert(err, check.IsNil) - defer fullFile.Close() //nolint:errcheck - _, err = fullFile.WriteString("test") - wrapped := wrapIOError(err) - // tests that the error message gives the user some informative description - c.Assert(wrapped, check.ErrorMatches, ".*review the settings.*no space.*") - - eof := wrapIOError(io.EOF) - // tests that the function does not change io.EOF - c.Assert(eof, check.Equals, io.EOF) -} - -func (s *fileBackendSuite) TestNoSpace(c *check.C) { - defer testleak.AfterTest(c)() - - fb := &fileBackEnd{ - fileName: "/dev/full", - serde: &encoding.MsgPackGenSerde{}, - } - w, err := fb.writer() - c.Assert(err, check.IsNil) - - err = w.writeNext(model.NewPolymorphicEvent(generateMockRawKV(0))) - if err == nil { - // Due to write buffering, `writeNext` might not return an error when the filesystem is full. - err = w.flushAndClose() - } - - c.Assert(err, check.ErrorMatches, ".*review the settings.*no space.*") - c.Assert(cerrors.ErrUnifiedSorterIOError.Equal(err), check.IsTrue) -} diff --git a/cdc/cdc/sorter/unified/heap.go b/cdc/cdc/sorter/unified/heap.go deleted file mode 100644 index a9ca1545..00000000 --- a/cdc/cdc/sorter/unified/heap.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package unified - -import "github.com/tikv/migration/cdc/cdc/model" - -type sortItem struct { - entry *model.PolymorphicEvent - data interface{} -} - -type sortHeap []*sortItem - -func (h sortHeap) Len() int { return len(h) } -func (h sortHeap) Less(i, j int) bool { - if h[i].entry.CRTs == h[j].entry.CRTs { - if h[j].entry.RawKV.OpType == model.OpTypeResolved && h[i].entry.RawKV.OpType != model.OpTypeResolved { - return true - } - if h[i].entry.RawKV.OpType == model.OpTypeDelete && h[j].entry.RawKV.OpType != model.OpTypeDelete { - return true - } - } - return h[i].entry.CRTs < h[j].entry.CRTs -} -func (h sortHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } -func (h *sortHeap) Push(x interface{}) { - *h = append(*h, x.(*sortItem)) -} - -func (h *sortHeap) Pop() interface{} { - old := *h - n := len(old) - x := old[n-1] - old[n-1] = nil - *h = old[0 : n-1] - return x -} diff --git a/cdc/cdc/sorter/unified/heap_sorter.go b/cdc/cdc/sorter/unified/heap_sorter.go deleted file mode 100644 index 390b7a4f..00000000 --- a/cdc/cdc/sorter/unified/heap_sorter.go +++ /dev/null @@ -1,393 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package unified - -import ( - "container/heap" - "context" - "sync" - "sync/atomic" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/config" - cerrors "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/util" - "github.com/tikv/migration/cdc/pkg/workerpool" - "go.uber.org/zap" -) - -const ( - flushRateLimitPerSecond = 10 - sortHeapCapacity = 32 - sortHeapInputChSize = 1024 -) - -type flushTask struct { - taskID int - heapSorterID int - reader backEndReader - tsLowerBound uint64 - maxResolvedTs uint64 - finished chan error - dealloc func() error - dataSize int64 - lastTs uint64 // for debugging TODO remove - canceller *asyncCanceller - - isEmpty bool // read only field - - deallocLock sync.RWMutex - isDeallocated bool // do not access directly - backend backEnd // do not access directly -} - -func (t *flushTask) markDeallocated() { - t.deallocLock.Lock() - defer t.deallocLock.Unlock() - - t.backend = nil - t.isDeallocated = true -} - -func (t *flushTask) GetBackEnd() backEnd { - t.deallocLock.RLock() - defer t.deallocLock.RUnlock() - - return t.backend -} - -type heapSorter struct { - id int - taskCounter int - inputCh chan *model.PolymorphicEvent - outputCh chan *flushTask - heap sortHeap - canceller *asyncCanceller - - poolHandle workerpool.EventHandle - internalState *heapSorterInternalState -} - -func newHeapSorter(id int, out chan *flushTask) *heapSorter { - return &heapSorter{ - id: id, - inputCh: make(chan *model.PolymorphicEvent, sortHeapInputChSize), - outputCh: out, - heap: make(sortHeap, 0, sortHeapCapacity), - canceller: new(asyncCanceller), - } -} - -// flush should only be called in the same goroutine where the heap is being written to. -func (h *heapSorter) flush(ctx context.Context, maxResolvedTs uint64) error { - captureAddr := util.CaptureAddrFromCtx(ctx) - changefeedID := util.ChangefeedIDFromCtx(ctx) - - var ( - backEnd backEnd - lowerBound uint64 - ) - - if h.heap.Len() > 0 { - lowerBound = h.heap[0].entry.CRTs - } else { - return nil - } - - sorterFlushCountHistogram.WithLabelValues(captureAddr, changefeedID).Observe(float64(h.heap.Len())) - - // We check if the heap contains only one entry and that entry is a ResolvedEvent. - // As an optimization, when the condition is true, we clear the heap and send an empty flush. - // Sending an empty flush saves CPU and potentially IO. - // Since when a table is mostly idle or near-idle, most flushes would contain one ResolvedEvent alone, - // this optimization will greatly improve performance when (1) total number of table is large, - // and (2) most tables do not have many events. - if h.heap.Len() == 1 && h.heap[0].entry.RawKV.OpType == model.OpTypeResolved { - h.heap.Pop() - } - - isEmptyFlush := h.heap.Len() == 0 - var finishCh chan error - if !isEmptyFlush { - failpoint.Inject("InjectErrorBackEndAlloc", func() { - failpoint.Return(cerrors.ErrUnifiedSorterIOError.Wrap(errors.New("injected alloc error")).FastGenWithCause()) - }) - - var err error - backEnd, err = pool.alloc(ctx) - if err != nil { - return errors.Trace(err) - } - - finishCh = make(chan error, 1) - } - - task := &flushTask{ - taskID: h.taskCounter, - heapSorterID: h.id, - backend: backEnd, - tsLowerBound: lowerBound, - maxResolvedTs: maxResolvedTs, - finished: finishCh, - canceller: h.canceller, - isEmpty: isEmptyFlush, - } - h.taskCounter++ - - var oldHeap sortHeap - if !isEmptyFlush { - task.dealloc = func() error { - backEnd := task.GetBackEnd() - if backEnd != nil { - defer task.markDeallocated() - return pool.dealloc(backEnd) - } - return nil - } - oldHeap = h.heap - h.heap = make(sortHeap, 0, sortHeapCapacity) - } else { - task.dealloc = func() error { - task.markDeallocated() - return nil - } - } - failpoint.Inject("sorterDebug", func() { - tableID, tableName := util.TableIDFromCtx(ctx) - log.Debug("Unified Sorter new flushTask", - zap.Int64("table-id", tableID), - zap.String("table-name", tableName), - zap.Int("heap-id", task.heapSorterID), - zap.Uint64("resolvedTs", task.maxResolvedTs)) - }) - - if !isEmptyFlush { - backEndFinal := backEnd - err := heapSorterIOPool.Go(ctx, func() { - failpoint.Inject("asyncFlushStartDelay", func() { - log.Debug("asyncFlushStartDelay") - }) - - h.canceller.EnterAsyncOp() - defer h.canceller.FinishAsyncOp() - - if h.canceller.IsCanceled() { - if backEndFinal != nil { - _ = task.dealloc() - } - task.finished <- cerrors.ErrAsyncIOCancelled.GenWithStackByArgs() - return - } - - writer, err := backEnd.writer() - if err != nil { - if backEndFinal != nil { - _ = task.dealloc() - } - task.finished <- errors.Trace(err) - return - } - - defer func() { - // handle errors (or aborts) gracefully to prevent resource leaking (especially FD's) - if writer != nil { - _ = writer.flushAndClose() - } - if backEndFinal != nil { - _ = task.dealloc() - } - close(task.finished) - }() - - failpoint.Inject("InjectErrorBackEndWrite", func() { - task.finished <- cerrors.ErrUnifiedSorterIOError.Wrap(errors.New("injected write error")).FastGenWithCause() - failpoint.Return() - }) - - counter := 0 - for oldHeap.Len() > 0 { - failpoint.Inject("asyncFlushInProcessDelay", func() { - log.Debug("asyncFlushInProcessDelay") - }) - // no need to check for cancellation so frequently. - if counter%10000 == 0 && h.canceller.IsCanceled() { - task.finished <- cerrors.ErrAsyncIOCancelled.GenWithStackByArgs() - return - } - counter++ - - event := heap.Pop(&oldHeap).(*sortItem).entry - err := writer.writeNext(event) - if err != nil { - task.finished <- errors.Trace(err) - return - } - } - - dataSize := writer.dataSize() - atomic.StoreInt64(&task.dataSize, int64(dataSize)) - eventCount := writer.writtenCount() - - writer1 := writer - writer = nil - err = writer1.flushAndClose() - if err != nil { - task.finished <- errors.Trace(err) - return - } - - backEndFinal = nil - - failpoint.Inject("sorterDebug", func() { - tableID, tableName := util.TableIDFromCtx(ctx) - log.Debug("Unified Sorter flushTask finished", - zap.Int("heap-id", task.heapSorterID), - zap.Int64("table-id", tableID), - zap.String("table-name", tableName), - zap.Uint64("resolvedTs", task.maxResolvedTs), - zap.Uint64("data-size", dataSize), - zap.Int("size", eventCount)) - }) - - task.finished <- nil // DO NOT access `task` beyond this point in this function - }) - if err != nil { - close(task.finished) - return errors.Trace(err) - } - } - - select { - case <-ctx.Done(): - return ctx.Err() - case h.outputCh <- task: - } - return nil -} - -var ( - heapSorterPool workerpool.WorkerPool - heapSorterIOPool workerpool.AsyncPool - poolOnce sync.Once -) - -type heapSorterInternalState struct { - maxResolved uint64 - heapSizeBytesEstimate int64 - rateCounter int - sorterConfig *config.SorterConfig - timerMultiplier int -} - -func (h *heapSorter) init(ctx context.Context, onError func(err error)) { - state := &heapSorterInternalState{ - sorterConfig: config.GetGlobalServerConfig().Sorter, - } - - poolHandle := heapSorterPool.RegisterEvent(func(ctx context.Context, eventI interface{}) error { - event := eventI.(*model.PolymorphicEvent) - heap.Push(&h.heap, &sortItem{entry: event}) - isResolvedEvent := event.RawKV != nil && event.RawKV.OpType == model.OpTypeResolved - - if isResolvedEvent { - if event.RawKV.CRTs < state.maxResolved { - log.Panic("ResolvedTs regression, bug?", zap.Uint64("event-resolvedTs", event.RawKV.CRTs), - zap.Uint64("max-resolvedTs", state.maxResolved)) - } - state.maxResolved = event.RawKV.CRTs - } - - if event.RawKV.CRTs < state.maxResolved { - log.Panic("Bad input to sorter", zap.Uint64("cur-ts", event.RawKV.CRTs), zap.Uint64("maxResolved", state.maxResolved)) - } - - // 5 * 8 is for the 5 fields in PolymorphicEvent - state.heapSizeBytesEstimate += event.RawKV.ApproximateDataSize() + 40 - needFlush := state.heapSizeBytesEstimate >= int64(state.sorterConfig.ChunkSizeLimit) || - (isResolvedEvent && state.rateCounter < flushRateLimitPerSecond) - - if needFlush { - state.rateCounter++ - err := h.flush(ctx, state.maxResolved) - if err != nil { - return errors.Trace(err) - } - state.heapSizeBytesEstimate = 0 - } - - return nil - }).SetTimer(ctx, 1*time.Second, func(ctx context.Context) error { - state.rateCounter = 0 - state.timerMultiplier = (state.timerMultiplier + 1) % 5 - if state.timerMultiplier == 0 && state.rateCounter < flushRateLimitPerSecond { - err := h.flush(ctx, state.maxResolved) - if err != nil { - return errors.Trace(err) - } - state.heapSizeBytesEstimate = 0 - } - return nil - }).OnExit(onError) - - h.poolHandle = poolHandle - h.internalState = state -} - -// asyncCanceller is a shared object used to cancel async IO operations. -// We do not use `context.Context` because (1) selecting on `ctx.Done()` is expensive -// especially if the context is shared by many goroutines, and (2) due to the complexity -// of managing contexts through the workerpools, using a special shared object seems more reasonable -// and readable. -type asyncCanceller struct { - exitRWLock sync.RWMutex // held when an asynchronous flush is taking place - hasExited int32 // this flag should be accessed atomically -} - -func (c *asyncCanceller) EnterAsyncOp() { - c.exitRWLock.RLock() -} - -func (c *asyncCanceller) FinishAsyncOp() { - c.exitRWLock.RUnlock() -} - -func (c *asyncCanceller) IsCanceled() bool { - return atomic.LoadInt32(&c.hasExited) == 1 -} - -func (c *asyncCanceller) Cancel() { - // Sets the flag - atomic.StoreInt32(&c.hasExited, 1) - - // By taking the lock, we are making sure that all IO operations that started before setting the flag have finished, - // so that by the returning of this function, no more IO operations will finish successfully. - // Since IO operations that are NOT successful will clean up themselves, the goroutine in which this - // function was called is responsible for releasing files written by only those IO operations that complete BEFORE - // this function returns. - // In short, we are creating a linearization point here. - c.exitRWLock.Lock() - defer c.exitRWLock.Unlock() -} - -func lazyInitWorkerPool() { - poolOnce.Do(func() { - sorterConfig := config.GetGlobalServerConfig().Sorter - heapSorterPool = workerpool.NewDefaultWorkerPool(sorterConfig.NumWorkerPoolGoroutine) - heapSorterIOPool = workerpool.NewDefaultAsyncPool(sorterConfig.NumWorkerPoolGoroutine * 2) - }) -} diff --git a/cdc/cdc/sorter/unified/memory_backend.go b/cdc/cdc/sorter/unified/memory_backend.go deleted file mode 100644 index 70c16f8f..00000000 --- a/cdc/cdc/sorter/unified/memory_backend.go +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package unified - -import ( - "sync/atomic" - - "github.com/pingcap/failpoint" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" - "go.uber.org/zap" -) - -type memoryBackEnd struct { - events []*model.PolymorphicEvent - estimatedSize int64 - borrowed int32 -} - -func newMemoryBackEnd() *memoryBackEnd { - return &memoryBackEnd{} -} - -func (m *memoryBackEnd) reader() (backEndReader, error) { - failpoint.Inject("sorterDebug", func() { - if atomic.SwapInt32(&m.borrowed, 1) != 0 { - log.Panic("memoryBackEnd: already borrowed") - } - }) - - return &memoryBackEndReader{ - backEnd: m, - readIndex: 0, - }, nil -} - -func (m *memoryBackEnd) writer() (backEndWriter, error) { - failpoint.Inject("sorterDebug", func() { - if atomic.SwapInt32(&m.borrowed, 1) != 0 { - log.Panic("memoryBackEnd: already borrowed") - } - }) - - return &memoryBackEndWriter{backEnd: m}, nil -} - -func (m *memoryBackEnd) free() error { - failpoint.Inject("sorterDebug", func() { - if atomic.LoadInt32(&m.borrowed) != 0 { - log.Panic("fileBackEnd: trying to free borrowed file") - } - }) - - if pool != nil { - atomic.AddInt64(&pool.memoryUseEstimate, -m.estimatedSize) - } - - return nil -} - -type memoryBackEndReader struct { - backEnd *memoryBackEnd - readIndex int -} - -func (r *memoryBackEndReader) readNext() (*model.PolymorphicEvent, error) { - // Check for "EOF" - if r.readIndex >= len(r.backEnd.events) { - return nil, nil - } - - ret := r.backEnd.events[r.readIndex] - // Sets the slot to nil to prevent delaying GC. - r.backEnd.events[r.readIndex] = nil - r.readIndex++ - return ret, nil -} - -func (r *memoryBackEndReader) resetAndClose() error { - failpoint.Inject("sorterDebug", func() { - atomic.StoreInt32(&r.backEnd.borrowed, 0) - }) - - if pool != nil { - atomic.AddInt64(&pool.memoryUseEstimate, -r.backEnd.estimatedSize) - } - r.backEnd.estimatedSize = 0 - - return nil -} - -type memoryBackEndWriter struct { - backEnd *memoryBackEnd - bytesWritten int64 - // for debugging only - maxTs uint64 -} - -func (w *memoryBackEndWriter) writeNext(event *model.PolymorphicEvent) error { - w.backEnd.events = append(w.backEnd.events, event) - // 8 * 5 is for the 5 fields in PolymorphicEvent, each of which is thought of as a 64-bit pointer - w.bytesWritten += 8*5 + event.RawKV.ApproximateDataSize() - - failpoint.Inject("sorterDebug", func() { - if event.CRTs < w.maxTs { - log.Panic("memoryBackEnd: ts regressed, bug?", - zap.Uint64("prev-ts", w.maxTs), - zap.Uint64("cur-ts", event.CRTs)) - } - w.maxTs = event.CRTs - }) - return nil -} - -func (w *memoryBackEndWriter) writtenCount() int { - return len(w.backEnd.events) -} - -// dataSize for the memoryBackEnd returns only an estimation, as there is no serialization taking place. -func (w *memoryBackEndWriter) dataSize() uint64 { - return uint64(w.bytesWritten) -} - -func (w *memoryBackEndWriter) flushAndClose() error { - failpoint.Inject("sorterDebug", func() { - atomic.StoreInt32(&w.backEnd.borrowed, 0) - }) - - w.backEnd.estimatedSize = w.bytesWritten - if pool != nil { - atomic.AddInt64(&pool.memoryUseEstimate, w.bytesWritten) - } - - return nil -} diff --git a/cdc/cdc/sorter/unified/memory_backend_test.go b/cdc/cdc/sorter/unified/memory_backend_test.go deleted file mode 100644 index b7622a49..00000000 --- a/cdc/cdc/sorter/unified/memory_backend_test.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package unified - -import ( - "runtime" - "sync/atomic" - "time" - - "github.com/pingcap/check" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/util/testleak" -) - -type memoryBackendSuite struct{} - -var _ = check.SerialSuites(&memoryBackendSuite{}) - -func (s *memoryBackendSuite) TestNoLeaking(c *check.C) { - defer testleak.AfterTest(c)() - - bknd := newMemoryBackEnd() - wrtr, err := bknd.writer() - c.Assert(err, check.IsNil) - - var objCount int64 - for i := 0; i < 10000; i++ { - atomic.AddInt64(&objCount, 1) - event := model.NewResolvedPolymorphicEvent(0, 1) - runtime.SetFinalizer(event, func(*model.PolymorphicEvent) { - atomic.AddInt64(&objCount, -1) - }) - err := wrtr.writeNext(event) - c.Assert(err, check.IsNil) - } - err = wrtr.flushAndClose() - c.Assert(err, check.IsNil) - - rdr, err := bknd.reader() - c.Assert(err, check.IsNil) - - for i := 0; i < 5000; i++ { - _, err := rdr.readNext() - c.Assert(err, check.IsNil) - } - - for i := 0; i < 10; i++ { - runtime.GC() - if atomic.LoadInt64(&objCount) <= 5000 { - break - } - time.Sleep(100 * time.Millisecond) - } - c.Assert(atomic.LoadInt64(&objCount), check.LessEqual, int64(5000)) - - err = rdr.resetAndClose() - c.Assert(err, check.IsNil) - - for i := 0; i < 10; i++ { - runtime.GC() - if atomic.LoadInt64(&objCount) == 0 { - break - } - time.Sleep(100 * time.Millisecond) - } - c.Assert(atomic.LoadInt64(&objCount), check.Equals, int64(0)) -} diff --git a/cdc/cdc/sorter/unified/merger.go b/cdc/cdc/sorter/unified/merger.go deleted file mode 100644 index 1f4501ef..00000000 --- a/cdc/cdc/sorter/unified/merger.go +++ /dev/null @@ -1,515 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package unified - -import ( - "container/heap" - "context" - "math" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/pingcap/log" - "github.com/tikv/client-go/v2/oracle" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/sorter" - cerrors "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/notify" - "github.com/tikv/migration/cdc/pkg/util" - "go.uber.org/zap" - "golang.org/x/sync/errgroup" -) - -// TODO refactor this into a struct Merger. -func runMerger(ctx context.Context, numSorters int, in <-chan *flushTask, out chan *model.PolymorphicEvent, onExit func()) error { - captureAddr := util.CaptureAddrFromCtx(ctx) - changefeedID := util.ChangefeedIDFromCtx(ctx) - - metricSorterEventCount := sorter.EventCount.MustCurryWith(map[string]string{ - "capture": captureAddr, - "changefeed": changefeedID, - }) - metricSorterResolvedTsGauge := sorter.ResolvedTsGauge.WithLabelValues(captureAddr, changefeedID) - metricSorterMergerStartTsGauge := sorterMergerStartTsGauge.WithLabelValues(captureAddr, changefeedID) - metricSorterMergeCountHistogram := sorterMergeCountHistogram.WithLabelValues(captureAddr, changefeedID) - - lastResolvedTs := make([]uint64, numSorters) - minResolvedTs := uint64(0) - var workingSet map[*flushTask]struct{} - pendingSet := &sync.Map{} - - defer func() { - log.Debug("Unified Sorter: merger exiting, cleaning up resources") - // cancel pending async IO operations. - onExit() - cleanUpTask := func(task *flushTask) { - select { - case err := <-task.finished: - _ = printError(err) - default: - // The task has not finished, so we give up. - // It does not matter because: - // 1) if the async workerpool has exited, it means the CDC process is exiting, CleanUp will - // take care of the temp files, - // 2) if the async workerpool is not exiting, the unfinished tasks will eventually be executed, - // and by that time, since the `onExit` have canceled them, they will not do any IO and clean up themselves. - return - } - - if task.reader != nil { - _ = printError(task.reader.resetAndClose()) - task.reader = nil - } - _ = printError(task.dealloc()) - } - - LOOP: - for { - var task *flushTask - select { - case task = <-in: - default: - break LOOP - } - - if task == nil { - log.Debug("Merger exiting, in-channel is exhausted") - break - } - - cleanUpTask(task) - } - - pendingSet.Range(func(task, _ interface{}) bool { - cleanUpTask(task.(*flushTask)) - return true - }) - for task := range workingSet { - cleanUpTask(task) - } - }() - - lastOutputTs := uint64(0) - lastOutputResolvedTs := uint64(0) - var lastEvent *model.PolymorphicEvent - var lastTask *flushTask - - sendResolvedEvent := func(ts uint64) error { - lastOutputResolvedTs = ts - if ts == 0 { - return nil - } - select { - case <-ctx.Done(): - return ctx.Err() - case out <- model.NewResolvedPolymorphicEvent(0, ts): - metricSorterEventCount.WithLabelValues("resolved").Inc() - metricSorterResolvedTsGauge.Set(float64(oracle.ExtractPhysical(ts))) - return nil - } - } - - onMinResolvedTsUpdate := func(minResolvedTs /* note the shadowing */ uint64) error { - metricSorterMergerStartTsGauge.Set(float64(oracle.ExtractPhysical(minResolvedTs))) - workingSet = make(map[*flushTask]struct{}) - sortHeap := new(sortHeap) - - // loopErr is used to return an error out of the closure taken by `pendingSet.Range`. - var loopErr error - // NOTE 1: We can block the closure passed to `pendingSet.Range` WITHOUT worrying about - // deadlocks because the closure is NOT called with any lock acquired in the implementation - // of Sync.Map. - // NOTE 2: It is safe to used `Range` to iterate through the pendingSet, in spite of NOT having - // a snapshot consistency because (1) pendingSet is updated first before minResolvedTs is updated, - // which guarantees that useful new flushTasks are not missed, and (2) by design, once minResolvedTs is updated, - // new flushTasks will satisfy `task.tsLowerBound > minResolvedTs`, and such flushTasks are ignored in - // the closure. - pendingSet.Range(func(iTask, iCache interface{}) bool { - task := iTask.(*flushTask) - var cache *model.PolymorphicEvent - if iCache != nil { - cache = iCache.(*model.PolymorphicEvent) - } - - if task.tsLowerBound > minResolvedTs { - // the condition above implies that for any event in task.backend, CRTs > minResolvedTs. - return true - } - var event *model.PolymorphicEvent - if cache != nil { - event = cache - } else { - select { - case <-ctx.Done(): - loopErr = ctx.Err() - // terminates the loop - return false - case err := <-task.finished: - if err != nil { - loopErr = errors.Trace(err) - // terminates the loop - return false - } - } - - if task.reader == nil { - var err error - task.reader, err = task.GetBackEnd().reader() - if err != nil { - loopErr = errors.Trace(err) - // terminates the loop - return false - } - } - - var err error - event, err = task.reader.readNext() - if err != nil { - loopErr = errors.Trace(err) - // terminates the loop - return false - } - - if event == nil { - log.Panic("Unexpected end of backEnd data, bug?", - zap.Uint64("minResolvedTs", task.maxResolvedTs)) - } - } - - if event.CRTs > minResolvedTs { - pendingSet.Store(task, event) - // continues the loop - return true - } - - pendingSet.Store(task, nil) - workingSet[task] = struct{}{} - - heap.Push(sortHeap, &sortItem{ - entry: event, - data: task, - }) - return true - }) - if loopErr != nil { - return errors.Trace(loopErr) - } - - resolvedTicker := time.NewTicker(1 * time.Second) - defer resolvedTicker.Stop() - - retire := func(task *flushTask) error { - delete(workingSet, task) - cached, ok := pendingSet.Load(task) - if !ok { - log.Panic("task not found in pendingSet") - } - - if cached != nil { - return nil - } - - nextEvent, err := task.reader.readNext() - if err != nil { - _ = task.reader.resetAndClose() // prevents fd leak - task.reader = nil - return errors.Trace(err) - } - - if nextEvent == nil { - pendingSet.Delete(task) - - err := task.reader.resetAndClose() - if err != nil { - return errors.Trace(err) - } - task.reader = nil - - err = task.dealloc() - if err != nil { - return errors.Trace(err) - } - } else { - pendingSet.Store(task, nextEvent) - if nextEvent.CRTs < minResolvedTs { - log.Panic("remaining event CRTs too small", - zap.Uint64("next-ts", nextEvent.CRTs), - zap.Uint64("minResolvedTs", minResolvedTs)) - } - } - return nil - } - - failpoint.Inject("sorterDebug", func() { - if sortHeap.Len() > 0 { - tableID, tableName := util.TableIDFromCtx(ctx) - log.Debug("Unified Sorter: start merging", - zap.Int64("table-id", tableID), - zap.String("table-name", tableName), - zap.Uint64("minResolvedTs", minResolvedTs)) - } - }) - - counter := 0 - for sortHeap.Len() > 0 { - failpoint.Inject("sorterMergeDelay", func() {}) - - item := heap.Pop(sortHeap).(*sortItem) - task := item.data.(*flushTask) - event := item.entry - - if event.CRTs < task.lastTs { - log.Panic("unified sorter: ts regressed in one backEnd, bug?", zap.Uint64("cur-ts", event.CRTs), zap.Uint64("last-ts", task.lastTs)) - } - task.lastTs = event.CRTs - - if event.RawKV != nil && event.RawKV.OpType != model.OpTypeResolved { - if event.CRTs < lastOutputTs { - for sortHeap.Len() > 0 { - item := heap.Pop(sortHeap).(*sortItem) - task := item.data.(*flushTask) - event := item.entry - log.Debug("dump", zap.Reflect("event", event), zap.Int("heap-id", task.heapSorterID)) - } - log.Panic("unified sorter: output ts regressed, bug?", - zap.Int("counter", counter), - zap.Uint64("minResolvedTs", minResolvedTs), - zap.Int("cur-heap-id", task.heapSorterID), - zap.Int("cur-task-id", task.taskID), - zap.Uint64("cur-task-resolved", task.maxResolvedTs), - zap.Reflect("cur-event", event), - zap.Uint64("cur-ts", event.CRTs), - zap.Int("last-heap-id", lastTask.heapSorterID), - zap.Int("last-task-id", lastTask.taskID), - zap.Uint64("last-task-resolved", task.maxResolvedTs), - zap.Reflect("last-event", lastEvent), - zap.Uint64("last-ts", lastOutputTs), - zap.Int("sort-heap-len", sortHeap.Len())) - } - - if event.CRTs <= lastOutputResolvedTs { - log.Panic("unified sorter: output ts smaller than resolved ts, bug?", zap.Uint64("minResolvedTs", minResolvedTs), - zap.Uint64("lastOutputResolvedTs", lastOutputResolvedTs), zap.Uint64("event-crts", event.CRTs)) - } - lastOutputTs = event.CRTs - lastEvent = event - lastTask = task - select { - case <-ctx.Done(): - return ctx.Err() - case out <- event: - metricSorterEventCount.WithLabelValues("kv").Inc() - } - } - counter += 1 - - select { - case <-resolvedTicker.C: - err := sendResolvedEvent(event.CRTs - 1) - if err != nil { - return errors.Trace(err) - } - default: - } - - event, err := task.reader.readNext() - if err != nil { - return errors.Trace(err) - } - - if event == nil { - // EOF - delete(workingSet, task) - pendingSet.Delete(task) - - err := task.reader.resetAndClose() - if err != nil { - return errors.Trace(err) - } - task.reader = nil - - err = task.dealloc() - if err != nil { - return errors.Trace(err) - } - - continue - } - - if event.CRTs > minResolvedTs || (event.CRTs == minResolvedTs && event.RawKV.OpType == model.OpTypeResolved) { - // we have processed all events from this task that need to be processed in this merge - if event.CRTs > minResolvedTs || event.RawKV.OpType != model.OpTypeResolved { - pendingSet.Store(task, event) - } - err := retire(task) - if err != nil { - return errors.Trace(err) - } - continue - } - - failpoint.Inject("sorterDebug", func() { - if counter%10 == 0 { - tableID, tableName := util.TableIDFromCtx(ctx) - log.Debug("Merging progress", - zap.Int64("table-id", tableID), - zap.String("table-name", tableName), - zap.Int("counter", counter)) - } - }) - - heap.Push(sortHeap, &sortItem{ - entry: event, - data: task, - }) - } - - if len(workingSet) != 0 { - log.Panic("unified sorter: merging ended prematurely, bug?", zap.Uint64("resolvedTs", minResolvedTs)) - } - - failpoint.Inject("sorterDebug", func() { - if counter > 0 { - tableID, tableName := util.TableIDFromCtx(ctx) - log.Debug("Unified Sorter: merging ended", - zap.Int64("table-id", tableID), - zap.String("table-name", tableName), - zap.Uint64("resolvedTs", minResolvedTs), zap.Int("count", counter)) - } - }) - err := sendResolvedEvent(minResolvedTs) - if err != nil { - return errors.Trace(err) - } - - if counter > 0 { - // ignore empty merges for better visualization of metrics - metricSorterMergeCountHistogram.Observe(float64(counter)) - } - - return nil - } - - resolvedTsNotifier := ¬ify.Notifier{} - defer resolvedTsNotifier.Close() - errg, ctx := errgroup.WithContext(ctx) - - errg.Go(func() error { - for { - var task *flushTask - select { - case <-ctx.Done(): - return ctx.Err() - case task = <-in: - } - - if task == nil { - tableID, tableName := util.TableIDFromCtx(ctx) - log.Debug("Merger input channel closed, exiting", - zap.Int64("table-id", tableID), - zap.String("table-name", tableName)) - return nil - } - - if !task.isEmpty { - pendingSet.Store(task, nil) - } // otherwise it is an empty flush - - if lastResolvedTs[task.heapSorterID] < task.maxResolvedTs { - lastResolvedTs[task.heapSorterID] = task.maxResolvedTs - } - - minTemp := uint64(math.MaxUint64) - for _, ts := range lastResolvedTs { - if minTemp > ts { - minTemp = ts - } - } - - if minTemp > minResolvedTs { - atomic.StoreUint64(&minResolvedTs, minTemp) - resolvedTsNotifier.Notify() - } - } - }) - - errg.Go(func() error { - resolvedTsReceiver, err := resolvedTsNotifier.NewReceiver(time.Second * 1) - if err != nil { - if cerrors.ErrOperateOnClosedNotifier.Equal(err) { - // This won't happen unless `resolvedTsNotifier` has been closed, which is - // impossible at this point. - log.Panic("unexpected error", zap.Error(err)) - } - return errors.Trace(err) - } - - defer resolvedTsReceiver.Stop() - - var lastResolvedTs uint64 - for { - select { - case <-ctx.Done(): - return ctx.Err() - case <-resolvedTsReceiver.C: - curResolvedTs := atomic.LoadUint64(&minResolvedTs) - if curResolvedTs > lastResolvedTs { - err := onMinResolvedTsUpdate(curResolvedTs) - if err != nil { - return errors.Trace(err) - } - } else if curResolvedTs < lastResolvedTs { - log.Panic("resolved-ts regressed in sorter", - zap.Uint64("cur-resolved-ts", curResolvedTs), - zap.Uint64("last-resolved-ts", lastResolvedTs)) - } - } - } - }) - - return errg.Wait() -} - -func mergerCleanUp(in <-chan *flushTask) { - for task := range in { - select { - case err := <-task.finished: - _ = printError(err) - default: - break - } - - if task.reader != nil { - _ = printError(task.reader.resetAndClose()) - } - _ = printError(task.dealloc()) - } -} - -// printError is a helper for tracing errors on function returns -func printError(err error) error { - if err != nil && errors.Cause(err) != context.Canceled && - errors.Cause(err) != context.DeadlineExceeded && - !strings.Contains(err.Error(), "context canceled") && - !strings.Contains(err.Error(), "context deadline exceeded") && - cerrors.ErrAsyncIOCancelled.NotEqual(errors.Cause(err)) { - - log.Warn("Unified Sorter: Error detected", zap.Error(err), zap.Stack("stack")) - } - return err -} diff --git a/cdc/cdc/sorter/unified/merger_test.go b/cdc/cdc/sorter/unified/merger_test.go deleted file mode 100644 index d18cbb45..00000000 --- a/cdc/cdc/sorter/unified/merger_test.go +++ /dev/null @@ -1,553 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package unified - -import ( - "context" - "sync/atomic" - "time" - - "github.com/pingcap/check" - "github.com/pingcap/failpoint" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/util/testleak" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - "golang.org/x/sync/errgroup" -) - -type mockFlushTaskBuilder struct { - task *flushTask - writer backEndWriter - totalCount int -} - -var backEndCounterForTest int64 - -func newMockFlushTaskBuilder() *mockFlushTaskBuilder { - backEnd := newMemoryBackEnd() - atomic.AddInt64(&backEndCounterForTest, 1) - - task := &flushTask{ - backend: backEnd, - tsLowerBound: 0, - maxResolvedTs: 0, - finished: make(chan error, 2), - } - - task.dealloc = func() error { - if task.backend != nil { - atomic.AddInt64(&backEndCounterForTest, -1) - task.backend = nil - return backEnd.free() - } - return nil - } - - writer, _ := backEnd.writer() - - return &mockFlushTaskBuilder{ - task: task, - writer: writer, - } -} - -func (b *mockFlushTaskBuilder) generateRowChanges(tsRangeBegin, tsRangeEnd uint64, count int) *mockFlushTaskBuilder { - if b.task.tsLowerBound == 0 { - b.task.tsLowerBound = tsRangeBegin - } - density := float64(tsRangeEnd-tsRangeBegin) / float64(count) - for fTs := float64(tsRangeBegin); fTs < float64(tsRangeEnd); fTs += density { - ts := uint64(fTs) - kvEntry := generateMockRawKV(ts) - _ = b.writer.writeNext(model.NewPolymorphicEvent(kvEntry)) - b.totalCount++ - } - return b -} - -func (b *mockFlushTaskBuilder) addResolved(ts uint64) *mockFlushTaskBuilder { - _ = b.writer.writeNext(model.NewResolvedPolymorphicEvent(0, ts)) - b.task.maxResolvedTs = ts - return b -} - -func (b *mockFlushTaskBuilder) build() *flushTask { - _ = b.writer.flushAndClose() - return b.task -} - -// TestMergerSingleHeap simulates a situation where there is only one data stream -// It tests the most basic scenario. -func (s *sorterSuite) TestMergerSingleHeap(c *check.C) { - defer testleak.AfterTest(c)() - err := failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/sorterDebug", "return(true)") - if err != nil { - log.Panic("Could not enable failpoint", zap.Error(err)) - } - - ctx, cancel := context.WithTimeout(context.TODO(), time.Second*10) - defer cancel() - wg, ctx := errgroup.WithContext(ctx) - inChan := make(chan *flushTask, 1024) - outChan := make(chan *model.PolymorphicEvent, 1024) - - wg.Go(func() error { - return runMerger(ctx, 1, inChan, outChan, func() {}) - }) - - totalCount := 0 - builder := newMockFlushTaskBuilder() - task1 := builder.generateRowChanges(1000, 100000, 2048).addResolved(100001).build() - totalCount += builder.totalCount - builder = newMockFlushTaskBuilder() - task2 := builder.generateRowChanges(100002, 200000, 2048).addResolved(200001).build() - totalCount += builder.totalCount - builder = newMockFlushTaskBuilder() - task3 := builder.generateRowChanges(200002, 300000, 2048).addResolved(300001).build() - totalCount += builder.totalCount - - wg.Go(func() error { - inChan <- task1 - close(task1.finished) - inChan <- task2 - close(task2.finished) - inChan <- task3 - close(task3.finished) - - return nil - }) - - wg.Go(func() error { - count := 0 - lastTs := uint64(0) - lastResolved := uint64(0) - for { - select { - case <-ctx.Done(): - return ctx.Err() - case event := <-outChan: - switch event.RawKV.OpType { - case model.OpTypePut: - count++ - c.Assert(event.CRTs, check.GreaterEqual, lastTs) - c.Assert(event.CRTs, check.GreaterEqual, lastResolved) - lastTs = event.CRTs - case model.OpTypeResolved: - c.Assert(event.CRTs, check.GreaterEqual, lastResolved) - lastResolved = event.CRTs - } - if lastResolved >= 300001 { - c.Assert(count, check.Equals, totalCount) - cancel() - return nil - } - } - } - }) - c.Assert(wg.Wait(), check.ErrorMatches, ".*context canceled.*") - c.Assert(atomic.LoadInt64(&backEndCounterForTest), check.Equals, int64(0)) -} - -// TestMergerSingleHeapRetire simulates a situation where the resolved event is not the last event in a flushTask -func (s *sorterSuite) TestMergerSingleHeapRetire(c *check.C) { - defer testleak.AfterTest(c)() - err := failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/sorterDebug", "return(true)") - if err != nil { - log.Panic("Could not enable failpoint", zap.Error(err)) - } - - ctx, cancel := context.WithTimeout(context.TODO(), time.Second*10) - defer cancel() - wg, ctx := errgroup.WithContext(ctx) - inChan := make(chan *flushTask, 1024) - outChan := make(chan *model.PolymorphicEvent, 1024) - - wg.Go(func() error { - return runMerger(ctx, 1, inChan, outChan, func() {}) - }) - - totalCount := 0 - builder := newMockFlushTaskBuilder() - task1 := builder.generateRowChanges(1000, 100000, 2048).addResolved(100001).build() - totalCount += builder.totalCount - builder = newMockFlushTaskBuilder() - task2 := builder.generateRowChanges(100002, 200000, 2048).build() - totalCount += builder.totalCount - builder = newMockFlushTaskBuilder() - task3 := builder.generateRowChanges(200002, 300000, 2048).addResolved(300001).build() - totalCount += builder.totalCount - - wg.Go(func() error { - inChan <- task1 - close(task1.finished) - inChan <- task2 - close(task2.finished) - inChan <- task3 - close(task3.finished) - - return nil - }) - - wg.Go(func() error { - count := 0 - lastTs := uint64(0) - lastResolved := uint64(0) - for { - select { - case <-ctx.Done(): - return ctx.Err() - case event := <-outChan: - switch event.RawKV.OpType { - case model.OpTypePut: - count++ - c.Assert(event.CRTs, check.GreaterEqual, lastResolved) - c.Assert(event.CRTs, check.GreaterEqual, lastTs) - lastTs = event.CRTs - case model.OpTypeResolved: - c.Assert(event.CRTs, check.GreaterEqual, lastResolved) - lastResolved = event.CRTs - } - if lastResolved >= 300001 { - c.Assert(count, check.Equals, totalCount) - cancel() - return nil - } - } - } - }) - - c.Assert(wg.Wait(), check.ErrorMatches, ".*context canceled.*") - c.Assert(atomic.LoadInt64(&backEndCounterForTest), check.Equals, int64(0)) -} - -// TestMergerSortDelay simulates a situation where merging takes a long time. -// Expects intermediate resolved events to be generated, so that the sink would not get stuck in a real life situation. -func (s *sorterSuite) TestMergerSortDelay(c *check.C) { - defer testleak.AfterTest(c)() - err := failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/sorterDebug", "return(true)") - c.Assert(err, check.IsNil) - - // enable the failpoint to simulate delays - err = failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/sorterMergeDelay", "sleep(5)") - c.Assert(err, check.IsNil) - defer func() { - _ = failpoint.Disable("github.com/tikv/migration/cdc/cdc/sorter/unified/sorterMergeDelay") - }() - - log.SetLevel(zapcore.DebugLevel) - defer log.SetLevel(zapcore.InfoLevel) - - ctx, cancel := context.WithTimeout(context.TODO(), time.Second*10) - defer cancel() - wg, ctx := errgroup.WithContext(ctx) - inChan := make(chan *flushTask, 1024) - outChan := make(chan *model.PolymorphicEvent, 1024) - - wg.Go(func() error { - return runMerger(ctx, 1, inChan, outChan, func() {}) - }) - - totalCount := 0 - builder := newMockFlushTaskBuilder() - task1 := builder.generateRowChanges(1000, 1000000, 1024).addResolved(1000001).build() - totalCount += builder.totalCount - - wg.Go(func() error { - inChan <- task1 - close(task1.finished) - return nil - }) - - wg.Go(func() error { - var ( - count int - lastTs uint64 - lastResolved uint64 - lastResolvedTime time.Time - ) - for { - select { - case <-ctx.Done(): - return ctx.Err() - case event := <-outChan: - switch event.RawKV.OpType { - case model.OpTypePut: - count++ - c.Assert(event.CRTs, check.GreaterEqual, lastResolved) - c.Assert(event.CRTs, check.GreaterEqual, lastTs) - lastTs = event.CRTs - case model.OpTypeResolved: - c.Assert(event.CRTs, check.GreaterEqual, lastResolved) - if !lastResolvedTime.IsZero() { - c.Assert(time.Since(lastResolvedTime), check.LessEqual, 2*time.Second) - } - log.Debug("resolved event received", zap.Uint64("ts", event.CRTs)) - lastResolvedTime = time.Now() - lastResolved = event.CRTs - } - if lastResolved >= 1000001 { - c.Assert(count, check.Equals, totalCount) - cancel() - return nil - } - } - } - }) - - c.Assert(wg.Wait(), check.ErrorMatches, ".*context canceled.*") - close(inChan) - mergerCleanUp(inChan) - c.Assert(atomic.LoadInt64(&backEndCounterForTest), check.Equals, int64(0)) -} - -// TestMergerCancel simulates a situation where the merger is cancelled with pending data. -// Expects proper clean-up of the data. -func (s *sorterSuite) TestMergerCancel(c *check.C) { - defer testleak.AfterTest(c)() - err := failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/sorterDebug", "return(true)") - c.Assert(err, check.IsNil) - - // enable the failpoint to simulate delays - err = failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/sorterMergeDelay", "sleep(10)") - c.Assert(err, check.IsNil) - defer func() { - _ = failpoint.Disable("github.com/tikv/migration/cdc/cdc/sorter/unified/sorterMergeDelay") - }() - - log.SetLevel(zapcore.DebugLevel) - defer log.SetLevel(zapcore.InfoLevel) - - ctx, cancel := context.WithTimeout(context.TODO(), time.Second*10) - defer cancel() - wg, ctx := errgroup.WithContext(ctx) - inChan := make(chan *flushTask, 1024) - outChan := make(chan *model.PolymorphicEvent, 1024) - - wg.Go(func() error { - return runMerger(ctx, 1, inChan, outChan, func() {}) - }) - - builder := newMockFlushTaskBuilder() - task1 := builder.generateRowChanges(1000, 100000, 2048).addResolved(100001).build() - builder = newMockFlushTaskBuilder() - task2 := builder.generateRowChanges(100002, 200000, 2048).addResolved(200001).build() - builder = newMockFlushTaskBuilder() - task3 := builder.generateRowChanges(200002, 300000, 2048).addResolved(300001).build() - - wg.Go(func() error { - inChan <- task1 - close(task1.finished) - inChan <- task2 - close(task2.finished) - inChan <- task3 - close(task3.finished) - return nil - }) - - wg.Go(func() error { - for { - select { - case <-ctx.Done(): - return ctx.Err() - case <-outChan: - // We just drain the data here. We don't care about it. - } - } - }) - - time.Sleep(5 * time.Second) - cancel() - c.Assert(wg.Wait(), check.ErrorMatches, ".*context canceled.*") - close(inChan) - mergerCleanUp(inChan) - c.Assert(atomic.LoadInt64(&backEndCounterForTest), check.Equals, int64(0)) -} - -// TestMergerCancel simulates a situation where the merger is cancelled with pending data. -// Expects proper clean-up of the data. -func (s *sorterSuite) TestMergerCancelWithUnfinishedFlushTasks(c *check.C) { - defer testleak.AfterTest(c)() - err := failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/sorterDebug", "return(true)") - c.Assert(err, check.IsNil) - - log.SetLevel(zapcore.DebugLevel) - defer log.SetLevel(zapcore.InfoLevel) - - ctx, cancel := context.WithTimeout(context.TODO(), time.Second*10) - wg, ctx := errgroup.WithContext(ctx) - inChan := make(chan *flushTask, 1024) - outChan := make(chan *model.PolymorphicEvent, 1024) - - wg.Go(func() error { - return runMerger(ctx, 1, inChan, outChan, func() {}) - }) - - builder := newMockFlushTaskBuilder() - task1 := builder.generateRowChanges(1000, 100000, 2048).addResolved(100001).build() - builder = newMockFlushTaskBuilder() - task2 := builder.generateRowChanges(100002, 200000, 2048).addResolved(200001).build() - builder = newMockFlushTaskBuilder() - task3 := builder.generateRowChanges(200002, 300000, 2048).addResolved(300001).build() - - wg.Go(func() error { - inChan <- task1 - inChan <- task2 - inChan <- task3 - close(task2.finished) - close(task1.finished) - time.Sleep(1 * time.Second) - cancel() - return nil - }) - - wg.Go(func() error { - for { - select { - case <-ctx.Done(): - return ctx.Err() - case <-outChan: - // We just drain the data here. We don't care about it. - } - } - }) - - c.Assert(wg.Wait(), check.ErrorMatches, ".*context canceled.*") - close(inChan) - mergerCleanUp(inChan) - // Leaking one task is expected - c.Assert(atomic.LoadInt64(&backEndCounterForTest), check.Equals, int64(1)) - atomic.StoreInt64(&backEndCounterForTest, 0) -} - -// TestMergerCancel simulates a situation where the input channel is abruptly closed. -// There is expected to be NO fatal error. -func (s *sorterSuite) TestMergerCloseChannel(c *check.C) { - defer testleak.AfterTest(c)() - err := failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/sorterDebug", "return(true)") - c.Assert(err, check.IsNil) - - log.SetLevel(zapcore.DebugLevel) - defer log.SetLevel(zapcore.InfoLevel) - - ctx, cancel := context.WithTimeout(context.TODO(), time.Second*15) - defer cancel() - wg, ctx := errgroup.WithContext(ctx) - inChan := make(chan *flushTask, 1024) - outChan := make(chan *model.PolymorphicEvent, 1024) - - builder := newMockFlushTaskBuilder() - task1 := builder.generateRowChanges(1000, 100000, 2048).addResolved(100001).build() - - inChan <- task1 - close(task1.finished) - - wg.Go(func() error { - return runMerger(ctx, 1, inChan, outChan, func() {}) - }) - - wg.Go(func() error { - for { - select { - case <-ctx.Done(): - return ctx.Err() - case <-outChan: - // We just drain the data here. We don't care about it. - } - } - }) - - time.Sleep(5 * time.Second) - close(inChan) - time.Sleep(5 * time.Second) - cancel() - c.Assert(wg.Wait(), check.ErrorMatches, ".*context canceled.*") - mergerCleanUp(inChan) - c.Assert(atomic.LoadInt64(&backEndCounterForTest), check.Equals, int64(0)) -} - -// TestMergerOutputBlocked simulates a situation where the output channel is blocked for -// a significant period of time. -func (s *sorterSuite) TestMergerOutputBlocked(c *check.C) { - defer testleak.AfterTest(c)() - err := failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/sorterDebug", "return(true)") - c.Assert(err, check.IsNil) - defer failpoint.Disable("github.com/tikv/migration/cdc/cdc/sorter/unified/sorterDebug") //nolint:errcheck - - ctx, cancel := context.WithTimeout(context.TODO(), time.Second*25) - defer cancel() - wg, ctx := errgroup.WithContext(ctx) - // use unbuffered channel to make sure that the input has been processed - inChan := make(chan *flushTask) - // make a small channel to test blocking - outChan := make(chan *model.PolymorphicEvent, 1) - - wg.Go(func() error { - return runMerger(ctx, 1, inChan, outChan, func() {}) - }) - - totalCount := 0 - builder := newMockFlushTaskBuilder() - task1 := builder.generateRowChanges(1000, 100000, 2048).addResolved(100001).build() - totalCount += builder.totalCount - builder = newMockFlushTaskBuilder() - task2 := builder.generateRowChanges(100002, 200000, 2048).addResolved(200001).build() - totalCount += builder.totalCount - builder = newMockFlushTaskBuilder() - task3 := builder.generateRowChanges(200002, 300000, 2048).addResolved(300001).build() - totalCount += builder.totalCount - - wg.Go(func() error { - inChan <- task1 - close(task1.finished) - inChan <- task2 - close(task2.finished) - inChan <- task3 - close(task3.finished) - - return nil - }) - - wg.Go(func() error { - time.Sleep(10 * time.Second) - count := 0 - lastTs := uint64(0) - lastResolved := uint64(0) - for { - select { - case <-ctx.Done(): - return ctx.Err() - case event := <-outChan: - switch event.RawKV.OpType { - case model.OpTypePut: - count++ - c.Assert(event.CRTs, check.GreaterEqual, lastTs) - c.Assert(event.CRTs, check.GreaterEqual, lastResolved) - lastTs = event.CRTs - case model.OpTypeResolved: - c.Assert(event.CRTs, check.GreaterEqual, lastResolved) - lastResolved = event.CRTs - } - if lastResolved >= 300001 { - c.Assert(count, check.Equals, totalCount) - cancel() - return nil - } - } - } - }) - c.Assert(wg.Wait(), check.ErrorMatches, ".*context canceled.*") - c.Assert(atomic.LoadInt64(&backEndCounterForTest), check.Equals, int64(0)) -} diff --git a/cdc/cdc/sorter/unified/metrics.go b/cdc/cdc/sorter/unified/metrics.go deleted file mode 100644 index d4f9a23e..00000000 --- a/cdc/cdc/sorter/unified/metrics.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package unified - -import ( - "github.com/prometheus/client_golang/prometheus" -) - -var ( - sorterConsumeCount = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "ticdc", - Subsystem: "sorter", - Name: "consume_count", - Help: "the number of events consumed by the sorter", - }, []string{"capture", "changefeed", "type"}) - - sorterMergerStartTsGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "ticdc", - Subsystem: "sorter", - Name: "merger_start_ts_gauge", - Help: "the start TS of each merge in the sorter", - }, []string{"capture", "changefeed"}) - - sorterFlushCountHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "ticdc", - Subsystem: "sorter", - Name: "flush_count_histogram", - Help: "Bucketed histogram of the number of events in individual flushes performed by the sorter", - Buckets: prometheus.ExponentialBuckets(4, 4, 10), - }, []string{"capture", "changefeed"}) - - sorterMergeCountHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "ticdc", - Subsystem: "sorter", - Name: "merge_count_histogram", - Help: "Bucketed histogram of the number of events in individual merges performed by the sorter", - Buckets: prometheus.ExponentialBuckets(16, 4, 10), - }, []string{"capture", "changefeed"}) -) - -// InitMetrics registers all metrics in this file -func InitMetrics(registry *prometheus.Registry) { - registry.MustRegister(sorterConsumeCount) - registry.MustRegister(sorterMergerStartTsGauge) - registry.MustRegister(sorterFlushCountHistogram) - registry.MustRegister(sorterMergeCountHistogram) -} diff --git a/cdc/cdc/sorter/unified/sorter_test.go b/cdc/cdc/sorter/unified/sorter_test.go deleted file mode 100644 index 7aa6d52b..00000000 --- a/cdc/cdc/sorter/unified/sorter_test.go +++ /dev/null @@ -1,478 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package unified - -import ( - "context" - "math" - _ "net/http/pprof" - "os" - "path/filepath" - "sync/atomic" - "testing" - "time" - - "github.com/pingcap/check" - "github.com/pingcap/failpoint" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/sorter" - "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/util/testleak" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - "golang.org/x/sync/errgroup" -) - -const ( - numProducers = 16 -) - -type sorterSuite struct{} - -var _ = check.SerialSuites(&sorterSuite{}) - -func Test(t *testing.T) { check.TestingT(t) } - -func generateMockRawKV(ts uint64) *model.RawKVEntry { - return &model.RawKVEntry{ - OpType: model.OpTypePut, - Key: []byte{}, - Value: []byte{}, - OldValue: nil, - StartTs: ts - 5, - CRTs: ts, - RegionID: 0, - } -} - -func (s *sorterSuite) TestSorterBasic(c *check.C) { - defer testleak.AfterTest(c)() - defer CleanUp() - - conf := config.GetDefaultServerConfig() - conf.DataDir = c.MkDir() - sortDir := filepath.Join(conf.DataDir, config.DefaultSortDir) - conf.Sorter = &config.SorterConfig{ - NumConcurrentWorker: 8, - ChunkSizeLimit: 1 * 1024 * 1024 * 1024, - MaxMemoryPressure: 60, - MaxMemoryConsumption: 16 * 1024 * 1024 * 1024, - NumWorkerPoolGoroutine: 4, - SortDir: sortDir, - } - config.StoreGlobalServerConfig(conf) - - err := os.MkdirAll(conf.Sorter.SortDir, 0o755) - c.Assert(err, check.IsNil) - sorter, err := NewUnifiedSorter(conf.Sorter.SortDir, "test-cf", "test", 0, "0.0.0.0:0") - c.Assert(err, check.IsNil) - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - err = testSorter(ctx, c, sorter, 10000) - c.Assert(err, check.ErrorMatches, ".*context cancel.*") -} - -func (s *sorterSuite) TestSorterCancel(c *check.C) { - defer testleak.AfterTest(c)() - defer CleanUp() - - conf := config.GetDefaultServerConfig() - conf.DataDir = c.MkDir() - sortDir := filepath.Join(conf.DataDir, config.DefaultSortDir) - conf.Sorter = &config.SorterConfig{ - NumConcurrentWorker: 8, - ChunkSizeLimit: 1 * 1024 * 1024 * 1024, - MaxMemoryPressure: 60, - MaxMemoryConsumption: 0, - NumWorkerPoolGoroutine: 4, - SortDir: sortDir, - } - config.StoreGlobalServerConfig(conf) - - err := os.MkdirAll(conf.Sorter.SortDir, 0o755) - c.Assert(err, check.IsNil) - sorter, err := NewUnifiedSorter(conf.Sorter.SortDir, "test-cf", "test", 0, "0.0.0.0:0") - c.Assert(err, check.IsNil) - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - finishedCh := make(chan struct{}) - go func() { - err := testSorter(ctx, c, sorter, 10000000) - c.Assert(err, check.ErrorMatches, ".*context deadline exceeded.*") - close(finishedCh) - }() - - after := time.After(30 * time.Second) - select { - case <-after: - c.Fatal("TestSorterCancel timed out") - case <-finishedCh: - } - - log.Info("Sorter successfully cancelled") -} - -func testSorter(ctx context.Context, c *check.C, sorter sorter.EventSorter, count int) error { - err := failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/sorterDebug", "return(true)") - if err != nil { - log.Panic("Could not enable failpoint", zap.Error(err)) - } - - c.Assert(failpoint.Enable("github.com/tikv/migration/cdc/pkg/util/InjectCheckDataDirSatisfied", ""), check.IsNil) - defer func() { - c.Assert(failpoint.Disable("github.com/tikv/migration/cdc/pkg/util/InjectCheckDataDirSatisfied"), check.IsNil) - }() - - ctx, cancel := context.WithCancel(ctx) - errg, ctx := errgroup.WithContext(ctx) - errg.Go(func() error { - return sorter.Run(ctx) - }) - errg.Go(func() error { - return RunWorkerPool(ctx) - }) - - producerProgress := make([]uint64, numProducers) - - // launch the producers - for i := 0; i < numProducers; i++ { - finalI := i - errg.Go(func() error { - for j := 1; j <= count; j++ { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - sorter.AddEntry(ctx, model.NewPolymorphicEvent(generateMockRawKV(uint64(j)<<5))) - if j%10000 == 0 { - atomic.StoreUint64(&producerProgress[finalI], uint64(j)<<5) - } - } - sorter.AddEntry(ctx, model.NewPolymorphicEvent(generateMockRawKV(uint64(count+1)<<5))) - atomic.StoreUint64(&producerProgress[finalI], uint64(count+1)<<5) - return nil - }) - } - - // launch the resolver - errg.Go(func() error { - ticker := time.NewTicker(1 * time.Second) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return ctx.Err() - case <-ticker.C: - resolvedTs := uint64(math.MaxUint64) - for i := range producerProgress { - ts := atomic.LoadUint64(&producerProgress[i]) - if resolvedTs > ts { - resolvedTs = ts - } - } - sorter.AddEntry(ctx, model.NewResolvedPolymorphicEvent(0, resolvedTs)) - if resolvedTs == uint64(count)<<5 { - return nil - } - } - } - }) - - // launch the consumer - errg.Go(func() error { - counter := 0 - lastTs := uint64(0) - ticker := time.NewTicker(1 * time.Second) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return ctx.Err() - case event := <-sorter.Output(): - if event.RawKV.OpType != model.OpTypeResolved { - if event.CRTs < lastTs { - panic("regressed") - } - lastTs = event.CRTs - counter += 1 - if counter%10000 == 0 { - log.Debug("Messages received", zap.Int("counter", counter)) - } - if counter >= numProducers*count { - log.Debug("Unified Sorter test successful") - cancel() - } - } - case <-ticker.C: - log.Debug("Consumer is alive") - } - } - }) - - return errg.Wait() -} - -func (s *sorterSuite) TestSortDirConfigChangeFeed(c *check.C) { - defer testleak.AfterTest(c)() - defer CleanUp() - - poolMu.Lock() - // Clean up the back-end pool if one has been created - pool = nil - poolMu.Unlock() - - dir := c.MkDir() - // We expect the changefeed setting to take effect - config.GetGlobalServerConfig().Sorter.SortDir = "" - - _, err := NewUnifiedSorter(dir, /* the changefeed setting */ - "test-cf", - "test", - 0, - "0.0.0.0:0") - c.Assert(err, check.IsNil) - - poolMu.Lock() - defer poolMu.Unlock() - - c.Assert(pool, check.NotNil) - c.Assert(pool.dir, check.Equals, dir) -} - -// TestSorterCancelRestart tests the situation where the Unified Sorter is repeatedly canceled and -// restarted. There should not be any problem, especially file corruptions. -func (s *sorterSuite) TestSorterCancelRestart(c *check.C) { - defer testleak.AfterTest(c)() - defer CleanUp() - - conf := config.GetDefaultServerConfig() - conf.DataDir = c.MkDir() - sortDir := filepath.Join(conf.DataDir, config.DefaultSortDir) - conf.Sorter = &config.SorterConfig{ - NumConcurrentWorker: 8, - ChunkSizeLimit: 1 * 1024 * 1024 * 1024, - MaxMemoryPressure: 0, // disable memory sort - MaxMemoryConsumption: 0, - NumWorkerPoolGoroutine: 4, - SortDir: sortDir, - } - config.StoreGlobalServerConfig(conf) - - err := os.MkdirAll(conf.Sorter.SortDir, 0o755) - c.Assert(err, check.IsNil) - - // enable the failpoint to simulate delays - err = failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/asyncFlushStartDelay", "sleep(100)") - c.Assert(err, check.IsNil) - defer func() { - _ = failpoint.Disable("github.com/tikv/migration/cdc/cdc/sorter/unified/asyncFlushStartDelay") - }() - - // enable the failpoint to simulate delays - err = failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/asyncFlushInProcessDelay", "1%sleep(1)") - c.Assert(err, check.IsNil) - defer func() { - _ = failpoint.Disable("github.com/tikv/migration/cdc/cdc/sorter/unified/asyncFlushInProcessDelay") - }() - - for i := 0; i < 5; i++ { - sorter, err := NewUnifiedSorter(conf.Sorter.SortDir, "test-cf", "test", 0, "0.0.0.0:0") - c.Assert(err, check.IsNil) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - err = testSorter(ctx, c, sorter, 100000000) - c.Assert(err, check.ErrorMatches, ".*context deadline exceeded.*") - cancel() - } -} - -func (s *sorterSuite) TestSorterIOError(c *check.C) { - defer testleak.AfterTest(c)() - defer CleanUp() - - conf := config.GetDefaultServerConfig() - conf.DataDir = c.MkDir() - sortDir := filepath.Join(conf.DataDir, config.DefaultSortDir) - conf.Sorter = &config.SorterConfig{ - NumConcurrentWorker: 8, - ChunkSizeLimit: 1 * 1024 * 1024 * 1024, - MaxMemoryPressure: 60, - MaxMemoryConsumption: 0, - NumWorkerPoolGoroutine: 4, - SortDir: sortDir, - } - config.StoreGlobalServerConfig(conf) - - err := os.MkdirAll(conf.Sorter.SortDir, 0o755) - c.Assert(err, check.IsNil) - sorter, err := NewUnifiedSorter(conf.Sorter.SortDir, "test-cf", "test", 0, "0.0.0.0:0") - c.Assert(err, check.IsNil) - - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - - // enable the failpoint to simulate backEnd allocation error (usually would happen when creating a file) - err = failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/InjectErrorBackEndAlloc", "return(true)") - c.Assert(err, check.IsNil) - defer func() { - _ = failpoint.Disable("github.com/tikv/migration/cdc/cdc/sorter/unified/InjectErrorBackEndAlloc") - }() - - finishedCh := make(chan struct{}) - go func() { - err := testSorter(ctx, c, sorter, 10000) - c.Assert(err, check.ErrorMatches, ".*injected alloc error.*") - close(finishedCh) - }() - - after := time.After(60 * time.Second) - select { - case <-after: - c.Fatal("TestSorterIOError timed out") - case <-finishedCh: - } - - CleanUp() - _ = failpoint.Disable("github.com/tikv/migration/cdc/cdc/sorter/unified/InjectErrorBackEndAlloc") - // enable the failpoint to simulate backEnd write error (usually would happen when writing to a file) - err = failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/InjectErrorBackEndWrite", "return(true)") - c.Assert(err, check.IsNil) - defer func() { - _ = failpoint.Disable("github.com/tikv/migration/cdc/cdc/sorter/unified/InjectErrorBackEndWrite") - }() - - // recreate the sorter - sorter, err = NewUnifiedSorter(conf.Sorter.SortDir, "test-cf", "test", 0, "0.0.0.0:0") - c.Assert(err, check.IsNil) - - finishedCh = make(chan struct{}) - go func() { - err := testSorter(ctx, c, sorter, 10000) - c.Assert(err, check.ErrorMatches, ".*injected write error.*") - close(finishedCh) - }() - - after = time.After(60 * time.Second) - select { - case <-after: - c.Fatal("TestSorterIOError timed out") - case <-finishedCh: - } -} - -func (s *sorterSuite) TestSorterErrorReportCorrect(c *check.C) { - defer testleak.AfterTest(c)() - defer CleanUp() - - log.SetLevel(zapcore.DebugLevel) - defer log.SetLevel(zapcore.InfoLevel) - - conf := config.GetDefaultServerConfig() - conf.DataDir = c.MkDir() - sortDir := filepath.Join(conf.DataDir, config.DefaultSortDir) - conf.Sorter = &config.SorterConfig{ - NumConcurrentWorker: 8, - ChunkSizeLimit: 1 * 1024 * 1024 * 1024, - MaxMemoryPressure: 60, - MaxMemoryConsumption: 0, - NumWorkerPoolGoroutine: 4, - SortDir: sortDir, - } - config.StoreGlobalServerConfig(conf) - - err := os.MkdirAll(conf.Sorter.SortDir, 0o755) - c.Assert(err, check.IsNil) - sorter, err := NewUnifiedSorter(conf.Sorter.SortDir, "test-cf", "test", 0, "0.0.0.0:0") - c.Assert(err, check.IsNil) - - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - - // enable the failpoint to simulate backEnd allocation error (usually would happen when creating a file) - err = failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/InjectHeapSorterExitDelay", "sleep(2000)") - c.Assert(err, check.IsNil) - defer func() { - _ = failpoint.Disable("github.com/tikv/migration/cdc/cdc/sorter/unified/InjectHeapSorterExitDelay") - }() - - err = failpoint.Enable("github.com/tikv/migration/cdc/cdc/sorter/unified/InjectErrorBackEndAlloc", "return(true)") - c.Assert(err, check.IsNil) - defer func() { - _ = failpoint.Disable("github.com/tikv/migration/cdc/cdc/sorter/unified/InjectErrorBackEndAlloc") - }() - - finishedCh := make(chan struct{}) - go func() { - err := testSorter(ctx, c, sorter, 10000) - c.Assert(err, check.ErrorMatches, ".*injected alloc error.*") - close(finishedCh) - }() - - after := time.After(60 * time.Second) - select { - case <-after: - c.Fatal("TestSorterIOError timed out") - case <-finishedCh: - } -} - -func (s *sorterSuite) TestSortClosedAddEntry(c *check.C) { - defer testleak.AfterTest(c)() - defer CleanUp() - - sorter, err := NewUnifiedSorter(c.MkDir(), - "test-cf", - "test", - 0, - "0.0.0.0:0") - c.Assert(err, check.IsNil) - - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*100) - defer cancel() - err = sorter.Run(ctx) - c.Assert(err, check.ErrorMatches, ".*deadline.*") - - ctx1, cancel1 := context.WithTimeout(context.Background(), time.Second*10) - defer cancel1() - for i := 0; i < 10000; i++ { - sorter.AddEntry(ctx1, model.NewPolymorphicEvent(generateMockRawKV(uint64(i)))) - } - - select { - case <-ctx1.Done(): - c.Fatal("TestSortClosedAddEntry timed out") - default: - } - cancel1() -} - -func (s *sorterSuite) TestUnifiedSorterFileLockConflict(c *check.C) { - defer testleak.AfterTest(c)() - defer CleanUp() - - dir := c.MkDir() - captureAddr := "0.0.0.0:0" - _, err := newBackEndPool(dir, captureAddr) - c.Assert(err, check.IsNil) - - // GlobalServerConfig overrides dir parameter in NewUnifiedSorter. - config.GetGlobalServerConfig().Sorter.SortDir = dir - _, err = NewUnifiedSorter(dir, "test-cf", "test", 0, captureAddr) - c.Assert(err, check.ErrorMatches, ".*file lock conflict.*") -} diff --git a/cdc/cdc/sorter/unified/unified_sorter.go b/cdc/cdc/sorter/unified/unified_sorter.go deleted file mode 100644 index 5419ace5..00000000 --- a/cdc/cdc/sorter/unified/unified_sorter.go +++ /dev/null @@ -1,285 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package unified - -import ( - "context" - "sync" - - "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/config" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/util" - "golang.org/x/sync/errgroup" -) - -const ( - inputChSize = 128 - outputChSize = 128 - heapCollectChSize = 128 // this should be not be too small, to guarantee IO concurrency -) - -// Sorter provides both sorting in memory and in file. Memory pressure is used to determine which one to use. -type Sorter struct { - inputCh chan *model.PolymorphicEvent - outputCh chan *model.PolymorphicEvent - dir string - metricsInfo *metricsInfo - - closeCh chan struct{} -} - -type metricsInfo struct { - changeFeedID model.ChangeFeedID - tableName string - tableID model.TableID - captureAddr string -} - -type ctxKey struct{} - -// NewUnifiedSorter creates a new Sorter -func NewUnifiedSorter( - dir string, - changeFeedID model.ChangeFeedID, - tableName string, - tableID model.TableID, - captureAddr string) (*Sorter, error) { - poolMu.Lock() - defer poolMu.Unlock() - - if pool == nil { - var err error - pool, err = newBackEndPool(dir, captureAddr) - if err != nil { - return nil, errors.Trace(err) - } - } - - lazyInitWorkerPool() - return &Sorter{ - inputCh: make(chan *model.PolymorphicEvent, inputChSize), - outputCh: make(chan *model.PolymorphicEvent, outputChSize), - dir: dir, - metricsInfo: &metricsInfo{ - changeFeedID: changeFeedID, - tableName: tableName, - tableID: tableID, - captureAddr: captureAddr, - }, - closeCh: make(chan struct{}, 1), - }, nil -} - -// CleanUp cleans up the files that might have been used. -func CleanUp() { - poolMu.Lock() - defer poolMu.Unlock() - - if pool != nil { - log.Info("Unified Sorter: starting cleaning up files") - pool.terminate() - pool = nil - } -} - -// ResetGlobalPoolWithoutCleanup reset the pool without cleaning up files. -// Note that it is used in tests only. -func ResetGlobalPoolWithoutCleanup() { - poolMu.Lock() - defer poolMu.Unlock() - - pool = nil -} - -// Run implements the EventSorter interface -func (s *Sorter) Run(ctx context.Context) error { - failpoint.Inject("sorterDebug", func() { - log.Info("sorterDebug: Running Unified Sorter in debug mode") - }) - - defer close(s.closeCh) - - finish := util.MonitorCancelLatency(ctx, "Unified Sorter") - defer finish() - - ctx = context.WithValue(ctx, ctxKey{}, s) - ctx = util.PutCaptureAddrInCtx(ctx, s.metricsInfo.captureAddr) - ctx = util.PutChangefeedIDInCtx(ctx, s.metricsInfo.changeFeedID) - ctx = util.PutTableInfoInCtx(ctx, s.metricsInfo.tableID, s.metricsInfo.tableName) - - sorterConfig := config.GetGlobalServerConfig().Sorter - numConcurrentHeaps := sorterConfig.NumConcurrentWorker - - errg, subctx := errgroup.WithContext(ctx) - heapSorterCollectCh := make(chan *flushTask, heapCollectChSize) - // mergerCleanUp will consumer the remaining elements in heapSorterCollectCh to prevent any FD leak. - defer mergerCleanUp(heapSorterCollectCh) - - heapSorterErrCh := make(chan error, 1) - defer close(heapSorterErrCh) - heapSorterErrOnce := &sync.Once{} - heapSorters := make([]*heapSorter, sorterConfig.NumConcurrentWorker) - for i := range heapSorters { - heapSorters[i] = newHeapSorter(i, heapSorterCollectCh) - heapSorters[i].init(subctx, func(err error) { - heapSorterErrOnce.Do(func() { - heapSorterErrCh <- err - }) - }) - } - - ioCancelFunc := func() { - for _, heapSorter := range heapSorters { - // cancels async IO operations - heapSorter.canceller.Cancel() - } - } - - errg.Go(func() error { - defer func() { - // cancelling the heapSorters from the outside - for _, hs := range heapSorters { - hs.poolHandle.Unregister() - } - // must wait for all writers to exit to close the channel. - close(heapSorterCollectCh) - failpoint.Inject("InjectHeapSorterExitDelay", func() {}) - }() - - select { - case <-subctx.Done(): - return errors.Trace(subctx.Err()) - case err := <-heapSorterErrCh: - return errors.Trace(err) - } - }) - - errg.Go(func() error { - return printError(runMerger(subctx, numConcurrentHeaps, heapSorterCollectCh, s.outputCh, ioCancelFunc)) - }) - - errg.Go(func() error { - captureAddr := util.CaptureAddrFromCtx(ctx) - changefeedID := util.ChangefeedIDFromCtx(ctx) - - metricSorterConsumeCount := sorterConsumeCount.MustCurryWith(map[string]string{ - "capture": captureAddr, - "changefeed": changefeedID, - }) - - nextSorterID := 0 - for { - select { - case <-subctx.Done(): - return subctx.Err() - case event := <-s.inputCh: - if event.RawKV != nil && event.RawKV.OpType == model.OpTypeResolved { - // broadcast resolved events - for _, sorter := range heapSorters { - select { - case <-subctx.Done(): - return subctx.Err() - default: - } - err := sorter.poolHandle.AddEvent(subctx, event) - if cerror.ErrWorkerPoolHandleCancelled.Equal(err) { - // no need to report ErrWorkerPoolHandleCancelled, - // as it may confuse the user - return nil - } - if err != nil { - return errors.Trace(err) - } - metricSorterConsumeCount.WithLabelValues("resolved").Inc() - } - continue - } - - // dispatch a row changed event - targetID := nextSorterID % numConcurrentHeaps - nextSorterID++ - select { - case <-subctx.Done(): - return subctx.Err() - default: - err := heapSorters[targetID].poolHandle.AddEvent(subctx, event) - if err != nil { - if cerror.ErrWorkerPoolHandleCancelled.Equal(err) { - // no need to report ErrWorkerPoolHandleCancelled, - // as it may confuse the user - return nil - } - return errors.Trace(err) - } - metricSorterConsumeCount.WithLabelValues("kv").Inc() - } - } - } - }) - - return printError(errg.Wait()) -} - -// AddEntry implements the EventSorter interface -func (s *Sorter) AddEntry(ctx context.Context, entry *model.PolymorphicEvent) { - select { - case <-ctx.Done(): - return - case <-s.closeCh: - case s.inputCh <- entry: - } -} - -// TryAddEntry implements the EventSorter interface -func (s *Sorter) TryAddEntry(ctx context.Context, entry *model.PolymorphicEvent) (bool, error) { - // add two select to guarantee the done/close condition is checked first. - select { - case <-ctx.Done(): - return false, ctx.Err() - case <-s.closeCh: - return false, cerror.ErrSorterClosed.GenWithStackByArgs() - default: - } - select { - case s.inputCh <- entry: - return true, nil - default: - return false, nil - } -} - -// Output implements the EventSorter interface -func (s *Sorter) Output() <-chan *model.PolymorphicEvent { - return s.outputCh -} - -// RunWorkerPool runs the worker pool used by the heapSorters -// It **must** be running for Unified Sorter to work. -func RunWorkerPool(ctx context.Context) error { - lazyInitWorkerPool() - errg, ctx := errgroup.WithContext(ctx) - errg.Go(func() error { - return errors.Trace(heapSorterPool.Run(ctx)) - }) - - errg.Go(func() error { - return errors.Trace(heapSorterIOPool.Run(ctx)) - }) - - return errors.Trace(errg.Wait()) -} diff --git a/cdc/cdc/sorter/unified/unified_sorter_test.go b/cdc/cdc/sorter/unified/unified_sorter_test.go deleted file mode 100644 index c050096f..00000000 --- a/cdc/cdc/sorter/unified/unified_sorter_test.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package unified - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - cerror "github.com/tikv/migration/cdc/pkg/errors" -) - -func TestUnifiedSorterTryAddEntry(t *testing.T) { - t.Parallel() - - events := []*model.PolymorphicEvent{ - model.NewPolymorphicEvent(&model.RawKVEntry{OpType: model.OpTypePut, StartTs: 1, CRTs: 0, RegionID: 0}), - model.NewResolvedPolymorphicEvent(0, 1), - } - for _, event := range events { - s := &Sorter{inputCh: make(chan *model.PolymorphicEvent, 2), closeCh: make(chan struct{}, 2)} - added, err := s.TryAddEntry(context.TODO(), event) - require.True(t, added) - require.Nil(t, err) - added, err = s.TryAddEntry(context.TODO(), event) - require.True(t, added) - require.Nil(t, err) - added, err = s.TryAddEntry(context.TODO(), event) - require.False(t, added) - require.Nil(t, err) - <-s.inputCh - added, err = s.TryAddEntry(context.TODO(), event) - require.True(t, added) - require.Nil(t, err) - <-s.inputCh - ctx, cancel := context.WithCancel(context.TODO()) - cancel() - added, err = s.TryAddEntry(ctx, event) - require.False(t, added) - require.False(t, cerror.ErrSorterClosed.Equal(err)) - <-s.inputCh - s.closeCh <- struct{}{} - added, err = s.TryAddEntry(context.TODO(), event) - require.False(t, added) - require.True(t, cerror.ErrSorterClosed.Equal(err)) - } -} diff --git a/cdc/cmd/kafka-consumer/main.go b/cdc/cmd/kafka-consumer/main.go deleted file mode 100644 index 6122dd78..00000000 --- a/cdc/cmd/kafka-consumer/main.go +++ /dev/null @@ -1,632 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "flag" - "fmt" - "math" - "net/url" - "os" - "os/signal" - "strconv" - "strings" - "sync" - "sync/atomic" - "syscall" - "time" - - "github.com/Shopify/sarama" - "github.com/google/uuid" - "github.com/pingcap/errors" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/sink" - "github.com/tikv/migration/cdc/cdc/sink/codec" - "github.com/tikv/migration/cdc/pkg/config" - cdcfilter "github.com/tikv/migration/cdc/pkg/filter" - "github.com/tikv/migration/cdc/pkg/logutil" - "github.com/tikv/migration/cdc/pkg/quotes" - "github.com/tikv/migration/cdc/pkg/security" - "github.com/tikv/migration/cdc/pkg/util" - "go.uber.org/zap" -) - -// Sarama configuration options -var ( - kafkaAddrs []string - kafkaTopic string - kafkaPartitionNum int32 - kafkaGroupID = fmt.Sprintf("ticdc_kafka_consumer_%s", uuid.New().String()) - kafkaVersion = "2.4.0" - kafkaMaxMessageBytes = math.MaxInt64 - kafkaMaxBatchSize = math.MaxInt64 - - downstreamURIStr string - - logPath string - logLevel string - timezone string - ca, cert, key string -) - -func init() { - var upstreamURIStr string - - flag.StringVar(&upstreamURIStr, "upstream-uri", "", "Kafka uri") - flag.StringVar(&downstreamURIStr, "downstream-uri", "", "downstream sink uri") - flag.StringVar(&logPath, "log-file", "cdc_kafka_consumer.log", "log file path") - flag.StringVar(&logLevel, "log-level", "info", "log file path") - flag.StringVar(&timezone, "tz", "System", "Specify time zone of Kafka consumer") - flag.StringVar(&ca, "ca", "", "CA certificate path for Kafka SSL connection") - flag.StringVar(&cert, "cert", "", "Certificate path for Kafka SSL connection") - flag.StringVar(&key, "key", "", "Private key path for Kafka SSL connection") - flag.Parse() - - err := logutil.InitLogger(&logutil.Config{ - Level: logLevel, - File: logPath, - }) - if err != nil { - log.Fatal("init logger failed", zap.Error(err)) - } - - upstreamURI, err := url.Parse(upstreamURIStr) - if err != nil { - log.Fatal("invalid upstream-uri", zap.Error(err)) - } - scheme := strings.ToLower(upstreamURI.Scheme) - if scheme != "kafka" { - log.Fatal("invalid upstream-uri scheme, the scheme of upstream-uri must be `kafka`", zap.String("upstream-uri", upstreamURIStr)) - } - s := upstreamURI.Query().Get("version") - if s != "" { - kafkaVersion = s - } - s = upstreamURI.Query().Get("consumer-group-id") - if s != "" { - kafkaGroupID = s - } - kafkaTopic = strings.TrimFunc(upstreamURI.Path, func(r rune) bool { - return r == '/' - }) - kafkaAddrs = strings.Split(upstreamURI.Host, ",") - - config, err := newSaramaConfig() - if err != nil { - log.Fatal("Error creating sarama config", zap.Error(err)) - } - - s = upstreamURI.Query().Get("partition-num") - if s == "" { - partition, err := getPartitionNum(kafkaAddrs, kafkaTopic, config) - if err != nil { - log.Fatal("can not get partition number", zap.String("topic", kafkaTopic), zap.Error(err)) - } - kafkaPartitionNum = partition - } else { - c, err := strconv.ParseInt(s, 10, 32) - if err != nil { - log.Fatal("invalid partition-num of upstream-uri") - } - kafkaPartitionNum = int32(c) - } - - s = upstreamURI.Query().Get("max-message-bytes") - if s != "" { - c, err := strconv.Atoi(s) - if err != nil { - log.Fatal("invalid max-message-bytes of upstream-uri") - } - log.Info("Setting max-message-bytes", zap.Int("max-message-bytes", c)) - kafkaMaxMessageBytes = c - } - - s = upstreamURI.Query().Get("max-batch-size") - if s != "" { - c, err := strconv.Atoi(s) - if err != nil { - log.Fatal("invalid max-batch-size of upstream-uri") - } - log.Info("Setting max-batch-size", zap.Int("max-batch-size", c)) - kafkaMaxBatchSize = c - } -} - -func getPartitionNum(address []string, topic string, cfg *sarama.Config) (int32, error) { - // get partition number or create topic automatically - admin, err := sarama.NewClusterAdmin(address, cfg) - if err != nil { - return 0, errors.Trace(err) - } - topics, err := admin.ListTopics() - if err != nil { - return 0, errors.Trace(err) - } - err = admin.Close() - if err != nil { - return 0, errors.Trace(err) - } - topicDetail, exist := topics[topic] - if !exist { - return 0, errors.Errorf("can not find topic %s", topic) - } - log.Info("get partition number of topic", zap.String("topic", topic), zap.Int32("partition_num", topicDetail.NumPartitions)) - return topicDetail.NumPartitions, nil -} - -func waitTopicCreated(address []string, topic string, cfg *sarama.Config) error { - admin, err := sarama.NewClusterAdmin(address, cfg) - if err != nil { - return errors.Trace(err) - } - defer admin.Close() - for i := 0; i <= 30; i++ { - topics, err := admin.ListTopics() - if err != nil { - return errors.Trace(err) - } - if _, ok := topics[topic]; ok { - return nil - } - log.Info("wait the topic created", zap.String("topic", topic)) - time.Sleep(1 * time.Second) - } - return errors.Errorf("wait the topic(%s) created timeout", topic) -} - -func newSaramaConfig() (*sarama.Config, error) { - config := sarama.NewConfig() - - version, err := sarama.ParseKafkaVersion(kafkaVersion) - if err != nil { - return nil, errors.Trace(err) - } - - config.ClientID = "ticdc_kafka_sarama_consumer" - config.Version = version - - config.Metadata.Retry.Max = 10000 - config.Metadata.Retry.Backoff = 500 * time.Millisecond - config.Consumer.Retry.Backoff = 500 * time.Millisecond - config.Consumer.Offsets.Initial = sarama.OffsetOldest - - if len(ca) != 0 { - config.Net.TLS.Enable = true - config.Net.TLS.Config, err = (&security.Credential{ - CAPath: ca, - CertPath: cert, - KeyPath: key, - }).ToTLSConfig() - if err != nil { - return nil, errors.Trace(err) - } - } - - return config, err -} - -func main() { - log.Info("Starting a new TiCDC open protocol consumer") - - /** - * Construct a new Sarama configuration. - * The Kafka cluster version has to be defined before the consumer/producer is initialized. - */ - config, err := newSaramaConfig() - if err != nil { - log.Fatal("Error creating sarama config", zap.Error(err)) - } - err = waitTopicCreated(kafkaAddrs, kafkaTopic, config) - if err != nil { - log.Fatal("wait topic created failed", zap.Error(err)) - } - /** - * Setup a new Sarama consumer group - */ - consumer, err := NewConsumer(context.TODO()) - if err != nil { - log.Fatal("Error creating consumer", zap.Error(err)) - } - - ctx, cancel := context.WithCancel(context.Background()) - client, err := sarama.NewConsumerGroup(kafkaAddrs, kafkaGroupID, config) - if err != nil { - log.Fatal("Error creating consumer group client", zap.Error(err)) - } - - wg := &sync.WaitGroup{} - wg.Add(1) - go func() { - defer wg.Done() - for { - // `Consume` should be called inside an infinite loop, when a - // server-side rebalance happens, the consumer session will need to be - // recreated to get the new claims - if err := client.Consume(ctx, strings.Split(kafkaTopic, ","), consumer); err != nil { - log.Fatal("Error from consumer: %v", zap.Error(err)) - } - // check if context was cancelled, signaling that the consumer should stop - if ctx.Err() != nil { - return - } - consumer.ready = make(chan bool) - } - }() - - go func() { - if err := consumer.Run(ctx); err != nil { - log.Fatal("Error running consumer: %v", zap.Error(err)) - } - }() - - <-consumer.ready // Await till the consumer has been set up - log.Info("TiCDC open protocol consumer up and running!...") - - sigterm := make(chan os.Signal, 1) - signal.Notify(sigterm, syscall.SIGINT, syscall.SIGTERM) - select { - case <-ctx.Done(): - log.Info("terminating: context cancelled") - case <-sigterm: - log.Info("terminating: via signal") - } - cancel() - wg.Wait() - if err = client.Close(); err != nil { - log.Fatal("Error closing client", zap.Error(err)) - } -} - -type partitionSink struct { - sink.Sink - resolvedTs uint64 - partitionNo int - tablesMap sync.Map -} - -// Consumer represents a Sarama consumer group consumer -type Consumer struct { - ready chan bool - - ddlList []*model.DDLEvent - maxDDLReceivedTs uint64 - ddlListMu sync.Mutex - - sinks []*partitionSink - sinksMu sync.Mutex - - ddlSink sink.Sink - fakeTableIDGenerator *fakeTableIDGenerator - - globalResolvedTs uint64 -} - -// NewConsumer creates a new cdc kafka consumer -func NewConsumer(ctx context.Context) (*Consumer, error) { - // TODO support filter in downstream sink - tz, err := util.GetTimezone(timezone) - if err != nil { - return nil, errors.Annotate(err, "can not load timezone") - } - ctx = util.PutTimezoneInCtx(ctx, tz) - filter, err := cdcfilter.NewFilter(config.GetDefaultReplicaConfig()) - if err != nil { - return nil, errors.Trace(err) - } - c := new(Consumer) - c.fakeTableIDGenerator = &fakeTableIDGenerator{ - tableIDs: make(map[string]int64), - } - c.sinks = make([]*partitionSink, kafkaPartitionNum) - ctx, cancel := context.WithCancel(ctx) - errCh := make(chan error, 1) - opts := map[string]string{} - for i := 0; i < int(kafkaPartitionNum); i++ { - s, err := sink.New(ctx, "kafka-consumer", downstreamURIStr, filter, config.GetDefaultReplicaConfig(), opts, errCh) - if err != nil { - cancel() - return nil, errors.Trace(err) - } - c.sinks[i] = &partitionSink{Sink: s, partitionNo: i} - } - sink, err := sink.New(ctx, "kafka-consumer", downstreamURIStr, filter, config.GetDefaultReplicaConfig(), opts, errCh) - if err != nil { - cancel() - return nil, errors.Trace(err) - } - go func() { - err := <-errCh - if errors.Cause(err) != context.Canceled { - log.Error("error on running consumer", zap.Error(err)) - } else { - log.Info("consumer exited") - } - cancel() - }() - c.ddlSink = sink - c.ready = make(chan bool) - return c, nil -} - -// Setup is run at the beginning of a new session, before ConsumeClaim -func (c *Consumer) Setup(sarama.ConsumerGroupSession) error { - // Mark the c as ready - close(c.ready) - return nil -} - -// Cleanup is run at the end of a session, once all ConsumeClaim goroutines have exited -func (c *Consumer) Cleanup(sarama.ConsumerGroupSession) error { - return nil -} - -// ConsumeClaim must start a consumer loop of ConsumerGroupClaim's Messages(). -func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { - ctx := context.TODO() - partition := claim.Partition() - c.sinksMu.Lock() - sink := c.sinks[partition] - c.sinksMu.Unlock() - if sink == nil { - panic("sink should initialized") - } -ClaimMessages: - for message := range claim.Messages() { - log.Info("Message claimed", zap.Int32("partition", message.Partition), zap.ByteString("key", message.Key), zap.ByteString("value", message.Value)) - batchDecoder, err := codec.NewJSONEventBatchDecoder(message.Key, message.Value) - if err != nil { - return errors.Trace(err) - } - - counter := 0 - for { - tp, hasNext, err := batchDecoder.HasNext() - if err != nil { - log.Fatal("decode message key failed", zap.Error(err)) - } - if !hasNext { - break - } - - counter++ - // If the message containing only one event exceeds the length limit, CDC will allow it and issue a warning. - if len(message.Key)+len(message.Value) > kafkaMaxMessageBytes && counter > 1 { - log.Fatal("kafka max-messages-bytes exceeded", zap.Int("max-message-bytes", kafkaMaxMessageBytes), - zap.Int("recevied-bytes", len(message.Key)+len(message.Value))) - } - - switch tp { - case model.MqMessageTypeDDL: - ddl, err := batchDecoder.NextDDLEvent() - if err != nil { - log.Fatal("decode message value failed", zap.ByteString("value", message.Value)) - } - c.appendDDL(ddl) - case model.MqMessageTypeRow: - row, err := batchDecoder.NextRowChangedEvent() - if err != nil { - log.Fatal("decode message value failed", zap.ByteString("value", message.Value)) - } - globalResolvedTs := atomic.LoadUint64(&c.globalResolvedTs) - if row.CommitTs <= globalResolvedTs || row.CommitTs <= sink.resolvedTs { - log.Debug("filter fallback row", zap.ByteString("row", message.Key), - zap.Uint64("globalResolvedTs", globalResolvedTs), - zap.Uint64("sinkResolvedTs", sink.resolvedTs), - zap.Int32("partition", partition)) - break ClaimMessages - } - // FIXME: hack to set start-ts in row changed event, as start-ts - // is not contained in TiCDC open protocol - row.StartTs = row.CommitTs - var partitionID int64 - if row.Table.IsPartition { - partitionID = row.Table.TableID - } - row.Table.TableID = - c.fakeTableIDGenerator.generateFakeTableID(row.Table.Schema, row.Table.Table, partitionID) - err = sink.EmitRowChangedEvents(ctx, row) - if err != nil { - log.Fatal("emit row changed event failed", zap.Error(err)) - } - log.Info("Emit RowChangedEvent", zap.Any("row", row)) - lastCommitTs, ok := sink.tablesMap.Load(row.Table.TableID) - if !ok || lastCommitTs.(uint64) < row.CommitTs { - sink.tablesMap.Store(row.Table.TableID, row.CommitTs) - } - case model.MqMessageTypeResolved: - ts, err := batchDecoder.NextResolvedEvent() - if err != nil { - log.Fatal("decode message value failed", zap.ByteString("value", message.Value)) - } - resolvedTs := atomic.LoadUint64(&sink.resolvedTs) - if resolvedTs < ts { - log.Debug("update sink resolved ts", - zap.Uint64("ts", ts), - zap.Int32("partition", partition)) - atomic.StoreUint64(&sink.resolvedTs, ts) - } - } - session.MarkMessage(message, "") - } - - if counter > kafkaMaxBatchSize { - log.Fatal("Open Protocol max-batch-size exceeded", zap.Int("max-batch-size", kafkaMaxBatchSize), - zap.Int("actual-batch-size", counter)) - } - } - - return nil -} - -func (c *Consumer) appendDDL(ddl *model.DDLEvent) { - c.ddlListMu.Lock() - defer c.ddlListMu.Unlock() - if ddl.CommitTs <= c.maxDDLReceivedTs { - return - } - globalResolvedTs := atomic.LoadUint64(&c.globalResolvedTs) - if ddl.CommitTs <= globalResolvedTs { - log.Error("unexpected ddl job", zap.Uint64("ddlts", ddl.CommitTs), zap.Uint64("globalResolvedTs", globalResolvedTs)) - return - } - c.ddlList = append(c.ddlList, ddl) - c.maxDDLReceivedTs = ddl.CommitTs -} - -func (c *Consumer) getFrontDDL() *model.DDLEvent { - c.ddlListMu.Lock() - defer c.ddlListMu.Unlock() - if len(c.ddlList) > 0 { - return c.ddlList[0] - } - return nil -} - -func (c *Consumer) popDDL() *model.DDLEvent { - c.ddlListMu.Lock() - defer c.ddlListMu.Unlock() - if len(c.ddlList) > 0 { - ddl := c.ddlList[0] - c.ddlList = c.ddlList[1:] - return ddl - } - return nil -} - -func (c *Consumer) forEachSink(fn func(sink *partitionSink) error) error { - c.sinksMu.Lock() - defer c.sinksMu.Unlock() - for _, sink := range c.sinks { - if err := fn(sink); err != nil { - return errors.Trace(err) - } - } - return nil -} - -// Run runs the Consumer -func (c *Consumer) Run(ctx context.Context) error { - var lastGlobalResolvedTs uint64 - for { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - time.Sleep(100 * time.Millisecond) - // handle ddl - globalResolvedTs := uint64(math.MaxUint64) - err := c.forEachSink(func(sink *partitionSink) error { - resolvedTs := atomic.LoadUint64(&sink.resolvedTs) - if resolvedTs < globalResolvedTs { - globalResolvedTs = resolvedTs - } - return nil - }) - if err != nil { - return errors.Trace(err) - } - todoDDL := c.getFrontDDL() - if todoDDL != nil && globalResolvedTs >= todoDDL.CommitTs { - // flush DMLs - err := c.forEachSink(func(sink *partitionSink) error { - return syncFlushRowChangedEvents(ctx, sink, todoDDL.CommitTs) - }) - if err != nil { - return errors.Trace(err) - } - - // execute ddl - err = c.ddlSink.EmitDDLEvent(ctx, todoDDL) - if err != nil { - return errors.Trace(err) - } - c.popDDL() - continue - } - - if todoDDL != nil && todoDDL.CommitTs < globalResolvedTs { - globalResolvedTs = todoDDL.CommitTs - } - if lastGlobalResolvedTs == globalResolvedTs { - continue - } - lastGlobalResolvedTs = globalResolvedTs - atomic.StoreUint64(&c.globalResolvedTs, globalResolvedTs) - log.Info("update globalResolvedTs", zap.Uint64("ts", globalResolvedTs)) - - err = c.forEachSink(func(sink *partitionSink) error { - return syncFlushRowChangedEvents(ctx, sink, globalResolvedTs) - }) - if err != nil { - return errors.Trace(err) - } - } -} - -func syncFlushRowChangedEvents(ctx context.Context, sink *partitionSink, resolvedTs uint64) error { - for { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - // tables are flushed - var ( - err error - checkpointTs uint64 - ) - flushedResolvedTs := true - sink.tablesMap.Range(func(key, value interface{}) bool { - tableID := key.(int64) - checkpointTs, err = sink.FlushRowChangedEvents(ctx, tableID, resolvedTs) - if err != nil { - return false - } - if checkpointTs < resolvedTs { - flushedResolvedTs = false - } - return true - }) - if err != nil { - return err - } - if flushedResolvedTs { - return nil - } - } -} - -type fakeTableIDGenerator struct { - tableIDs map[string]int64 - currentTableID int64 - mu sync.Mutex -} - -func (g *fakeTableIDGenerator) generateFakeTableID(schema, table string, partition int64) int64 { - g.mu.Lock() - defer g.mu.Unlock() - key := quotes.QuoteSchema(schema, table) - if partition != 0 { - key = fmt.Sprintf("%s.`%d`", key, partition) - } - if tableID, ok := g.tableIDs[key]; ok { - return tableID - } - g.currentTableID++ - g.tableIDs[key] = g.currentTableID - return g.currentTableID -} diff --git a/cdc/go.mod b/cdc/go.mod index 1e122157..5095ee27 100644 --- a/cdc/go.mod +++ b/cdc/go.mod @@ -4,19 +4,15 @@ go 1.16 require ( github.com/BurntSushi/toml v0.3.1 - github.com/DATA-DOG/go-sqlmock v1.5.0 + github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 // indirect github.com/Shopify/sarama v1.27.2 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 - github.com/apache/pulsar-client-go v0.6.0 - github.com/aws/aws-sdk-go v1.35.3 github.com/benbjohnson/clock v1.1.0 - github.com/bradleyjkemp/grpc-tools v0.2.5 github.com/cenkalti/backoff v2.2.1+incompatible github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e github.com/cockroachdb/pebble v0.0.0-20210719141320-8c3bd06debb5 github.com/coreos/go-semver v0.3.0 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect - github.com/davecgh/go-spew v1.1.1 github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 github.com/fatih/color v1.10.0 github.com/frankban/quicktest v1.11.1 // indirect @@ -26,7 +22,6 @@ require ( github.com/go-sql-driver/mysql v1.6.0 github.com/gogo/protobuf v1.3.2 github.com/golang-jwt/jwt v3.2.2+incompatible // indirect - github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.5.2 github.com/google/btree v1.0.0 github.com/google/go-cmp v0.5.6 @@ -35,14 +30,12 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc github.com/jarcoal/httpmock v1.0.5 - github.com/jmoiron/sqlx v1.3.3 github.com/json-iterator/go v1.1.12 // indirect - github.com/lib/pq v1.3.0 // indirect github.com/linkedin/goavro/v2 v2.9.8 github.com/mattn/go-colorable v0.1.11 // indirect github.com/mattn/go-shellwords v1.0.12 - github.com/mattn/go-sqlite3 v2.0.2+incompatible // indirect github.com/modern-go/reflect2 v1.0.2 + github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 github.com/philhofer/fwd v1.0.0 // indirect github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 @@ -54,9 +47,9 @@ require ( github.com/pingcap/tidb-tools v5.2.3-0.20211105044302-2dabb6641a6e+incompatible github.com/pingcap/tidb/parser v0.0.0-20220124083611-18fc286fbf0d github.com/prometheus/client_golang v1.7.1 - github.com/prometheus/client_model v0.2.0 github.com/r3labs/diff v1.1.0 github.com/soheilhy/cmux v0.1.5 + github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cobra v1.2.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 @@ -67,6 +60,7 @@ require ( github.com/tikv/client-go/v2 v2.0.0-rc.0.20211229051614-62d6b4a2e8f7 github.com/tikv/pd v1.1.0-beta.0.20211118054146-02848d2660ee github.com/tinylib/msgp v1.1.0 + github.com/twmb/murmur3 v1.1.3 github.com/uber-go/atomic v1.4.0 github.com/ugorji/go v1.2.6 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 @@ -75,7 +69,6 @@ require ( go.etcd.io/etcd v0.5.0-alpha.5.0.20210512015243-d19fbe541bf9 go.uber.org/atomic v1.9.0 go.uber.org/goleak v1.1.12 - go.uber.org/multierr v1.7.0 go.uber.org/zap v1.19.1 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect golang.org/x/net v0.0.0-20211020060615-d418f374d309 @@ -83,9 +76,7 @@ require ( golang.org/x/text v0.3.7 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba google.golang.org/grpc v1.40.0 - gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect sigs.k8s.io/yaml v1.2.0 // indirect - upper.io/db.v3 v3.7.1+incompatible ) replace ( diff --git a/cdc/go.sum b/cdc/go.sum index dc19fdcf..65b6e53f 100644 --- a/cdc/go.sum +++ b/cdc/go.sum @@ -45,11 +45,7 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 cloud.google.com/go/storage v1.16.1 h1:sMEIc4wxvoY3NXG7Rn9iP7jb/2buJgWR1vNXCR/UPfs= cloud.google.com/go/storage v1.16.1/go.mod h1:LaNorbty3ehnU3rEjXSNV/NRgQA0O8Y+uh6bPe5UOk4= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/99designs/keyring v1.1.5 h1:wLv7QyzYpFIyMSwOADq1CLTF9KbjbBfcnfmOGJ64aO4= -github.com/99designs/keyring v1.1.5/go.mod h1:7hsVvt2qXgtadGevGJ4ujg+u8m6SpJ5TpHqTozIPqf0= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/AthenZ/athenz v1.10.15 h1:8Bc2W313k/ev/SGokuthNbzpwfg9W3frg3PKq1r943I= -github.com/AthenZ/athenz v1.10.15/go.mod h1:7KMpEuJ9E4+vMCMI3UQJxwWs0RZtQq7YXZ1IteUjdsc= github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 h1:KQgdWmEOmaJKxaUUZwHAYh12t+b+ZJf8q3friycK1kA= github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0/go.mod h1:ZPW/Z0kLCTdDZaDbYTetxc9Cxl/2lNqxYHYNOF2bti0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0 h1:VBvHGLJbaY0+c66NZHdS9cgjHVYSH6DDa0XJMyrblsI= @@ -104,28 +100,19 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/pulsar-client-go v0.6.0 h1:yKX7NsmJxR5mL6uIUxTTatNhMFlhurTASSZRJ9IULDg= -github.com/apache/pulsar-client-go v0.6.0/go.mod h1:A1P5VjjljsFKAD13w7/jmU3Dly2gcRvcobiULqQXhz4= -github.com/apache/pulsar-client-go/oauth2 v0.0.0-20201120111947-b8bd55bc02bd h1:P5kM7jcXJ7TaftX0/EMKiSJgvQc/ct+Fw0KMvcH3WuY= -github.com/apache/pulsar-client-go/oauth2 v0.0.0-20201120111947-b8bd55bc02bd/go.mod h1:0UtvvETGDdvXNDCHa8ZQpxl+w3HbdFtfYZvDHLgWGTY= github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 h1:Jz3KVLYY5+JO7rDiX0sAuRGtuv2vG01r17Y9nLMWNUw= github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/appleboy/gin-jwt/v2 v2.6.3/go.mod h1:MfPYA4ogzvOcVkRwAxT7quHOtQmVKDpTwxyUrC2DNw0= github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= -github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4= -github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAFAONtv2Dr7HUI= -github.com/ardielle/ardielle-tools v1.5.4/go.mod h1:oZN+JRMnqGiIhrzkRN9l26Cej9dEx4jeNG6A+AdkShk= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.32.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.35.3 h1:r0puXncSaAfRt7Btml2swUo74Kao+vKhO3VLjwDjK54= github.com/aws/aws-sdk-go v1.35.3/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= -github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -136,10 +123,6 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d h1:rQlvB2AYWme2bIB18r/SipGiMEVJYE9U0z+MGoU/LtQ= github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU= -github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= -github.com/bradleyjkemp/cupaloy/v2 v2.5.0/go.mod h1:TD5UU0rdYTbu/TtuwFuWrtiRARuN7mtRipvs/bsShSE= -github.com/bradleyjkemp/grpc-tools v0.2.5 h1:zZhwRxFktKIZliZ7g+V6zwNl0m9o/W1kvWJFWRxkZ/Q= -github.com/bradleyjkemp/grpc-tools v0.2.5/go.mod h1:9OM0QfQGzMUC98I2kvHMK4Lw0memhg8j2BosoL4ME0M= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= github.com/carlmjohnson/flagext v0.21.0 h1:/c4uK3ie786Z7caXLcIMvePNSSiH3bQVGDvmGLMme60= @@ -218,15 +201,11 @@ github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 h1:LpMLYGyy67BoAFGd github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= github.com/cznic/y v0.0.0-20170802143616-045f81c6662a/go.mod h1:1rk5VM7oSnA4vjp+hrLQ3HWHa+Y4yPCa3/CsJrcNnvs= -github.com/danieljoos/wincred v1.0.2 h1:zf4bhty2iLuwgjgpraD2E9UbvO+fe54XXGJbOwe23fU= -github.com/danieljoos/wincred v1.0.2/go.mod h1:SnuYRW9lp1oJrZX/dXJqr0cPK5gYXqx3EJbmjhLdK9U= github.com/danjacques/gofslock v0.0.0-20191023191349-0a45f885bc37 h1:X6mKGhCFOxrKeeHAjv/3UvT6e5RRxW6wRdlqlV6/H4w= github.com/danjacques/gofslock v0.0.0-20191023191349-0a45f885bc37/go.mod h1:DC3JtzuG7kxMvJ6dZmf2ymjNyoXwgtklr7FN+Um2B0U= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I= -github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/ristretto v0.0.1 h1:cJwdnj42uV8Jg4+KLrYovLiCgIfz9wtWm6E6KA+1tLs= github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= @@ -234,8 +213,6 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA= -github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= @@ -244,8 +221,6 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dvsekhvalnov/jose2go v0.0.0-20180829124132-7f401d37b68a h1:mq+R6XEM6lJX5VlLyZIrUSP8tSuJp82xTK89hvBwJbU= -github.com/dvsekhvalnov/jose2go v0.0.0-20180829124132-7f401d37b68a/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM= github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= @@ -361,8 +336,6 @@ github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22 github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/goccy/go-graphviz v0.0.5/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk= -github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= -github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -481,7 +454,6 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -495,8 +467,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= -github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/gtank/cryptopasta v0.0.0-20170601214702-1f550f6f2f69/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= @@ -529,8 +499,6 @@ github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334/go.mod h1:SK73t github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= -github.com/improbable-eng/grpc-web v0.12.0 h1:GlCS+lMZzIkfouf7CNqY+qqpowdKuJLSLLcKVfM1oLc= -github.com/improbable-eng/grpc-web v0.12.0/go.mod h1:6hRR09jOEG81ADP5wCQju1z71g6OL4eEvELdran/3cs= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/tdigest v0.0.1/go.mod h1:Z0kXnxzbTC2qrx4NaIzYkE1k66+6oEDQTvL95hQFh5Y= @@ -542,14 +510,11 @@ github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62 github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jarcoal/httpmock v1.0.5 h1:cHtVEcTxRSX4J0je7mWPfc9BpDpqzXSJ5HbymZmyHck= github.com/jarcoal/httpmock v1.0.5/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= -github.com/jawher/mow.cli v1.0.4/go.mod h1:5hQj2V8g+qYmLUVWqu4Wuja1pI57M83EChYLVZ0sMKk= -github.com/jawher/mow.cli v1.2.0/go.mod h1:y+pcA3jBAdo/GIZx/0rFjw/K2bVEODP9rfZOfaiq8Ko= github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jedib0t/go-pretty/v6 v6.2.2/go.mod h1:+nE9fyyHGil+PuISTCrp7avEdo6bqoMwqZnuiK2r2a0= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= @@ -558,8 +523,6 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jmoiron/sqlx v1.3.3 h1:j82X0bf7oQ27XeqxicSZsTU5suPwKElg3oyxNn43iTk= -github.com/jmoiron/sqlx v1.3.3/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df h1:Zrb0IbuLOGHL7nrO2WrcuNWgDTlzFv3zY69QMx4ggQE= github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df/go.mod h1:mAVCUAYtW9NG31eB30umMSLKcDt6mCUWSjoSn5qBh0k= @@ -589,8 +552,6 @@ github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4 github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= -github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d h1:Z+RDyXzjKE0i2sTjZ/b1uxiGtPhFy34Ou/Tk0qwN0kM= -github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d/go.mod h1:JJNrCn9otv/2QP4D7SMJBgaleKpOf66PnW6F5WGNRIc= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -600,7 +561,6 @@ github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.10.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= @@ -611,7 +571,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -626,9 +585,6 @@ github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdA github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= -github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/linkedin/goavro/v2 v2.9.8 h1:jN50elxBsGBDGVDEKqUlDuU1cFwJ11K/yrJCBMe/7Wg= github.com/linkedin/goavro/v2 v2.9.8/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= @@ -662,9 +618,6 @@ github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRC github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= -github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v2.0.2+incompatible h1:qzw9c2GNT8UFrgWNDhCTqRqYUSmu/Dav/9Z58LGpk7U= -github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -677,7 +630,6 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N github.com/minio/sio v0.3.0/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -696,11 +648,7 @@ github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3P github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk= github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= -github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= -github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= @@ -747,7 +695,6 @@ github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5e github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d/go.mod h1:lXfE4PvvTW5xOjO6Mba8zDPyw8M93B6AQ7frTGnMlA8= github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pingcap/badger v1.5.1-0.20210831093107-2f6cb8008145 h1:t7sdxmfyZ3p9K7gD8t5B50TerzTvHuAPYt+VubTVKDY= @@ -865,7 +812,6 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -1156,7 +1102,6 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1182,7 +1127,6 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1195,7 +1139,6 @@ golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -1267,7 +1210,6 @@ golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1327,7 +1269,6 @@ golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1468,7 +1409,6 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -1525,7 +1465,6 @@ google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwy google.golang.org/genproto v0.0.0-20210825212027-de86158e7fda h1:iT5uhT54PtbqUsWddv/nnEWdE5e/MTr+Nv3vjxlBP1A= google.golang.org/genproto v0.0.0-20210825212027-de86158e7fda/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1599,12 +1538,9 @@ gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuv gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= -gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= @@ -1661,5 +1597,3 @@ sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 h1:ucqkfp sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 h1:e1sMhtVq9AfcEy8AXNb8eSg6gbzfdpYhoNqnPJa+GzI= sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67/go.mod h1:L5q+DGLGOQFpo1snNEkLOJT2d1YTW66rWNzatr3He1k= -upper.io/db.v3 v3.7.1+incompatible h1:GiK/NmDUClH3LrZd54qj5OQsz8brGFv652QXyRXtg2U= -upper.io/db.v3 v3.7.1+incompatible/go.mod h1:FgTdD24eBjJAbPKsQSiHUNgXjOR4Lub3u1UMHSIh82Y= diff --git a/cdc/pkg/actor/actor.go b/cdc/pkg/actor/actor.go index ba0520b5..3e48f73d 100644 --- a/cdc/pkg/actor/actor.go +++ b/cdc/pkg/actor/actor.go @@ -12,102 +12,3 @@ // limitations under the License. package actor - -import ( - "context" - - "github.com/tikv/migration/cdc/pkg/actor/message" - cerrors "github.com/tikv/migration/cdc/pkg/errors" -) - -var errMailboxFull = cerrors.ErrMailboxFull.FastGenByArgs() - -// ID is ID for actors. -type ID uint64 - -// Actor is a universal primitive of concurrent computation. -// See more https://en.wikipedia.org/wiki/Actor_model -type Actor interface { - // Poll handles messages that are sent to actor's mailbox. - // - // The ctx is only for cancellation, and an actor must be aware of - // the cancellation. - // - // If it returns true, then the actor will be rescheduled and polled later. - // If it returns false, then the actor will be removed from Router and - // polled if there are still messages in its mailbox. - // Once it returns false, it must always return false. - // - // We choose message to have a concrete type instead of an interface to save - // memory allocation. - Poll(ctx context.Context, msgs []message.Message) (running bool) -} - -// Mailbox sends messages to an actor. -// Mailbox is threadsafe. -type Mailbox interface { - ID() ID - // Send a message to its actor. - // It's a non-blocking send, returns ErrMailboxFull when it's full. - Send(msg message.Message) error - // SendB sends a message to its actor, blocks when it's full. - // It may return context.Canceled or context.DeadlineExceeded. - SendB(ctx context.Context, msg message.Message) error - - // Receive a message. - // It must be nonblocking and should only be called by System. - Receive() (message.Message, bool) - // Return the length of a mailbox. - // It should only be called by System. - len() int -} - -// NewMailbox creates a fixed capacity mailbox. -func NewMailbox(id ID, cap int) Mailbox { - return &mailbox{ - id: id, - ch: make(chan message.Message, cap), - } -} - -var _ Mailbox = (*mailbox)(nil) - -type mailbox struct { - id ID - ch chan message.Message -} - -func (m *mailbox) ID() ID { - return m.id -} - -func (m *mailbox) Send(msg message.Message) error { - select { - case m.ch <- msg: - return nil - default: - return errMailboxFull - } -} - -func (m *mailbox) SendB(ctx context.Context, msg message.Message) error { - select { - case <-ctx.Done(): - return ctx.Err() - case m.ch <- msg: - return nil - } -} - -func (m *mailbox) Receive() (message.Message, bool) { - select { - case msg, ok := <-m.ch: - return msg, ok - default: - } - return message.Message{}, false -} - -func (m *mailbox) len() int { - return len(m.ch) -} diff --git a/cdc/pkg/actor/actor_test.go b/cdc/pkg/actor/actor_test.go index b7d06a46..3e48f73d 100644 --- a/cdc/pkg/actor/actor_test.go +++ b/cdc/pkg/actor/actor_test.go @@ -12,92 +12,3 @@ // limitations under the License. package actor - -import ( - "context" - _ "net/http/pprof" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/actor/message" -) - -// Make sure mailbox implementation follows Mailbox definition. -func testMailbox(t *testing.T, mb Mailbox) { - // Empty mailbox. - require.Equal(t, 0, mb.len()) - _, ok := mb.Receive() - require.False(t, ok) - - // Send and receive. - err := mb.Send(message.BarrierMessage(model.Ts(1))) - require.Nil(t, err) - require.Equal(t, 1, mb.len()) - msg, ok := mb.Receive() - require.Equal(t, message.BarrierMessage(1), msg) - require.True(t, ok) - - // Empty mailbox. - _, ok = mb.Receive() - require.False(t, ok) - - // Mailbox has a bounded capacity. - for { - err = mb.Send(message.BarrierMessage(model.Ts(1))) - if err != nil { - break - } - } - // SendB should be blocked. - ch := make(chan error) - ctx, cancel := context.WithCancel(context.Background()) - go func() { - ch <- nil - ch <- mb.SendB(ctx, message.BarrierMessage(2)) - }() - // Wait for goroutine start. - <-ch - select { - case <-time.After(100 * time.Millisecond): - case err = <-ch: - t.Fatalf("must timeout, got error %v", err) - } - // Receive unblocks SendB - msg, ok = mb.Receive() - require.Equal(t, message.BarrierMessage(1), msg) - require.True(t, ok) - select { - case <-time.After(100 * time.Millisecond): - t.Fatal("must not timeout") - case err = <-ch: - require.Nil(t, err) - } - - // SendB must be aware of context cancel. - ch = make(chan error) - go func() { - ch <- nil - ch <- mb.SendB(ctx, message.BarrierMessage(2)) - }() - // Wait for goroutine start. - <-ch - select { - case <-time.After(100 * time.Millisecond): - case err = <-ch: - t.Fatalf("must timeout, got error %v", err) - } - cancel() - select { - case <-time.After(100 * time.Millisecond): - t.Fatal("must not timeout") - case err = <-ch: - require.Error(t, err) - } -} - -func TestMailbox(t *testing.T) { - mb := NewMailbox(ID(1), 1) - testMailbox(t, mb) -} diff --git a/cdc/pkg/actor/message/message.go b/cdc/pkg/actor/message/message.go index 0e0344a4..6dee97c5 100644 --- a/cdc/pkg/actor/message/message.go +++ b/cdc/pkg/actor/message/message.go @@ -12,64 +12,3 @@ // limitations under the License. package message - -import ( - "github.com/tikv/migration/cdc/cdc/model" - sorter "github.com/tikv/migration/cdc/cdc/sorter/leveldb/message" -) - -// Type is the type of Message -type Type int - -// types of Message -const ( - TypeUnknown Type = iota - TypeTick - TypeStop - TypeBarrier - TypeSorterTask - // Add a new type when adding a new message. -) - -// Message is a vehicle for transferring information between nodes -type Message struct { - // Tp is the type of Message - Tp Type - // BarrierTs - BarrierTs model.Ts - // Leveldb sorter task - // TODO: find a way to hide it behind an interface while saving - // memory allocation. - // See https://cs.opensource.google/go/go/+/refs/tags/go1.17.2:src/runtime/iface.go;l=325 - SorterTask sorter.Task -} - -// TickMessage creates the message of Tick -func TickMessage() Message { - return Message{ - Tp: TypeTick, - } -} - -// StopMessage creates the message of Stop -func StopMessage() Message { - return Message{ - Tp: TypeStop, - } -} - -// BarrierMessage creates the message of Command -func BarrierMessage(barrierTs model.Ts) Message { - return Message{ - Tp: TypeBarrier, - BarrierTs: barrierTs, - } -} - -// SorterMessage creates the message of sorter -func SorterMessage(task sorter.Task) Message { - return Message{ - Tp: TypeSorterTask, - SorterTask: task, - } -} diff --git a/cdc/pkg/actor/message/message_test.go b/cdc/pkg/actor/message/message_test.go index 34e51395..6dee97c5 100644 --- a/cdc/pkg/actor/message/message_test.go +++ b/cdc/pkg/actor/message/message_test.go @@ -12,40 +12,3 @@ // limitations under the License. package message - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/require" - sorter "github.com/tikv/migration/cdc/cdc/sorter/leveldb/message" - "github.com/tikv/migration/cdc/pkg/leakutil" -) - -func TestMain(m *testing.M) { - leakutil.SetUpLeakTest(m) -} - -// Make sure Message can be printed in JSON format, so that it can be logged by -// pingcap/log package. -func TestJSONPrint(t *testing.T) { - _, err := json.Marshal(Message{}) - require.Nil(t, err) -} - -func TestTickMessage(t *testing.T) { - msg := TickMessage() - require.Equal(t, TypeTick, msg.Tp) -} - -func TestBarrierMessage(t *testing.T) { - msg := BarrierMessage(1) - require.Equal(t, TypeBarrier, msg.Tp) -} - -func TestSorterMessage(t *testing.T) { - task := sorter.Task{UID: 1, TableID: 2} - msg := SorterMessage(task) - require.Equal(t, TypeSorterTask, msg.Tp) - require.Equal(t, task, msg.SorterTask) -} diff --git a/cdc/pkg/actor/metrics.go b/cdc/pkg/actor/metrics.go index ea5f4f53..3e48f73d 100644 --- a/cdc/pkg/actor/metrics.go +++ b/cdc/pkg/actor/metrics.go @@ -12,64 +12,3 @@ // limitations under the License. package actor - -import ( - "github.com/prometheus/client_golang/prometheus" -) - -var ( - totalWorkers = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "ticdc", - Subsystem: "actor", - Name: "number_of_workers", - Help: "The total number of workers in an actor system.", - }, []string{"name"}) - workingWorkers = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "ticdc", - Subsystem: "actor", - Name: "number_of_working_workers", - Help: "The number of working workers in an actor system.", - }, []string{"name"}) - workingDuration = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "ticdc", - Subsystem: "actor", - Name: "worker_cpu_seconds_total", - Help: "Total user and system CPU time spent by workers in seconds.", - }, []string{"name", "id"}) - batchSizeHistogram = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: "ticdc", - Subsystem: "actor", - Name: "batch", - Help: "Bucketed histogram of batch size of an actor system.", - Buckets: prometheus.ExponentialBuckets(1, 2, 10), - }, []string{"name", "type"}) - pollActorDuration = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: "ticdc", - Subsystem: "actor", - Name: "poll_duration_seconds", - Help: "Bucketed histogram of actor poll time (s).", - Buckets: prometheus.ExponentialBuckets(0.01, 2, 16), - }, []string{"name"}) - dropMsgCount = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "ticdc", - Subsystem: "actor", - Name: "drop_message_total", - Help: "The total number of dropped messages in an actor system.", - }, []string{"name"}) -) - -// InitMetrics registers all metrics in this file -func InitMetrics(registry *prometheus.Registry) { - registry.MustRegister(totalWorkers) - registry.MustRegister(workingWorkers) - registry.MustRegister(workingDuration) - registry.MustRegister(batchSizeHistogram) - registry.MustRegister(pollActorDuration) - registry.MustRegister(dropMsgCount) -} diff --git a/cdc/pkg/actor/system.go b/cdc/pkg/actor/system.go index 5e39e277..3e48f73d 100644 --- a/cdc/pkg/actor/system.go +++ b/cdc/pkg/actor/system.go @@ -12,507 +12,3 @@ // limitations under the License. package actor - -import ( - "container/list" - "context" - "runtime" - "runtime/pprof" - "strconv" - "sync" - "sync/atomic" - "time" - - "github.com/pingcap/log" - "github.com/prometheus/client_golang/prometheus" - "github.com/tikv/migration/cdc/pkg/actor/message" - cerrors "github.com/tikv/migration/cdc/pkg/errors" - "go.uber.org/zap" - "golang.org/x/sync/errgroup" -) - -const ( - // The max number of workers of a system. - maxWorkerNum = 64 - // The default size of polled actor batch. - defaultActorBatchSize = 1 - // The default size of receive message batch. - defaultMsgBatchSizePerActor = 64 -) - -var ( - errActorStopped = cerrors.ErrActorStopped.FastGenByArgs() - errActorNotFound = cerrors.ErrActorNotFound.FastGenByArgs() -) - -// proc is wrapper of a running actor. -type proc struct { - mb Mailbox - actor Actor - closed uint64 -} - -// batchReceiveMsgs receives messages into batchMsg. -func (p *proc) batchReceiveMsgs(batchMsg []message.Message) int { - n := 0 - max := len(batchMsg) - for i := 0; i < max; i++ { - msg, ok := p.mb.Receive() - if !ok { - // Stop receive if there is no more messages. - break - } - batchMsg[i] = msg - n++ - } - return n -} - -// close its actor. -// close is threadsafe. -func (p *proc) close() { - atomic.StoreUint64(&p.closed, 1) -} - -// isClosed returns ture, means its actor is closed. -// isClosed is threadsafe. -func (p *proc) isClosed() bool { - return atomic.LoadUint64(&p.closed) == 1 -} - -// ready is a centralize notification struct, shared by a router and a system. -// It schedules notification and actors. -type ready struct { - sync.Mutex - cond *sync.Cond - - // TODO: replace with a memory efficient queue, - // e.g., an array based queue to save allocation. - queue list.List - // In the set, an actor is either polling by system - // or is pending to be polled. - procs map[ID]struct{} - stopped bool - - metricDropMessage prometheus.Counter -} - -func (rd *ready) stop() { - rd.Lock() - rd.stopped = true - rd.Unlock() - rd.cond.Broadcast() -} - -// enqueueLocked enqueues ready proc. -// If the proc is already enqueued, it ignores. -// If the proc is closed, it ignores, drop messages in mailbox and return error. -// Set force to true to force enqueue. It is useful to force the proc to be -// polled again. -func (rd *ready) enqueueLocked(p *proc, force bool) error { - if p.isClosed() { - // Drop all remaining messages. - counter := 0 - _, ok := p.mb.Receive() - for ; ok; _, ok = p.mb.Receive() { - counter++ - } - rd.metricDropMessage.Add(float64(counter)) - return errActorStopped - } - id := p.mb.ID() - if _, ok := rd.procs[id]; !ok || force { - rd.queue.PushBack(p) - rd.procs[id] = struct{}{} - } - - return nil -} - -// schedule schedules the proc to system. -func (rd *ready) schedule(p *proc) error { - rd.Lock() - err := rd.enqueueLocked(p, false) - rd.Unlock() - if err != nil { - return err - } - // Notify system to poll the proc. - rd.cond.Signal() - return nil -} - -// scheduleN schedules a slice of procs to system. -// It ignores stopped procs. -func (rd *ready) scheduleN(procs []*proc) { - rd.Lock() - for _, p := range procs { - _ = rd.enqueueLocked(p, false) - } - rd.Unlock() - rd.cond.Broadcast() -} - -// batchReceiveProcs receives ready procs into batchP. -func (rd *ready) batchReceiveProcs(batchP []*proc) int { - n := 0 - max := len(batchP) - for i := 0; i < max; i++ { - if rd.queue.Len() == 0 { - // Stop receive if there is no more ready procs. - break - } - element := rd.queue.Front() - rd.queue.Remove(element) - p := element.Value.(*proc) - batchP[i] = p - n++ - } - return n -} - -// Router send messages to actors. -type Router struct { - rd *ready - - // Map of ID to proc - procs sync.Map -} - -// NewRouter returns a new router. -func NewRouter(name string) *Router { - r := &Router{ - rd: &ready{}, - } - r.rd.cond = sync.NewCond(&r.rd.Mutex) - r.rd.procs = make(map[ID]struct{}) - r.rd.queue.Init() - r.rd.metricDropMessage = dropMsgCount.WithLabelValues(name) - return r -} - -// Send a message to an actor. It's a non-blocking send. -// ErrMailboxFull when the actor full. -// ErrActorNotFound when the actor not found. -func (r *Router) Send(id ID, msg message.Message) error { - value, ok := r.procs.Load(id) - if !ok { - return errActorNotFound - } - p := value.(*proc) - err := p.mb.Send(msg) - if err != nil { - return err - } - return r.rd.schedule(p) -} - -// SendB sends a message to an actor, blocks when it's full. -// ErrActorNotFound when the actor not found. -// Canceled or DeadlineExceeded when the context is canceled or done. -func (r *Router) SendB(ctx context.Context, id ID, msg message.Message) error { - value, ok := r.procs.Load(id) - if !ok { - return errActorNotFound - } - p := value.(*proc) - err := p.mb.SendB(ctx, msg) - if err != nil { - return err - } - return r.rd.schedule(p) -} - -// Broadcast a message to all actors in the router. -// The message may be dropped when a actor is full. -func (r *Router) Broadcast(msg message.Message) { - batchSize := 128 - ps := make([]*proc, 0, batchSize) - r.procs.Range(func(key, value interface{}) bool { - p := value.(*proc) - if err := p.mb.Send(msg); err != nil { - log.Warn("failed to send to message", - zap.Error(err), zap.Uint64("id", uint64(p.mb.ID())), - zap.Reflect("msg", msg)) - // Skip schedule the proc. - return true - } - ps = append(ps, p) - if len(ps) == batchSize { - r.rd.scheduleN(ps) - ps = ps[:0] - } - return true - }) - - if len(ps) != 0 { - r.rd.scheduleN(ps) - } -} - -func (r *Router) insert(id ID, p *proc) error { - _, exist := r.procs.LoadOrStore(id, p) - if exist { - return cerrors.ErrActorDuplicate.FastGenByArgs() - } - return nil -} - -func (r *Router) remove(id ID) bool { - _, present := r.procs.LoadAndDelete(id) - return present -} - -// SystemBuilder is a builder of a system. -type SystemBuilder struct { - name string - numWorker int - actorBatchSize int - msgBatchSizePerActor int - - fatalHandler func(string, ID) -} - -// NewSystemBuilder returns a new system builder. -func NewSystemBuilder(name string) *SystemBuilder { - defaultWorkerNum := maxWorkerNum - goMaxProcs := runtime.GOMAXPROCS(0) - if goMaxProcs*8 < defaultWorkerNum { - defaultWorkerNum = goMaxProcs * 8 - } - - return &SystemBuilder{ - name: name, - numWorker: defaultWorkerNum, - actorBatchSize: defaultActorBatchSize, - msgBatchSizePerActor: defaultMsgBatchSizePerActor, - } -} - -// WorkerNumber sets the number of workers of a system. -func (b *SystemBuilder) WorkerNumber(numWorker int) *SystemBuilder { - if numWorker <= 0 { - numWorker = 1 - } else if numWorker > maxWorkerNum { - numWorker = maxWorkerNum - } - b.numWorker = numWorker - return b -} - -// Throughput sets the throughput per-poll of a system. -func (b *SystemBuilder) Throughput( - actorBatchSize, msgBatchSizePerActor int, -) *SystemBuilder { - if actorBatchSize <= 0 { - actorBatchSize = 1 - } - if msgBatchSizePerActor <= 0 { - msgBatchSizePerActor = 1 - } - - b.actorBatchSize = actorBatchSize - b.msgBatchSizePerActor = msgBatchSizePerActor - return b -} - -// handleFatal sets the fatal handler of a system. -func (b *SystemBuilder) handleFatal( - fatalHandler func(string, ID), -) *SystemBuilder { - b.fatalHandler = fatalHandler - return b -} - -// Build builds a system and a router. -func (b *SystemBuilder) Build() (*System, *Router) { - router := NewRouter(b.name) - metricWorkingDurations := make([]prometheus.Counter, b.numWorker) - for i := range metricWorkingDurations { - metricWorkingDurations[i] = - workingDuration.WithLabelValues(b.name, strconv.Itoa(i)) - } - return &System{ - name: b.name, - numWorker: b.numWorker, - actorBatchSize: b.actorBatchSize, - msgBatchSizePerActor: b.msgBatchSizePerActor, - - rd: router.rd, - router: router, - - fatalHandler: b.fatalHandler, - - metricTotalWorkers: totalWorkers.WithLabelValues(b.name), - metricWorkingWorkers: workingWorkers.WithLabelValues(b.name), - metricWorkingDurations: metricWorkingDurations, - metricPollDuration: pollActorDuration.WithLabelValues(b.name), - metricProcBatch: batchSizeHistogram.WithLabelValues(b.name, "proc"), - metricMsgBatch: batchSizeHistogram.WithLabelValues(b.name, "msg"), - }, router -} - -// System is the runtime of Actors. -type System struct { - name string - numWorker int - actorBatchSize int - msgBatchSizePerActor int - - rd *ready - router *Router - wg *errgroup.Group - cancel context.CancelFunc - - fatalHandler func(string, ID) - - // Metrics - metricTotalWorkers prometheus.Gauge - metricWorkingWorkers prometheus.Gauge - metricWorkingDurations []prometheus.Counter - metricPollDuration prometheus.Observer - metricProcBatch prometheus.Observer - metricMsgBatch prometheus.Observer -} - -// Start the system. Cancelling the context to stop the system. -// Start is not threadsafe. -func (s *System) Start(ctx context.Context) { - s.wg, ctx = errgroup.WithContext(ctx) - ctx, s.cancel = context.WithCancel(ctx) - - s.metricTotalWorkers.Add(float64(s.numWorker)) - for i := 0; i < s.numWorker; i++ { - id := i - s.wg.Go(func() error { - defer pprof.SetGoroutineLabels(ctx) - pctx := pprof.WithLabels(ctx, pprof.Labels("actor", s.name)) - pprof.SetGoroutineLabels(pctx) - - s.poll(pctx, id) - return nil - }) - } -} - -// Stop the system, cancels all actors. It should be called after Start. -// Stop is not threadsafe. -func (s *System) Stop() error { - s.metricTotalWorkers.Add(-float64(s.numWorker)) - if s.cancel != nil { - s.cancel() - } - s.rd.stop() - return s.wg.Wait() -} - -// Spawn spawns an actor in the system. -// Spawn is threadsafe. -func (s *System) Spawn(mb Mailbox, actor Actor) error { - id := mb.ID() - p := &proc{mb: mb, actor: actor} - return s.router.insert(id, p) -} - -const slowReceiveThreshold = time.Second - -// The main poll of actor system. -func (s *System) poll(ctx context.Context, id int) { - batchPBuf := make([]*proc, s.actorBatchSize) - batchMsgBuf := make([]message.Message, s.msgBatchSizePerActor) - rd := s.rd - rd.Lock() - - startTime := time.Now() - s.metricWorkingWorkers.Inc() - for { - var batchP []*proc - for { - if rd.stopped { - rd.Unlock() - return - } - // Batch receive ready procs. - n := rd.batchReceiveProcs(batchPBuf) - if n != 0 { - batchP = batchPBuf[:n] - s.metricProcBatch.Observe(float64(n)) - break - } - // Recording metrics. - s.metricWorkingDurations[id].Add(time.Since(startTime).Seconds()) - s.metricWorkingWorkers.Dec() - // Park the poll until it is awakened. - rd.cond.Wait() - startTime = time.Now() - s.metricWorkingWorkers.Inc() - } - rd.Unlock() - - for _, p := range batchP { - closed := p.isClosed() - if closed { - s.handleFatal( - "closed actor can never receive new messages again", - p.mb.ID()) - } - - // Batch receive actor's messages. - n := p.batchReceiveMsgs(batchMsgBuf) - if n == 0 { - continue - } - batchMsg := batchMsgBuf[:n] - s.metricMsgBatch.Observe(float64(n)) - - // Poll actor. - pollStartTime := time.Now() - running := p.actor.Poll(ctx, batchMsg) - if !running { - // Close the actor. - p.close() - } - receiveDuration := time.Since(pollStartTime) - if receiveDuration > slowReceiveThreshold { - log.Warn("actor handle received messages too slow", - zap.Duration("duration", receiveDuration), - zap.Uint64("id", uint64(p.mb.ID())), - zap.String("name", s.name)) - } - s.metricPollDuration.Observe(receiveDuration.Seconds()) - } - - rd.Lock() - for _, p := range batchP { - if p.mb.len() == 0 { - // At this point, there is no more message needs to be polled - // by the actor. Delete the actor from ready set. - delete(rd.procs, p.mb.ID()) - } else { - // Force enqueue to poll remaining messages. - // Also it drops remaining messages if proc is stopped. - _ = rd.enqueueLocked(p, true) - } - if p.isClosed() { - // Remove closed actor from router. - present := s.router.remove(p.mb.ID()) - if !present { - s.handleFatal( - "try to remove non-existent actor", p.mb.ID()) - } - } - } - } -} - -func (s *System) handleFatal(msg string, id ID) { - handler := defaultFatalhandler - if s.fatalHandler != nil { - handler = s.fatalHandler - } - handler(msg, id) -} - -func defaultFatalhandler(msg string, id ID) { - log.Panic(msg, zap.Uint64("id", uint64(id))) -} diff --git a/cdc/pkg/actor/system_test.go b/cdc/pkg/actor/system_test.go index 057109a3..3e48f73d 100644 --- a/cdc/pkg/actor/system_test.go +++ b/cdc/pkg/actor/system_test.go @@ -12,725 +12,3 @@ // limitations under the License. package actor - -import ( - "context" - "fmt" - "math" - _ "net/http/pprof" - "strings" - "sync/atomic" - "testing" - "time" - - dto "github.com/prometheus/client_model/go" - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/pkg/actor/message" - "github.com/tikv/migration/cdc/pkg/leakutil" -) - -func TestMain(m *testing.M) { - leakutil.SetUpLeakTest(m) -} - -func makeTestSystem(name string, t interface { - Fatalf(format string, args ...interface{}) -}) (*System, *Router) { - return NewSystemBuilder(name). - WorkerNumber(2). - handleFatal(func(s string, i ID) { - t.Fatalf("%s actorID: %d", s, i) - }). - Build() -} - -func TestSystemBuilder(t *testing.T) { - t.Parallel() - b := NewSystemBuilder("test") - require.LessOrEqual(t, b.numWorker, maxWorkerNum) - require.Greater(t, b.numWorker, 0) - - b.WorkerNumber(0) - require.Equal(t, 1, b.numWorker) - - b.WorkerNumber(2) - require.Equal(t, 2, b.numWorker) - - require.Greater(t, b.actorBatchSize, 0) - require.Greater(t, b.msgBatchSizePerActor, 0) - - b.Throughput(0, 0) - require.Greater(t, b.actorBatchSize, 0) - require.Greater(t, b.msgBatchSizePerActor, 0) - - b.Throughput(7, 8) - require.Equal(t, 7, b.actorBatchSize) - require.Equal(t, 8, b.msgBatchSizePerActor) -} - -func TestMailboxSendAndSendB(t *testing.T) { - t.Parallel() - mb := NewMailbox(ID(0), 1) - err := mb.Send(message.TickMessage()) - require.Nil(t, err) - - err = mb.Send(message.TickMessage()) - require.True(t, strings.Contains(err.Error(), "mailbox is full")) - - msg, ok := mb.Receive() - require.Equal(t, true, ok) - require.Equal(t, message.TickMessage(), msg) - - // Test SendB can be canceled by context. - ch := make(chan error) - ctx, cancel := context.WithCancel(context.Background()) - go func() { - err := mb.Send(message.TickMessage()) - ch <- err - err = mb.SendB(ctx, message.TickMessage()) - ch <- err - }() - - require.Nil(t, <-ch) - cancel() - require.Equal(t, context.Canceled, <-ch) -} - -func TestRouterSendAndSendB(t *testing.T) { - t.Parallel() - id := ID(0) - mb := NewMailbox(id, 1) - router := NewRouter(t.Name()) - err := router.insert(id, &proc{mb: mb}) - require.Nil(t, err) - err = router.Send(id, message.TickMessage()) - require.Nil(t, err) - - err = router.Send(id, message.TickMessage()) - require.True(t, strings.Contains(err.Error(), "mailbox is full")) - - msg, ok := mb.Receive() - require.Equal(t, true, ok) - require.Equal(t, message.TickMessage(), msg) - - // Test SendB can be canceled by context. - ch := make(chan error) - ctx, cancel := context.WithCancel(context.Background()) - go func() { - err := router.Send(id, message.TickMessage()) - ch <- err - err = router.SendB(ctx, id, message.TickMessage()) - ch <- err - }() - - require.Nil(t, <-ch) - cancel() - require.Equal(t, context.Canceled, <-ch) -} - -func wait(t *testing.T, f func()) { - wait := make(chan int) - go func() { - f() - wait <- 0 - }() - select { - case <-wait: - case <-time.After(5 * time.Second): - // There may be a deadlock if f takes more than 5 seconds. - t.Fatal("Timed out") - } -} - -func TestSystemStartStop(t *testing.T) { - t.Parallel() - ctx := context.Background() - sys, _ := makeTestSystem(t.Name(), t) - sys.Start(ctx) - err := sys.Stop() - require.Nil(t, err) -} - -func TestSystemSpawnDuplicateActor(t *testing.T) { - t.Parallel() - ctx := context.Background() - sys, _ := makeTestSystem(t.Name(), t) - sys.Start(ctx) - - id := 1 - fa := &forwardActor{} - mb := NewMailbox(ID(id), 1) - require.Nil(t, sys.Spawn(mb, fa)) - require.NotNil(t, sys.Spawn(mb, fa)) - - wait(t, func() { - err := sys.Stop() - require.Nil(t, err) - }) -} - -type forwardActor struct { - contextAware bool - - ch chan<- message.Message -} - -func (f *forwardActor) Poll(ctx context.Context, msgs []message.Message) bool { - for _, msg := range msgs { - if f.contextAware { - select { - case f.ch <- msg: - case <-ctx.Done(): - } - } else { - f.ch <- msg - } - } - return true -} - -func TestActorSendReceive(t *testing.T) { - t.Parallel() - ctx := context.Background() - sys, router := makeTestSystem(t.Name(), t) - sys.Start(ctx) - - // Send to a non-existing actor. - id := ID(777) - err := router.Send(id, message.BarrierMessage(0)) - require.Equal(t, errActorNotFound, err) - - ch := make(chan message.Message, 1) - fa := &forwardActor{ - ch: ch, - } - mb := NewMailbox(id, 1) - - // The actor is not in router yet. - err = router.Send(id, message.BarrierMessage(1)) - require.Equal(t, errActorNotFound, err) - - // Spawn adds the actor to the router. - require.Nil(t, sys.Spawn(mb, fa)) - err = router.Send(id, message.BarrierMessage(2)) - require.Nil(t, err) - select { - case msg := <-ch: - require.Equal(t, message.BarrierMessage(2), msg) - case <-time.After(time.Second): - t.Fatal("Timed out") - } - - wait(t, func() { - err := sys.Stop() - require.Nil(t, err) - }) -} - -func testBroadcast(t *testing.T, actorNum, workerNum int) { - ctx := context.Background() - sys, router := NewSystemBuilder("test").WorkerNumber(workerNum).Build() - sys.Start(ctx) - - ch := make(chan message.Message, 1) - - for id := 0; id < actorNum; id++ { - fa := &forwardActor{ - ch: ch, - } - mb := NewMailbox(ID(id), 1) - require.Nil(t, sys.Spawn(mb, fa)) - } - - // Broadcase tick to actors. - router.Broadcast(message.TickMessage()) - for i := 0; i < actorNum; i++ { - select { - case msg := <-ch: - require.Equal(t, message.TickMessage(), msg) - case <-time.After(time.Second): - t.Fatal("Timed out") - } - } - select { - case msg := <-ch: - t.Fatal("Unexpected message", msg) - case <-time.After(200 * time.Millisecond): - } - - wait(t, func() { - err := sys.Stop() - require.Nil(t, err) - }) -} - -func TestBroadcast(t *testing.T) { - t.Parallel() - for _, workerNum := range []int{1, 2, 16, 32, 64} { - for _, actorNum := range []int{0, 1, 64, 128, 195, 1024} { - testBroadcast(t, actorNum, workerNum) - } - } -} - -func TestSystemStopCancelActors(t *testing.T) { - t.Parallel() - ctx := context.Background() - sys, router := makeTestSystem(t.Name(), t) - sys.Start(ctx) - - id := ID(777) - ch := make(chan message.Message, 1) - fa := &forwardActor{ - ch: ch, - contextAware: true, - } - mb := NewMailbox(id, 1) - require.Nil(t, sys.Spawn(mb, fa)) - err := router.Send(id, message.TickMessage()) - require.Nil(t, err) - - id = ID(778) - fa = &forwardActor{ - ch: ch, - contextAware: true, - } - mb = NewMailbox(id, 1) - require.Nil(t, sys.Spawn(mb, fa)) - err = router.Send(id, message.TickMessage()) - require.Nil(t, err) - - // Do not receive ch. - _ = ch - - wait(t, func() { - err := sys.Stop() - require.Nil(t, err) - }) -} - -func TestActorManyMessageOneSchedule(t *testing.T) { - t.Parallel() - ctx := context.Background() - sys, router := makeTestSystem(t.Name(), t) - sys.Start(ctx) - - id := ID(777) - // To avoid blocking, use a large buffer. - size := defaultMsgBatchSizePerActor * 4 - ch := make(chan message.Message, size) - fa := &forwardActor{ - ch: ch, - } - mb := NewMailbox(id, size) - require.Nil(t, sys.Spawn(mb, fa)) - - for total := 1; total < size; total *= 2 { - for j := 0; j < total-1; j++ { - require.Nil(t, mb.Send(message.TickMessage())) - } - - // Sending to mailbox does not trigger scheduling. - select { - case msg := <-ch: - t.Fatal("Unexpected message", msg) - case <-time.After(100 * time.Millisecond): - } - - require.Nil(t, router.Send(id, message.TickMessage())) - - acc := 0 - for i := 0; i < total; i++ { - select { - case <-ch: - acc++ - case <-time.After(time.Second): - t.Fatal("Timed out, get ", acc, " expect ", total) - } - } - select { - case msg := <-ch: - t.Fatal("Unexpected message", msg, total, acc) - case <-time.After(100 * time.Millisecond): - } - } - - wait(t, func() { - err := sys.Stop() - require.Nil(t, err) - }) -} - -type flipflopActor struct { - t *testing.T - level int64 - - syncCount int - ch chan int64 - acc int64 -} - -func (f *flipflopActor) Poll(ctx context.Context, msgs []message.Message) bool { - for range msgs { - level := atomic.LoadInt64(&f.level) - newLevel := 0 - if level == 0 { - newLevel = 1 - } else { - newLevel = 0 - } - swapped := atomic.CompareAndSwapInt64(&f.level, level, int64(newLevel)) - require.True(f.t, swapped) - - if atomic.AddInt64(&f.acc, 1)%int64(f.syncCount) == 0 { - f.ch <- 0 - } - } - return true -} - -// An actor can only be polled by one goroutine at the same time. -func TestConcurrentPollSameActor(t *testing.T) { - t.Parallel() - concurrency := 4 - sys, router := NewSystemBuilder("test").WorkerNumber(concurrency).Build() - sys.Start(context.Background()) - - syncCount := 1_000_000 - ch := make(chan int64) - fa := &flipflopActor{ - t: t, - ch: ch, - syncCount: syncCount, - } - id := ID(777) - mb := NewMailbox(id, defaultMsgBatchSizePerActor) - require.Nil(t, sys.Spawn(mb, fa)) - - // Test 5 seconds - timer := time.After(5 * time.Second) - for { - total := int64(0) - for i := 0; i < syncCount; i++ { - _ = router.Send(id, message.TickMessage()) - } - total += int64(syncCount) - select { - case acc := <-ch: - require.Equal(t, total, acc) - case <-timer: - wait(t, func() { - err := sys.Stop() - require.Nil(t, err) - }) - return - } - } -} - -type closedActor struct { - acc int - ch chan int -} - -func (c *closedActor) Poll(ctx context.Context, msgs []message.Message) bool { - c.acc += len(msgs) - c.ch <- c.acc - // closed - return false -} - -func TestPollStoppedActor(t *testing.T) { - ctx := context.Background() - sys, router := makeTestSystem(t.Name(), t) - sys.Start(ctx) - - id := ID(777) - // To avoid blocking, use a large buffer. - cap := defaultMsgBatchSizePerActor * 4 - mb := NewMailbox(id, cap) - ch := make(chan int) - require.Nil(t, sys.Spawn(mb, &closedActor{ch: ch})) - - for i := 0; i < (cap - 1); i++ { - require.Nil(t, mb.Send(message.TickMessage())) - } - // Trigger scheduling - require.Nil(t, router.Send(id, message.TickMessage())) - - <-ch - select { - case <-time.After(500 * time.Millisecond): - case <-ch: - t.Fatal("must timeout") - } - wait(t, func() { - err := sys.Stop() - require.Nil(t, err) - }) -} - -func TestStoppedActorIsRemovedFromRouter(t *testing.T) { - t.Parallel() - ctx := context.Background() - sys, router := makeTestSystem(t.Name(), t) - sys.Start(ctx) - - id := ID(777) - mb := NewMailbox(id, defaultMsgBatchSizePerActor) - ch := make(chan int) - require.Nil(t, sys.Spawn(mb, &closedActor{ch: ch})) - - // Trigger scheduling - require.Nil(t, router.Send(id, message.TickMessage())) - timeout := time.After(5 * time.Second) - select { - case <-timeout: - t.Fatal("timeout") - case <-ch: - } - - for i := 0; i < 50; i++ { - // Wait for actor to be removed. - time.Sleep(100 * time.Millisecond) - err := router.Send(id, message.TickMessage()) - if strings.Contains(err.Error(), "actor not found") { - break - } - if i == 49 { - t.Fatal("actor is still in router") - } - } - - wait(t, func() { - err := sys.Stop() - require.Nil(t, err) - }) -} - -type slowActor struct { - ch chan struct{} -} - -func (c *slowActor) Poll(ctx context.Context, msgs []message.Message) bool { - c.ch <- struct{}{} - <-c.ch - // closed - return false -} - -// Test router send during actor poll and before close. -// -// ----------------------> time -// '-----------' Poll -// ' Send -// ' Close -func TestSendBeforeClose(t *testing.T) { - t.Parallel() - ctx := context.Background() - sys, router := makeTestSystem(t.Name(), t) - sys.Start(ctx) - - id := ID(777) - mb := NewMailbox(id, defaultMsgBatchSizePerActor) - ch := make(chan struct{}) - require.Nil(t, sys.Spawn(mb, &slowActor{ch: ch})) - - // Trigger scheduling - require.Nil(t, router.Send(id, message.TickMessage())) - - // Wait for actor to be polled. - a := <-ch - - // Send message before close. - err := router.Send(id, message.TickMessage()) - require.Nil(t, err) - - // Unblock poll. - ch <- a - - // Wait for actor to be removed. - for { - time.Sleep(100 * time.Millisecond) - _, ok := router.procs.Load(id) - if !ok { - break - } - } - // Must drop 1 message. - m := &dto.Metric{} - require.Nil(t, sys.rd.metricDropMessage.Write(m)) - dropped := int(*m.Counter.Value) - require.Equal(t, 1, dropped) - - // Let send and close race - // sys.rd.Lock() - - wait(t, func() { - err := sys.Stop() - require.Nil(t, err) - }) -} - -// Test router send after close and before enqueue. -// -// ----------------------> time -// '-----' Poll -// ' Close -// ' Send -// 'Enqueue -func TestSendAfterClose(t *testing.T) { - t.Parallel() - ctx := context.Background() - sys, router := makeTestSystem(t.Name(), t) - sys.Start(ctx) - - id := ID(777) - dropCount := 1 - cap := defaultMsgBatchSizePerActor + dropCount - mb := NewMailbox(id, cap) - ch := make(chan struct{}) - require.Nil(t, sys.Spawn(mb, &slowActor{ch: ch})) - pi, ok := router.procs.Load(id) - require.True(t, ok) - p := pi.(*proc) - - for i := 0; i < cap-1; i++ { - require.Nil(t, mb.Send(message.TickMessage())) - } - // Trigger scheduling - require.Nil(t, router.Send(id, message.TickMessage())) - - // Wait for actor to be polled. - a := <-ch - - // Block enqueue. - sys.rd.Lock() - - // Unblock poll. - ch <- a - - // Wait for actor to be closed. - for { - time.Sleep(100 * time.Millisecond) - if p.isClosed() { - break - } - } - - // enqueue must return actor stopped error. - err := router.rd.enqueueLocked(p, false) - require.Equal(t, errActorStopped, err) - - // Unblock enqueue. - sys.rd.Unlock() - // Wait for actor to be removed. - for { - time.Sleep(100 * time.Millisecond) - _, ok := router.procs.Load(id) - if !ok { - break - } - } - - // Must drop 1 message. - m := &dto.Metric{} - require.Nil(t, sys.rd.metricDropMessage.Write(m)) - dropped := int(*m.Counter.Value) - require.Equal(t, dropCount, dropped) - - wait(t, func() { - err := sys.Stop() - require.Nil(t, err) - }) -} - -// Run the benchmark -// go test -benchmem -run='^$' -bench '^(BenchmarkActorSendReceive)$' github.com/tikv/migration/cdc/pkg/actor -func BenchmarkActorSendReceive(b *testing.B) { - ctx := context.Background() - sys, router := makeTestSystem(b.Name(), b) - sys.Start(ctx) - - id := ID(777) - size := defaultMsgBatchSizePerActor * 4 - ch := make(chan message.Message, size) - fa := &forwardActor{ - ch: ch, - } - mb := NewMailbox(id, size) - err := sys.Spawn(mb, fa) - if err != nil { - b.Fatal(err) - } - - b.Run("BenchmarkActorSendReceive", func(b *testing.B) { - for total := 1; total <= size; total *= 2 { - b.Run(fmt.Sprintf("%d message(s)", total), func(b *testing.B) { - for i := 0; i < b.N; i++ { - for j := 0; j < total; j++ { - err = router.Send(id, message.TickMessage()) - if err != nil { - b.Fatal(err) - } - } - for j := 0; j < total; j++ { - <-ch - } - } - }) - } - }) - - if err := sys.Stop(); err != nil { - b.Fatal(err) - } -} - -// Run the benchmark -// go test -benchmem -run='^$' -bench '^(BenchmarkPollActor)$' github.com/tikv/migration/cdc/pkg/actor -func BenchmarkPollActor(b *testing.B) { - ctx := context.Background() - sys, router := makeTestSystem(b.Name(), b) - sys.Start(ctx) - - actorCount := int(math.Exp2(15)) - // To avoid blocking, use a large buffer. - ch := make(chan message.Message, actorCount) - - b.Run("BenchmarkPollActor", func(b *testing.B) { - id := 1 - for total := 1; total <= actorCount; total *= 2 { - for ; id <= total; id++ { - fa := &forwardActor{ - ch: ch, - } - mb := NewMailbox(ID(id), 1) - err := sys.Spawn(mb, fa) - if err != nil { - b.Fatal(err) - } - } - - b.ResetTimer() - b.Run(fmt.Sprintf("%d actor(s)", total), func(b *testing.B) { - for i := 0; i < b.N; i++ { - for j := 1; j <= total; j++ { - err := router.Send(ID(j), message.TickMessage()) - if err != nil { - b.Fatal(err) - } - } - for j := 1; j <= total; j++ { - <-ch - } - } - }) - b.StopTimer() - } - }) - - if err := sys.Stop(); err != nil { - b.Fatal(err) - } -} diff --git a/cdc/pkg/actor/testing.go b/cdc/pkg/actor/testing.go index ec058b08..3e48f73d 100644 --- a/cdc/pkg/actor/testing.go +++ b/cdc/pkg/actor/testing.go @@ -12,8 +12,3 @@ // limitations under the License. package actor - -// InsertMailbox4Test add a mailbox into router. Test only. -func (r *Router) InsertMailbox4Test(id ID, mb Mailbox) { - r.procs.Store(id, &proc{mb: mb}) -} diff --git a/cdc/pkg/applier/redo.go b/cdc/pkg/applier/redo.go deleted file mode 100644 index 66a04998..00000000 --- a/cdc/pkg/applier/redo.go +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package applier - -import ( - "context" - "net/url" - - "github.com/pingcap/errors" - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/redo" - "github.com/tikv/migration/cdc/cdc/redo/reader" - "github.com/tikv/migration/cdc/cdc/sink" - "github.com/tikv/migration/cdc/pkg/config" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/filter" - "go.uber.org/zap" - "golang.org/x/sync/errgroup" -) - -const ( - applierChangefeed = "redo-applier" - emitBatch = sink.DefaultMaxTxnRow - readBatch = sink.DefaultWorkerCount * emitBatch -) - -var errApplyFinished = errors.New("apply finished, can exit safely") - -// RedoApplierConfig is the configuration used by a redo log applier -type RedoApplierConfig struct { - SinkURI string - Storage string - Dir string -} - -// RedoApplier implements a redo log applier -type RedoApplier struct { - cfg *RedoApplierConfig - - rd reader.RedoLogReader - errCh chan error -} - -// NewRedoApplier creates a new RedoApplier instance -func NewRedoApplier(cfg *RedoApplierConfig) *RedoApplier { - return &RedoApplier{ - cfg: cfg, - } -} - -// toLogReaderConfig is an adapter to translate from applier config to redo reader config -// returns storageType, *reader.toLogReaderConfig and error -func (rac *RedoApplierConfig) toLogReaderConfig() (string, *reader.LogReaderConfig, error) { - uri, err := url.Parse(rac.Storage) - if err != nil { - return "", nil, cerror.WrapError(cerror.ErrConsistentStorage, err) - } - cfg := &reader.LogReaderConfig{ - Dir: uri.Path, - S3Storage: redo.IsS3StorageEnabled(uri.Scheme), - } - if cfg.S3Storage { - cfg.S3URI = *uri - // If use s3 as backend, applier will download redo logs to local dir. - cfg.Dir = rac.Dir - } - return uri.Scheme, cfg, nil -} - -func (ra *RedoApplier) catchError(ctx context.Context) error { - for { - select { - case <-ctx.Done(): - return nil - case err := <-ra.errCh: - return err - } - } -} - -func (ra *RedoApplier) consumeLogs(ctx context.Context) error { - checkpointTs, resolvedTs, err := ra.rd.ReadMeta(ctx) - if err != nil { - return err - } - err = ra.rd.ResetReader(ctx, checkpointTs, resolvedTs) - if err != nil { - return err - } - log.Info("apply redo log starts", zap.Uint64("checkpoint-ts", checkpointTs), zap.Uint64("resolved-ts", resolvedTs)) - - // MySQL sink will use the following replication config - // - EnableOldValue: default true - // - ForceReplicate: default false - // - filter: default []string{"*.*"} - replicaConfig := config.GetDefaultReplicaConfig() - ft, err := filter.NewFilter(replicaConfig) - if err != nil { - return err - } - opts := map[string]string{} - s, err := sink.New(ctx, applierChangefeed, ra.cfg.SinkURI, ft, replicaConfig, opts, ra.errCh) - if err != nil { - return err - } - defer func() { - ra.rd.Close() //nolint:errcheck - s.Close(ctx) //nolint:errcheck - }() - - // TODO: split events for large transaction - // We use lastSafeResolvedTs and lastResolvedTs to ensure the events in one - // transaction are flushed in a single batch. - // lastSafeResolvedTs records the max resolved ts of a closed transaction. - // Closed transaction means all events of this transaction have been received. - lastSafeResolvedTs := checkpointTs - 1 - // lastResolvedTs records the max resolved ts we have seen from redo logs. - lastResolvedTs := checkpointTs - cachedRows := make([]*model.RowChangedEvent, 0, emitBatch) - tableResolvedTsMap := make(map[model.TableID]model.Ts) - for { - redoLogs, err := ra.rd.ReadNextLog(ctx, readBatch) - if err != nil { - return err - } - if len(redoLogs) == 0 { - break - } - - for _, redoLog := range redoLogs { - tableID := redoLog.Row.Table.TableID - if _, ok := tableResolvedTsMap[redoLog.Row.Table.TableID]; !ok { - tableResolvedTsMap[tableID] = lastSafeResolvedTs - } - if len(cachedRows) >= emitBatch { - err := s.EmitRowChangedEvents(ctx, cachedRows...) - if err != nil { - return err - } - cachedRows = make([]*model.RowChangedEvent, 0, emitBatch) - } - cachedRows = append(cachedRows, redo.LogToRow(redoLog)) - - if redoLog.Row.CommitTs > tableResolvedTsMap[tableID] { - tableResolvedTsMap[tableID], lastResolvedTs = lastResolvedTs, redoLog.Row.CommitTs - } - } - - for tableID, tableLastResolvedTs := range tableResolvedTsMap { - _, err = s.FlushRowChangedEvents(ctx, tableID, tableLastResolvedTs) - if err != nil { - return err - } - } - } - err = s.EmitRowChangedEvents(ctx, cachedRows...) - if err != nil { - return err - } - - for tableID := range tableResolvedTsMap { - _, err = s.FlushRowChangedEvents(ctx, tableID, resolvedTs) - if err != nil { - return err - } - err = s.Barrier(ctx, tableID) - if err != nil { - return err - } - } - return errApplyFinished -} - -var createRedoReader = createRedoReaderImpl - -func createRedoReaderImpl(ctx context.Context, cfg *RedoApplierConfig) (reader.RedoLogReader, error) { - storageType, readerCfg, err := cfg.toLogReaderConfig() - if err != nil { - return nil, err - } - return redo.NewRedoReader(ctx, storageType, readerCfg) -} - -// ReadMeta creates a new redo applier and read meta from reader -func (ra *RedoApplier) ReadMeta(ctx context.Context) (checkpointTs uint64, resolvedTs uint64, err error) { - rd, err := createRedoReader(ctx, ra.cfg) - if err != nil { - return 0, 0, err - } - return rd.ReadMeta(ctx) -} - -// Apply applies redo log to given target -func (ra *RedoApplier) Apply(ctx context.Context) error { - rd, err := createRedoReader(ctx, ra.cfg) - if err != nil { - return err - } - ra.rd = rd - ra.errCh = make(chan error, 1024) - - wg, ctx := errgroup.WithContext(ctx) - wg.Go(func() error { - return ra.consumeLogs(ctx) - }) - wg.Go(func() error { - return ra.catchError(ctx) - }) - - err = wg.Wait() - if errors.Cause(err) != errApplyFinished { - return err - } - return nil -} diff --git a/cdc/pkg/applier/redo_test.go b/cdc/pkg/applier/redo_test.go deleted file mode 100644 index 9e30d847..00000000 --- a/cdc/pkg/applier/redo_test.go +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package applier - -import ( - "context" - "database/sql" - "fmt" - "testing" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/phayes/freeport" - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/redo" - "github.com/tikv/migration/cdc/cdc/redo/reader" - "github.com/tikv/migration/cdc/cdc/sink" -) - -// MockReader is a mock redo log reader that implements LogReader interface -type MockReader struct { - checkpointTs uint64 - resolvedTs uint64 - redoLogCh chan *model.RedoRowChangedEvent - ddlEventCh chan *model.RedoDDLEvent -} - -// NewMockReader creates a new MockReader -func NewMockReader( - checkpointTs uint64, - resolvedTs uint64, - redoLogCh chan *model.RedoRowChangedEvent, - ddlEventCh chan *model.RedoDDLEvent, -) *MockReader { - return &MockReader{ - checkpointTs: checkpointTs, - resolvedTs: resolvedTs, - redoLogCh: redoLogCh, - ddlEventCh: ddlEventCh, - } -} - -// ResetReader implements LogReader.ReadLog -func (br *MockReader) ResetReader(ctx context.Context, startTs, endTs uint64) error { - return nil -} - -// ReadNextLog implements LogReader.ReadNextLog -func (br *MockReader) ReadNextLog(ctx context.Context, maxNumberOfMessages uint64) ([]*model.RedoRowChangedEvent, error) { - cached := make([]*model.RedoRowChangedEvent, 0) - for { - select { - case <-ctx.Done(): - return cached, nil - case redoLog, ok := <-br.redoLogCh: - if !ok { - return cached, nil - } - cached = append(cached, redoLog) - if len(cached) >= int(maxNumberOfMessages) { - return cached, nil - } - } - } -} - -// ReadNextDDL implements LogReader.ReadNextDDL -func (br *MockReader) ReadNextDDL(ctx context.Context, maxNumberOfDDLs uint64) ([]*model.RedoDDLEvent, error) { - cached := make([]*model.RedoDDLEvent, 0) - for { - select { - case <-ctx.Done(): - return cached, nil - case ddl, ok := <-br.ddlEventCh: - if !ok { - return cached, nil - } - cached = append(cached, ddl) - if len(cached) >= int(maxNumberOfDDLs) { - return cached, nil - } - } - } -} - -// ReadMeta implements LogReader.ReadMeta -func (br *MockReader) ReadMeta(ctx context.Context) (checkpointTs, resolvedTs uint64, err error) { - return br.checkpointTs, br.resolvedTs, nil -} - -// Close implements LogReader.Close. -func (br *MockReader) Close() error { - return nil -} - -func TestApplyDMLs(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - checkpointTs := uint64(1000) - resolvedTs := uint64(2000) - redoLogCh := make(chan *model.RedoRowChangedEvent, 1024) - ddlEventCh := make(chan *model.RedoDDLEvent, 1024) - createMockReader := func(ctx context.Context, cfg *RedoApplierConfig) (reader.RedoLogReader, error) { - return NewMockReader(checkpointTs, resolvedTs, redoLogCh, ddlEventCh), nil - } - - dbIndex := 0 - mockGetDBConn := func(ctx context.Context, dsnStr string) (*sql.DB, error) { - defer func() { - dbIndex++ - }() - if dbIndex == 0 { - // mock for test db, which is used querying TiDB session variable - db, mock, err := sqlmock.New() - if err != nil { - return nil, err - } - columns := []string{"Variable_name", "Value"} - mock.ExpectQuery("show session variables like 'allow_auto_random_explicit_insert';").WillReturnRows( - sqlmock.NewRows(columns).AddRow("allow_auto_random_explicit_insert", "0"), - ) - mock.ExpectQuery("show session variables like 'tidb_txn_mode';").WillReturnRows( - sqlmock.NewRows(columns).AddRow("tidb_txn_mode", "pessimistic"), - ) - mock.ExpectClose() - return db, nil - } - // normal db - db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) - require.Nil(t, err) - mock.ExpectBegin() - mock.ExpectExec("REPLACE INTO `test`.`t1`(`a`,`b`) VALUES (?,?)"). - WithArgs(1, "2"). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - mock.ExpectBegin() - mock.ExpectExec("DELETE FROM `test`.`t1` WHERE `a` = ? LIMIT 1;"). - WithArgs(1). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectExec("REPLACE INTO `test`.`t1`(`a`,`b`) VALUES (?,?)"). - WithArgs(2, "3"). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - mock.ExpectClose() - return db, nil - } - - getDBConnBak := sink.GetDBConnImpl - sink.GetDBConnImpl = mockGetDBConn - createRedoReaderBak := createRedoReader - createRedoReader = createMockReader - defer func() { - createRedoReader = createRedoReaderBak - sink.GetDBConnImpl = getDBConnBak - }() - - dmls := []*model.RowChangedEvent{ - { - StartTs: 1100, - CommitTs: 1200, - Table: &model.TableName{Schema: "test", Table: "t1"}, - Columns: []*model.Column{ - { - Name: "a", - Value: 1, - Flag: model.HandleKeyFlag, - }, { - Name: "b", - Value: "2", - Flag: 0, - }, - }, - }, - { - StartTs: 1200, - CommitTs: 1300, - Table: &model.TableName{Schema: "test", Table: "t1"}, - PreColumns: []*model.Column{ - { - Name: "a", - Value: 1, - Flag: model.HandleKeyFlag, - }, { - Name: "b", - Value: "2", - Flag: 0, - }, - }, - Columns: []*model.Column{ - { - Name: "a", - Value: 2, - Flag: model.HandleKeyFlag, - }, { - Name: "b", - Value: "3", - Flag: 0, - }, - }, - }, - } - for _, dml := range dmls { - redoLogCh <- redo.RowToRedo(dml) - } - close(redoLogCh) - close(ddlEventCh) - - cfg := &RedoApplierConfig{SinkURI: "mysql://127.0.0.1:4000/?worker-count=1&max-txn-row=1"} - ap := NewRedoApplier(cfg) - err := ap.Apply(ctx) - require.Nil(t, err) -} - -func TestApplyMeetSinkError(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - port, err := freeport.GetFreePort() - require.Nil(t, err) - cfg := &RedoApplierConfig{ - Storage: "blackhole://", - SinkURI: fmt.Sprintf("mysql://127.0.0.1:%d/?read-timeout=1s&timeout=1s", port), - } - ap := NewRedoApplier(cfg) - err = ap.Apply(ctx) - require.Regexp(t, "CDC:ErrMySQLConnectionError", err) -} diff --git a/cdc/pkg/cmd/cli/cli_changefeed.go b/cdc/pkg/cmd/cli/cli_changefeed.go index c8176fd9..c96562f0 100644 --- a/cdc/pkg/cmd/cli/cli_changefeed.go +++ b/cdc/pkg/cmd/cli/cli_changefeed.go @@ -75,7 +75,7 @@ func newCmdChangefeed(f factory.Factory) *cobra.Command { cmds.AddCommand(newCmdCreateChangefeed(f)) cmds.AddCommand(newCmdUpdateChangefeed(f)) cmds.AddCommand(newCmdStatisticsChangefeed(f)) - cmds.AddCommand(newCmdCyclicChangefeed(f)) + // cmds.AddCommand(newCmdCyclicChangefeed(f)) cmds.AddCommand(newCmdListChangefeed(f)) cmds.AddCommand(newCmdPauseChangefeed(f)) cmds.AddCommand(newCmdQueryChangefeed(f)) diff --git a/cdc/pkg/cmd/cli/cli_changefeed_create.go b/cdc/pkg/cmd/cli/cli_changefeed_create.go index 0dc3273e..84caaabe 100644 --- a/cdc/pkg/cmd/cli/cli_changefeed_create.go +++ b/cdc/pkg/cmd/cli/cli_changefeed_create.go @@ -31,7 +31,6 @@ import ( "github.com/tikv/migration/cdc/pkg/cmd/factory" "github.com/tikv/migration/cdc/pkg/cmd/util" "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/cyclic" cerror "github.com/tikv/migration/cdc/pkg/errors" "github.com/tikv/migration/cdc/pkg/etcd" "github.com/tikv/migration/cdc/pkg/filter" @@ -391,6 +390,7 @@ func (o *createChangefeedOptions) run(ctx context.Context, cmd *cobra.Command) e } } + /* TiKV CDC don't have tables ineligibleTables, eligibleTables, err := getTables(o.pdAddr, o.credential, o.cfg, o.startTs) if err != nil { return err @@ -412,7 +412,7 @@ func (o *createChangefeedOptions) run(ctx context.Context, cmd *cobra.Command) e if o.cfg.Cyclic.IsEnabled() && !cyclic.IsTablesPaired(eligibleTables) { return errors.New("normal tables and mark tables are not paired, " + "please run `cdc cli changefeed cyclic create-marktables`") - } + } */ info := o.getInfo(cmd) diff --git a/cdc/pkg/cmd/cli/cli_changefeed_cyclic.go b/cdc/pkg/cmd/cli/cli_changefeed_cyclic.go index e66b923b..5fcfc887 100644 --- a/cdc/pkg/cmd/cli/cli_changefeed_cyclic.go +++ b/cdc/pkg/cmd/cli/cli_changefeed_cyclic.go @@ -13,9 +13,10 @@ package cli +/* import ( - "github.com/spf13/cobra" "github.com/tikv/migration/cdc/pkg/cmd/factory" + "github.com/spf13/cobra" ) // newCmdCyclicChangefeed creates the `cli changefeed cyclic` command. @@ -29,3 +30,4 @@ func newCmdCyclicChangefeed(f factory.Factory) *cobra.Command { return cmds } +*/ diff --git a/cdc/pkg/cmd/cli/cli_changefeed_cyclic_create_marktables.go b/cdc/pkg/cmd/cli/cli_changefeed_cyclic_create_marktables.go index d3bfc3dd..86477568 100644 --- a/cdc/pkg/cmd/cli/cli_changefeed_cyclic_create_marktables.go +++ b/cdc/pkg/cmd/cli/cli_changefeed_cyclic_create_marktables.go @@ -12,129 +12,3 @@ // limitations under the License. package cli - -import ( - "github.com/spf13/cobra" - "github.com/tikv/client-go/v2/oracle" - "github.com/tikv/migration/cdc/pkg/cmd/context" - "github.com/tikv/migration/cdc/pkg/cmd/factory" - "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/cyclic/mark" - "github.com/tikv/migration/cdc/pkg/security" - pd "github.com/tikv/pd/client" -) - -// cyclicCreateMarktablesOptions defines flags for the `cli changefeed cyclic create-marktables` command. -type cyclicCreateMarktablesOptions struct { - createCommonOptions changefeedCommonOptions - - pdClient pd.Client - - pdAddr string - credential *security.Credential - - startTs uint64 - cyclicUpstreamDSN string - upstreamSslCaPath string - upstreamSslCertPath string - upstreamSslKeyPath string -} - -// newCyclicCreateMarktablesOptions creates new options for the `cli changefeed cyclic create-marktables` command. -func newCyclicCreateMarktablesOptions() *cyclicCreateMarktablesOptions { - return &cyclicCreateMarktablesOptions{} -} - -func (o *cyclicCreateMarktablesOptions) getUpstreamCredential() *security.Credential { - return &security.Credential{ - CAPath: o.upstreamSslCaPath, - CertPath: o.upstreamSslCertPath, - KeyPath: o.upstreamSslKeyPath, - } -} - -// addFlags receives a *cobra.Command reference and binds -// flags related to template printing to it. -func (o *cyclicCreateMarktablesOptions) addFlags(cmd *cobra.Command) { - cmd.PersistentFlags().Uint64Var(&o.startTs, "start-ts", 0, "Start ts of changefeed") - cmd.PersistentFlags().StringVar(&o.cyclicUpstreamDSN, "cyclic-upstream-dsn", "", "(Expremental) Upstream TiDB DSN in the form of [user[:password]@][net[(addr)]]/") - cmd.PersistentFlags().StringVar(&o.upstreamSslCaPath, "cyclic-upstream-ssl-ca", "", "CA certificate path for TLS connection") - cmd.PersistentFlags().StringVar(&o.upstreamSslCertPath, "cyclic-upstream-ssl-cert", "", "Certificate path for TLS connection") - cmd.PersistentFlags().StringVar(&o.upstreamSslKeyPath, "cyclic-upstream-ssl-key", "", "Private key path for TLS connection") -} - -// complete adapts from the command line args to the data and client required. -func (o *cyclicCreateMarktablesOptions) complete(f factory.Factory) error { - pdClient, err := f.PdClient() - if err != nil { - return err - } - - o.pdClient = pdClient - - o.pdAddr = f.GetPdAddr() - o.credential = f.GetCredential() - - return nil -} - -// run the `cli changefeed cyclic create-marktables` command. -func (o *cyclicCreateMarktablesOptions) run(cmd *cobra.Command) error { - ctx := context.GetDefaultContext() - - cfg := config.GetDefaultReplicaConfig() - if len(o.createCommonOptions.configFile) > 0 { - if err := o.createCommonOptions.strictDecodeConfig("TiCDC changefeed", cfg); err != nil { - return err - } - } - - ts, logical, err := o.pdClient.GetTS(ctx) - if err != nil { - return err - } - o.startTs = oracle.ComposeTS(ts, logical) - - _, eligibleTables, err := getTables(o.pdAddr, o.credential, cfg, o.startTs) - if err != nil { - return err - } - - tables := make([]mark.TableName, len(eligibleTables)) - for i := range eligibleTables { - tables[i] = &eligibleTables[i] - } - - err = mark.CreateMarkTables(ctx, o.cyclicUpstreamDSN, o.getUpstreamCredential(), tables...) - if err != nil { - return err - } - - cmd.Printf("Create cyclic replication mark tables successfully! Total tables: %d\n", len(eligibleTables)) - - return nil -} - -// newCmdCyclicCreateMarktables creates the `cli changefeed cyclic create-marktables` command. -func newCmdCyclicCreateMarktables(f factory.Factory) *cobra.Command { - o := newCyclicCreateMarktablesOptions() - - command := - &cobra.Command{ - Use: "create-marktables", - Short: "Create cyclic replication mark tables", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - err := o.complete(f) - if err != nil { - return err - } - - return o.run(cmd) - }, - } - - o.addFlags(command) - - return command -} diff --git a/cdc/pkg/cmd/cli/cli_changefeed_helper.go b/cdc/pkg/cmd/cli/cli_changefeed_helper.go index 50ff610a..dfe199f1 100644 --- a/cdc/pkg/cmd/cli/cli_changefeed_helper.go +++ b/cdc/pkg/cmd/cli/cli_changefeed_helper.go @@ -24,13 +24,9 @@ import ( "github.com/spf13/cobra" "github.com/tikv/client-go/v2/oracle" "github.com/tikv/migration/cdc/cdc" - "github.com/tikv/migration/cdc/cdc/entry" - "github.com/tikv/migration/cdc/cdc/kv" "github.com/tikv/migration/cdc/cdc/model" "github.com/tikv/migration/cdc/pkg/cmd/util" - "github.com/tikv/migration/cdc/pkg/config" "github.com/tikv/migration/cdc/pkg/etcd" - "github.com/tikv/migration/cdc/pkg/filter" "github.com/tikv/migration/cdc/pkg/httputil" "github.com/tikv/migration/cdc/pkg/security" ) @@ -63,6 +59,7 @@ func confirmLargeDataGap(cmd *cobra.Command, currentPhysical int64, startTs uint return nil } +/* // confirmIgnoreIneligibleTables confirm if user need to ignore ineligible tables. func confirmIgnoreIneligibleTables(cmd *cobra.Command) error { cmd.Printf("Could you agree to ignore those tables, and continue to replicate [Y/N]\n") @@ -96,7 +93,7 @@ func getTables(cliPdAddr string, credential *security.Credential, cfg *config.Re return nil, nil, errors.Trace(err) } - snap, err := entry.NewSingleSchemaSnapshotFromMeta(meta, startTs, false /* explicitTables */) + snap, err := entry.NewSingleSchemaSnapshotFromMeta(meta, startTs, false) if err != nil { return nil, nil, errors.Trace(err) } @@ -105,7 +102,7 @@ func getTables(cliPdAddr string, credential *security.Credential, cfg *config.Re if filter.ShouldIgnoreTable(tableInfo.TableName.Schema, tableInfo.TableName.Table) { continue } - if !tableInfo.IsEligible(false /* forceReplicate */) { + if !tableInfo.IsEligible(false) { ineligibleTables = append(ineligibleTables, tableInfo.TableName) } else { eligibleTables = append(eligibleTables, tableInfo.TableName) @@ -114,6 +111,7 @@ func getTables(cliPdAddr string, credential *security.Credential, cfg *config.Re return } +*/ // sendOwnerChangefeedQuery sends owner changefeed query request. func sendOwnerChangefeedQuery(ctx context.Context, etcdClient *etcd.CDCEtcdClient, diff --git a/cdc/pkg/cmd/cli/cli_changefeed_helper_test.go b/cdc/pkg/cmd/cli/cli_changefeed_helper_test.go index 97c72808..c7e00113 100644 --- a/cdc/pkg/cmd/cli/cli_changefeed_helper_test.go +++ b/cdc/pkg/cmd/cli/cli_changefeed_helper_test.go @@ -58,34 +58,3 @@ func (s *changefeedHelperSuite) TestConfirmLargeDataGap(c *check.C) { err = confirmLargeDataGap(cmd, currentTs, startTs) c.Assert(err, check.IsNil) } - -func (s *changefeedHelperSuite) TestConfirmIgnoreIneligibleTables(c *check.C) { - defer testleak.AfterTest(c)() - - cmd := &cobra.Command{} - - // check start ts more than 1 day before current ts, and type N when confirming - dir := c.MkDir() - path := filepath.Join(dir, "confirm.txt") - err := os.WriteFile(path, []byte("n"), 0o644) - c.Assert(err, check.IsNil) - f, err := os.Open(path) - c.Assert(err, check.IsNil) - stdin := os.Stdin - os.Stdin = f - defer func() { - os.Stdin = stdin - }() - - err = confirmIgnoreIneligibleTables(cmd) - c.Assert(err, check.ErrorMatches, "abort changefeed create or resume") - - // check start ts more than 1 day before current ts, and type Y when confirming - err = os.WriteFile(path, []byte("Y"), 0o644) - c.Assert(err, check.IsNil) - f, err = os.Open(path) - c.Assert(err, check.IsNil) - os.Stdin = f - err = confirmIgnoreIneligibleTables(cmd) - c.Assert(err, check.IsNil) -} diff --git a/cdc/pkg/cmd/cli/cli_changefeed_query.go b/cdc/pkg/cmd/cli/cli_changefeed_query.go index 1499603a..82d7b7aa 100644 --- a/cdc/pkg/cmd/cli/cli_changefeed_query.go +++ b/cdc/pkg/cmd/cli/cli_changefeed_query.go @@ -154,6 +154,5 @@ func newCmdQueryChangefeed(f factory.Factory) *cobra.Command { } o.addFlags(command) - return command } diff --git a/cdc/pkg/cmd/cli/cli_processor_query.go b/cdc/pkg/cmd/cli/cli_processor_query.go index 15c3c347..52398256 100644 --- a/cdc/pkg/cmd/cli/cli_processor_query.go +++ b/cdc/pkg/cmd/cli/cli_processor_query.go @@ -121,9 +121,9 @@ func (o *queryProcessorOptions) runCliWithAPIClient(ctx context.Context, cmd *co return err } - tables := make(map[int64]*model.TableReplicaInfo) - for _, tableID := range processor.Tables { - tables[tableID] = &model.TableReplicaInfo{ + keyspans := make(map[uint64]*model.KeySpanReplicaInfo) + for _, keyspanID := range processor.KeySpans { + keyspans[keyspanID] = &model.KeySpanReplicaInfo{ // to be compatible with old version `cli processor query`, // set this field to 0 StartTs: 0, @@ -132,7 +132,7 @@ func (o *queryProcessorOptions) runCliWithAPIClient(ctx context.Context, cmd *co meta := &processorMeta{ Status: &model.TaskStatus{ - Tables: tables, + KeySpans: keyspans, // Operations, AdminJobType and ModRevision are vacant }, Position: &model.TaskPosition{ diff --git a/cdc/pkg/cmd/cmd.go b/cdc/pkg/cmd/cmd.go index bef3f71f..0c3e666a 100644 --- a/cdc/pkg/cmd/cmd.go +++ b/cdc/pkg/cmd/cmd.go @@ -18,7 +18,6 @@ import ( "github.com/spf13/cobra" "github.com/tikv/migration/cdc/pkg/cmd/cli" - "github.com/tikv/migration/cdc/pkg/cmd/redo" "github.com/tikv/migration/cdc/pkg/cmd/server" "github.com/tikv/migration/cdc/pkg/cmd/version" ) @@ -45,7 +44,7 @@ func Run() { cmd.AddCommand(server.NewCmdServer()) cmd.AddCommand(cli.NewCmdCli()) cmd.AddCommand(version.NewCmdVersion()) - cmd.AddCommand(redo.NewCmdRedo()) + // cmd.AddCommand(redo.NewCmdRedo()) if err := cmd.Execute(); err != nil { cmd.PrintErrln(err) diff --git a/cdc/pkg/cmd/redo/apply.go b/cdc/pkg/cmd/redo/apply.go index 3807c13f..ef491d2a 100644 --- a/cdc/pkg/cmd/redo/apply.go +++ b/cdc/pkg/cmd/redo/apply.go @@ -12,62 +12,3 @@ // limitations under the License. package redo - -import ( - "github.com/spf13/cobra" - "github.com/tikv/migration/cdc/pkg/applier" - cmdcontext "github.com/tikv/migration/cdc/pkg/cmd/context" -) - -// applyRedoOptions defines flags for the `redo apply` command. -type applyRedoOptions struct { - options - sinkURI string -} - -// newapplyRedoOptions creates new applyRedoOptions for the `redo apply` command. -func newapplyRedoOptions() *applyRedoOptions { - return &applyRedoOptions{} -} - -// addFlags receives a *cobra.Command reference and binds -// flags related to template printing to it. -func (o *applyRedoOptions) addFlags(cmd *cobra.Command) { - cmd.Flags().StringVar(&o.sinkURI, "sink-uri", "", "target database sink-uri") - // the possible error returned from MarkFlagRequired is `no such flag` - cmd.MarkFlagRequired("sink-uri") //nolint:errcheck -} - -// run runs the `redo apply` command. -func (o *applyRedoOptions) run(cmd *cobra.Command) error { - ctx := cmdcontext.GetDefaultContext() - - cfg := &applier.RedoApplierConfig{ - Storage: o.storage, - SinkURI: o.sinkURI, - Dir: o.dir, - } - ap := applier.NewRedoApplier(cfg) - err := ap.Apply(ctx) - if err != nil { - return err - } - cmd.Println("Apply redo log successfully") - return nil -} - -// newCmdApply creates the `redo apply` command. -func newCmdApply(opt *options) *cobra.Command { - o := newapplyRedoOptions() - command := &cobra.Command{ - Use: "apply", - Short: "Apply redo logs in target sink", - RunE: func(cmd *cobra.Command, args []string) error { - o.options = *opt - return o.run(cmd) - }, - } - o.addFlags(command) - - return command -} diff --git a/cdc/pkg/cmd/redo/meta.go b/cdc/pkg/cmd/redo/meta.go index 5d2b61d8..ef491d2a 100644 --- a/cdc/pkg/cmd/redo/meta.go +++ b/cdc/pkg/cmd/redo/meta.go @@ -12,51 +12,3 @@ // limitations under the License. package redo - -import ( - "github.com/spf13/cobra" - "github.com/tikv/migration/cdc/pkg/applier" - cmdcontext "github.com/tikv/migration/cdc/pkg/cmd/context" -) - -// metaOptions defines flags for the `redo meta` command. -type metaOptions struct { - options -} - -// newMetaOptions creates new MetaOptions for the `redo apply` command. -func newMetaOptions() *metaOptions { - return &metaOptions{} -} - -// run runs the `redo apply` command. -func (o *metaOptions) run(cmd *cobra.Command) error { - ctx := cmdcontext.GetDefaultContext() - - cfg := &applier.RedoApplierConfig{ - Storage: o.storage, - Dir: o.dir, - } - ap := applier.NewRedoApplier(cfg) - checkpointTs, resolvedTs, err := ap.ReadMeta(ctx) - if err != nil { - return err - } - cmd.Printf("checkpoint-ts:%d, resolved-ts:%d\n", checkpointTs, resolvedTs) - return nil -} - -// newCmdMeta creates the `redo meta` command. -func newCmdMeta(opt *options) *cobra.Command { - command := &cobra.Command{ - Use: "meta", - Short: "read redo log meta", - RunE: func(cmd *cobra.Command, args []string) error { - o := newMetaOptions() - o.options = *opt - return o.run(cmd) - }, - } - - return command -} diff --git a/cdc/pkg/cmd/redo/redo.go b/cdc/pkg/cmd/redo/redo.go index 21c07660..ef491d2a 100644 --- a/cdc/pkg/cmd/redo/redo.go +++ b/cdc/pkg/cmd/redo/redo.go @@ -12,57 +12,3 @@ // limitations under the License. package redo - -import ( - "github.com/spf13/cobra" - "github.com/tikv/migration/cdc/pkg/cmd/util" - "github.com/tikv/migration/cdc/pkg/logutil" -) - -// options defines flags for the `redo` command. -type options struct { - storage string - dir string - logLevel string -} - -// newOptions creates new options for the `server` command. -func newOptions() *options { - return &options{} -} - -// addFlags receives a *cobra.Command reference and binds -// flags related to template printing to it. -func (o *options) addFlags(cmd *cobra.Command) { - cmd.PersistentFlags().StringVar(&o.storage, "storage", "", "storage of redo log, specify the url where backup redo logs will store, eg, \"s3://bucket/path/prefix\"") - cmd.PersistentFlags().StringVar(&o.dir, "tmp-dir", "", "temporary path used to download redo log with S3 backend") - cmd.PersistentFlags().StringVar(&o.logLevel, "log-level", "info", "log level (etc: debug|info|warn|error)") - // the possible error returned from MarkFlagRequired is `no such flag` - cmd.MarkFlagRequired("storage") //nolint:errcheck -} - -// NewCmdRedo creates the `redo` command. -func NewCmdRedo() *cobra.Command { - o := newOptions() - - cmds := &cobra.Command{ - Use: "redo", - Short: "Manage redo logs of TiCDC cluster", - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - // Here we will initialize the logging configuration and set the current default context. - util.InitCmd(cmd, &logutil.Config{Level: o.logLevel}) - util.LogHTTPProxies() - - return nil - }, - Run: func(cmd *cobra.Command, args []string) { - }, - } - o.addFlags(cmds) - - // Add subcommands. - cmds.AddCommand(newCmdApply(o)) - cmds.AddCommand(newCmdMeta(o)) - - return cmds -} diff --git a/cdc/pkg/cmd/server/server.go b/cdc/pkg/cmd/server/server.go index 9f516f9b..b57c541e 100644 --- a/cdc/pkg/cmd/server/server.go +++ b/cdc/pkg/cmd/server/server.go @@ -23,10 +23,11 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/log" ticonfig "github.com/pingcap/tidb/config" + "github.com/tikv/migration/cdc/cdc" + + // "github.com/tikv/migration/cdc/cdc/sorter/unified" "github.com/spf13/cobra" "github.com/spf13/pflag" - "github.com/tikv/migration/cdc/cdc" - "github.com/tikv/migration/cdc/cdc/sorter/unified" cmdcontext "github.com/tikv/migration/cdc/pkg/cmd/context" "github.com/tikv/migration/cdc/pkg/cmd/util" "github.com/tikv/migration/cdc/pkg/config" @@ -132,7 +133,7 @@ func (o *options) run(cmd *cobra.Command) error { return errors.Annotate(err, "run server") } server.Close() - unified.CleanUp() + // unified.CleanUp() log.Info("cdc server exits successfully") return nil diff --git a/cdc/pkg/cmd/server/server_test.go b/cdc/pkg/cmd/server/server_test.go index fb40444c..700011fe 100644 --- a/cdc/pkg/cmd/server/server_test.go +++ b/cdc/pkg/cmd/server/server_test.go @@ -168,15 +168,15 @@ func TestParseCfg(t *testing.T) { KeyPath: "cc", CertAllowedCN: []string{"dd", "ee"}, }, - PerTableMemoryQuota: 10 * 1024 * 1024, // 10M + PerKeySpanMemoryQuota: 10 * 1024 * 1024, // 10M KVClient: &config.KVClientConfig{ WorkerConcurrent: 8, WorkerPoolSize: 0, RegionScanLimit: 40, }, Debug: &config.DebugConfig{ - EnableTableActor: false, - EnableDBSorter: false, + EnableKeySpanActor: false, + EnableDBSorter: false, DB: &config.DBConfig{ Count: 8, Concurrency: 128, @@ -306,16 +306,16 @@ server-worker-pool-size = 16 NumWorkerPoolGoroutine: 5, SortDir: config.DefaultSortDir, }, - Security: &config.SecurityConfig{}, - PerTableMemoryQuota: 10 * 1024 * 1024, // 10M + Security: &config.SecurityConfig{}, + PerKeySpanMemoryQuota: 10 * 1024 * 1024, // 10M KVClient: &config.KVClientConfig{ WorkerConcurrent: 8, WorkerPoolSize: 0, RegionScanLimit: 40, }, Debug: &config.DebugConfig{ - EnableTableActor: false, - EnableDBSorter: false, + EnableKeySpanActor: false, + EnableDBSorter: false, DB: &config.DBConfig{ Count: 5, Concurrency: 6, @@ -444,15 +444,15 @@ cert-allowed-cn = ["dd","ee"] KeyPath: "cc", CertAllowedCN: []string{"dd", "ee"}, }, - PerTableMemoryQuota: 10 * 1024 * 1024, // 10M + PerKeySpanMemoryQuota: 10 * 1024 * 1024, // 10M KVClient: &config.KVClientConfig{ WorkerConcurrent: 8, WorkerPoolSize: 0, RegionScanLimit: 40, }, Debug: &config.DebugConfig{ - EnableTableActor: false, - EnableDBSorter: false, + EnableKeySpanActor: false, + EnableDBSorter: false, DB: &config.DBConfig{ Count: 8, Concurrency: 128, @@ -507,8 +507,8 @@ unknown3 = 3 err = o.validate() require.Nil(t, err) require.Equal(t, &config.DebugConfig{ - EnableTableActor: false, - EnableDBSorter: false, + EnableKeySpanActor: false, + EnableDBSorter: false, DB: &config.DBConfig{ Count: 8, Concurrency: 128, diff --git a/cdc/pkg/config/config_test_data.go b/cdc/pkg/config/config_test_data.go index 7c8d254f..5d19fe43 100644 --- a/cdc/pkg/config/config_test_data.go +++ b/cdc/pkg/config/config_test_data.go @@ -57,7 +57,7 @@ const ( "sync-ddl": false }, "scheduler": { - "type": "table-number", + "type": "keyspan-number", "polling-time": -1 }, "consistent": { @@ -101,14 +101,14 @@ const ( "key-path": "", "cert-allowed-cn": null }, - "per-table-memory-quota": 10485760, + "per-keyspan-memory-quota": 10485760, "kv-client": { "worker-concurrent": 8, "worker-pool-size": 0, "region-scan-limit": 40 }, "debug": { - "enable-table-actor": false, + "enable-keyspan-actor": false, "enable-db-sorter": false, "db": { "count": 8, @@ -177,7 +177,7 @@ const ( "sync-ddl": false }, "scheduler": { - "type": "table-number", + "type": "keyspan-number", "polling-time": -1 }, "consistent": { @@ -225,7 +225,7 @@ const ( "sync-ddl": false }, "scheduler": { - "type": "table-number", + "type": "keyspan-number", "polling-time": -1 }, "consistent": { diff --git a/cdc/pkg/config/debug.go b/cdc/pkg/config/debug.go index 73ccca44..daa1e62c 100644 --- a/cdc/pkg/config/debug.go +++ b/cdc/pkg/config/debug.go @@ -17,9 +17,9 @@ import "github.com/pingcap/errors" // DebugConfig represents config for ticdc unexposed feature configurations type DebugConfig struct { - // identify if the table actor is enabled for table pipeline + // identify if the keyspan actor is enabled for keyspan pipeline // TODO: turn on after GA. - EnableTableActor bool `toml:"enable-table-actor" json:"enable-table-actor"` + EnableKeySpanActor bool `toml:"enable-keyspan-actor" json:"enable-keyspan-actor"` // EnableDBSorter enables db sorter. // diff --git a/cdc/pkg/config/replica_config.go b/cdc/pkg/config/replica_config.go index 6793c116..e45f34f4 100644 --- a/cdc/pkg/config/replica_config.go +++ b/cdc/pkg/config/replica_config.go @@ -40,7 +40,7 @@ var defaultReplicaConfig = &ReplicaConfig{ Enable: false, }, Scheduler: &SchedulerConfig{ - Tp: "table-number", + Tp: "keyspan-number", PollingTime: -1, }, Consistent: &ConsistentConfig{ @@ -55,16 +55,17 @@ var defaultReplicaConfig = &ReplicaConfig{ type ReplicaConfig replicaConfig type replicaConfig struct { - CaseSensitive bool `toml:"case-sensitive" json:"case-sensitive"` - EnableOldValue bool `toml:"enable-old-value" json:"enable-old-value"` - ForceReplicate bool `toml:"force-replicate" json:"force-replicate"` - CheckGCSafePoint bool `toml:"check-gc-safe-point" json:"check-gc-safe-point"` - Filter *FilterConfig `toml:"filter" json:"filter"` - Mounter *MounterConfig `toml:"mounter" json:"mounter"` - Sink *SinkConfig `toml:"sink" json:"sink"` - Cyclic *CyclicConfig `toml:"cyclic-replication" json:"cyclic-replication"` - Scheduler *SchedulerConfig `toml:"scheduler" json:"scheduler"` - Consistent *ConsistentConfig `toml:"consistent" json:"consistent"` + CaseSensitive bool `toml:"case-sensitive" json:"case-sensitive"` + EnableOldValue bool `toml:"enable-old-value" json:"enable-old-value"` + ForceReplicate bool `toml:"force-replicate" json:"force-replicate"` + CheckGCSafePoint bool `toml:"check-gc-safe-point" json:"check-gc-safe-point"` + // TODO(zeminzhou): Maybe TiKV CDC don't need this + Filter *FilterConfig `toml:"filter" json:"filter"` + Mounter *MounterConfig `toml:"mounter" json:"mounter"` + Sink *SinkConfig `toml:"sink" json:"sink"` + Cyclic *CyclicConfig `toml:"cyclic-replication" json:"cyclic-replication"` + Scheduler *SchedulerConfig `toml:"scheduler" json:"scheduler"` + Consistent *ConsistentConfig `toml:"consistent" json:"consistent"` } // Marshal returns the json marshal format of a ReplicationConfig diff --git a/cdc/pkg/config/server_config.go b/cdc/pkg/config/server_config.go index 5491d434..4b135492 100644 --- a/cdc/pkg/config/server_config.go +++ b/cdc/pkg/config/server_config.go @@ -90,15 +90,15 @@ var defaultServerConfig = &ServerConfig{ NumWorkerPoolGoroutine: 16, SortDir: DefaultSortDir, }, - Security: &SecurityConfig{}, - PerTableMemoryQuota: 10 * 1024 * 1024, // 10MB + Security: &SecurityConfig{}, + PerKeySpanMemoryQuota: 10 * 1024 * 1024, // 10MB KVClient: &KVClientConfig{ WorkerConcurrent: 8, WorkerPoolSize: 0, // 0 will use NumCPU() * 2 RegionScanLimit: 40, }, Debug: &DebugConfig{ - EnableTableActor: false, + EnableKeySpanActor: false, EnableNewScheduler: false, // Default leveldb sorter config EnableDBSorter: false, @@ -144,11 +144,11 @@ type ServerConfig struct { OwnerFlushInterval TomlDuration `toml:"owner-flush-interval" json:"owner-flush-interval"` ProcessorFlushInterval TomlDuration `toml:"processor-flush-interval" json:"processor-flush-interval"` - Sorter *SorterConfig `toml:"sorter" json:"sorter"` - Security *SecurityConfig `toml:"security" json:"security"` - PerTableMemoryQuota uint64 `toml:"per-table-memory-quota" json:"per-table-memory-quota"` - KVClient *KVClientConfig `toml:"kv-client" json:"kv-client"` - Debug *DebugConfig `toml:"debug" json:"debug"` + Sorter *SorterConfig `toml:"sorter" json:"sorter"` + Security *SecurityConfig `toml:"security" json:"security"` + PerKeySpanMemoryQuota uint64 `toml:"per-keyspan-memory-quota" json:"per-keyspan-memory-quota"` + KVClient *KVClientConfig `toml:"kv-client" json:"kv-client"` + Debug *DebugConfig `toml:"debug" json:"debug"` } // Marshal returns the json marshal format of a ServerConfig @@ -240,8 +240,8 @@ func (c *ServerConfig) ValidateAndAdjust() error { return err } - if c.PerTableMemoryQuota == 0 { - c.PerTableMemoryQuota = defaultCfg.PerTableMemoryQuota + if c.PerKeySpanMemoryQuota == 0 { + c.PerKeySpanMemoryQuota = defaultCfg.PerKeySpanMemoryQuota } if c.KVClient == nil { diff --git a/cdc/pkg/config/server_config_test.go b/cdc/pkg/config/server_config_test.go index b164defe..541d5ab7 100644 --- a/cdc/pkg/config/server_config_test.go +++ b/cdc/pkg/config/server_config_test.go @@ -68,12 +68,12 @@ func TestServerConfigValidateAndAdjust(t *testing.T) { conf.AdvertiseAddr = "advertise" require.Regexp(t, ".*does not contain a port", conf.ValidateAndAdjust()) conf.AdvertiseAddr = "advertise:1234" - conf.PerTableMemoryQuota = 1 + conf.PerKeySpanMemoryQuota = 1 require.Nil(t, conf.ValidateAndAdjust()) - require.EqualValues(t, 1, conf.PerTableMemoryQuota) - conf.PerTableMemoryQuota = 0 + require.EqualValues(t, 1, conf.PerKeySpanMemoryQuota) + conf.PerKeySpanMemoryQuota = 0 require.Nil(t, conf.ValidateAndAdjust()) - require.EqualValues(t, GetDefaultServerConfig().PerTableMemoryQuota, conf.PerTableMemoryQuota) + require.EqualValues(t, GetDefaultServerConfig().PerKeySpanMemoryQuota, conf.PerKeySpanMemoryQuota) conf.Debug.Messages.ServerWorkerPoolSize = 0 require.Nil(t, conf.ValidateAndAdjust()) require.EqualValues(t, GetDefaultServerConfig().Debug.Messages.ServerWorkerPoolSize, conf.Debug.Messages.ServerWorkerPoolSize) diff --git a/cdc/pkg/context/context.go b/cdc/pkg/context/context.go index 3d0a472e..acfd71cf 100644 --- a/cdc/pkg/context/context.go +++ b/cdc/pkg/context/context.go @@ -23,8 +23,8 @@ import ( "github.com/tikv/client-go/v2/tikv" "github.com/tikv/migration/cdc/cdc/kv" "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/cdc/processor/pipeline/system" - ssystem "github.com/tikv/migration/cdc/cdc/sorter/leveldb/system" + + // ssystem "github.com/tikv/migration/cdc/cdc/sorter/leveldb/system" "github.com/tikv/migration/cdc/pkg/config" "github.com/tikv/migration/cdc/pkg/etcd" "github.com/tikv/migration/cdc/pkg/p2p" @@ -38,15 +38,15 @@ import ( // the lifecycle of vars in the GlobalVars should be aligned with the ticdc server process. // All field in Vars should be READ-ONLY and THREAD-SAFE type GlobalVars struct { - PDClient pd.Client - KVStorage tidbkv.Storage - CaptureInfo *model.CaptureInfo - EtcdClient *etcd.CDCEtcdClient - GrpcPool kv.GrpcPool - RegionCache *tikv.RegionCache - TimeAcquirer pdtime.TimeAcquirer - TableActorSystem *system.System - SorterSystem *ssystem.System + PDClient pd.Client + KVStorage tidbkv.Storage + CaptureInfo *model.CaptureInfo + EtcdClient *etcd.CDCEtcdClient + GrpcPool kv.GrpcPool + RegionCache *tikv.RegionCache + TimeAcquirer pdtime.TimeAcquirer + // KeySpanActorSystem *system.System + // SorterSystem *ssystem.System // OwnerRevision is the Etcd revision when the owner got elected. OwnerRevision int64 diff --git a/cdc/pkg/cyclic/filter.go b/cdc/pkg/cyclic/filter.go deleted file mode 100644 index 7232a3f8..00000000 --- a/cdc/pkg/cyclic/filter.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package cyclic - -import ( - "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/cyclic/mark" - "go.uber.org/zap" -) - -// ExtractReplicaID extracts replica ID from the given mark row. -func ExtractReplicaID(markRow *model.RowChangedEvent) uint64 { - for _, c := range markRow.Columns { - if c == nil { - continue - } - if c.Name == mark.CyclicReplicaIDCol { - return c.Value.(uint64) - } - } - log.Panic("bad mark table, " + mark.CyclicReplicaIDCol + " not found") - return 0 -} - -// TxnMap maps start ts to txn may cross multiple tables. -type TxnMap map[uint64]map[model.TableName][]*model.RowChangedEvent - -// MarkMap maps start ts to mark table rows. -// There is at most one mark table row that is modified for each transaction. -type MarkMap map[uint64]*model.RowChangedEvent - -func (m MarkMap) shouldFilterTxn(startTs uint64, filterReplicaIDs []uint64, replicaID uint64) (*model.RowChangedEvent, bool) { - markRow, markFound := m[startTs] - if !markFound { - return nil, false - } - from := ExtractReplicaID(markRow) - if from == replicaID { - log.Panic("cyclic replication loopback detected", - zap.Any("markRow", markRow), - zap.Uint64("replicaID", replicaID)) - } - for i := range filterReplicaIDs { - if filterReplicaIDs[i] == from { - return markRow, true - } - } - return markRow, false -} - -// FilterAndReduceTxns filters duplicate txns bases on filterReplicaIDs and -// if the mark table dml is exist in the txn, this function will set the replicaID by mark table dml -// if the mark table dml is not exist, this function will set the replicaID by config -func FilterAndReduceTxns( - txnsMap map[model.TableID][]*model.SingleTableTxn, filterReplicaIDs []uint64, replicaID uint64, -) (skippedRowCount int) { - markMap := make(MarkMap) - for _, txns := range txnsMap { - if !mark.IsMarkTable(txns[0].Table.Schema, txns[0].Table.Table) { - continue - } - for _, txn := range txns { - for _, event := range txn.Rows { - first, ok := markMap[txn.StartTs] - if ok { - // TiKV may emit the same row multiple times. - if event.CommitTs != first.CommitTs || - event.RowID != first.RowID { - log.Panic( - "there should be at most one mark row for each txn", - zap.Uint64("start-ts", event.StartTs), - zap.Any("first", first), - zap.Any("second", event)) - } - } - markMap[event.StartTs] = event - } - } - } - for table, txns := range txnsMap { - if mark.IsMarkTable(txns[0].Table.Schema, txns[0].Table.Table) { - delete(txnsMap, table) - for i := range txns { - // For simplicity, we do not count mark table rows in statistics. - skippedRowCount += len(txns[i].Rows) - } - continue - } - filteredTxns := make([]*model.SingleTableTxn, 0, len(txns)) - for _, txn := range txns { - // Check if we should skip this event - markRow, needSkip := markMap.shouldFilterTxn(txn.StartTs, filterReplicaIDs, replicaID) - if needSkip { - // Found cyclic mark, skip this event as it originly created from - // downstream. - skippedRowCount += len(txn.Rows) - continue - } - txn.ReplicaID = replicaID - if markRow != nil { - txn.ReplicaID = ExtractReplicaID(markRow) - } - filteredTxns = append(filteredTxns, txn) - } - if len(filteredTxns) == 0 { - delete(txnsMap, table) - } else { - txnsMap[table] = filteredTxns - } - } - return -} diff --git a/cdc/pkg/cyclic/filter_test.go b/cdc/pkg/cyclic/filter_test.go deleted file mode 100644 index 55784f04..00000000 --- a/cdc/pkg/cyclic/filter_test.go +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package cyclic - -import ( - "testing" - - "github.com/davecgh/go-spew/spew" - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/cyclic/mark" -) - -func TestFilterAndReduceTxns(t *testing.T) { - t.Parallel() - rID := mark.CyclicReplicaIDCol - testCases := []struct { - input map[model.TableID][]*model.SingleTableTxn - output map[model.TableID][]*model.SingleTableTxn - filterID []uint64 - replicaID uint64 - }{ - { - input: map[model.TableID][]*model.SingleTableTxn{}, - output: map[model.TableID][]*model.SingleTableTxn{}, - filterID: []uint64{}, - replicaID: 0, - }, - { - input: map[model.TableID][]*model.SingleTableTxn{1: {{Table: &model.TableName{Table: "a"}, StartTs: 1}}}, - output: map[model.TableID][]*model.SingleTableTxn{1: {{Table: &model.TableName{Table: "a"}, StartTs: 1, ReplicaID: 1}}}, - filterID: []uint64{}, - replicaID: 1, - }, - { - input: map[model.TableID][]*model.SingleTableTxn{ - 2: { - { - Table: &model.TableName{Schema: "tidb_cdc"}, /* cyclic.SchemaName */ - StartTs: 1, - Rows: []*model.RowChangedEvent{{StartTs: 1, Columns: []*model.Column{{Name: rID, Value: uint64(10)}}}}, - }, - }, - }, - output: map[model.TableID][]*model.SingleTableTxn{}, - filterID: []uint64{}, - replicaID: 1, - }, - { - input: map[model.TableID][]*model.SingleTableTxn{ - 1: {{Table: &model.TableName{Table: "a"}, StartTs: 1}}, - 2: { - { - Table: &model.TableName{Schema: "tidb_cdc"}, /* cyclic.SchemaName */ - StartTs: 1, - Rows: []*model.RowChangedEvent{{StartTs: 1, Columns: []*model.Column{{Name: rID, Value: uint64(10)}}}}, - }, - }, - }, - output: map[model.TableID][]*model.SingleTableTxn{}, - filterID: []uint64{10}, - replicaID: 1, - }, - { - input: map[model.TableID][]*model.SingleTableTxn{ - 1: {{Table: &model.TableName{Table: "a"}, StartTs: 1}}, - 2: { - { - Table: &model.TableName{Schema: "tidb_cdc", Table: "1"}, - StartTs: 1, - Rows: []*model.RowChangedEvent{{StartTs: 1, Columns: []*model.Column{{Name: rID, Value: uint64(10)}}}}, - }, - }, - 3: { - { - Table: &model.TableName{Schema: "tidb_cdc", Table: "2"}, - StartTs: 2, - Rows: []*model.RowChangedEvent{{StartTs: 2, Columns: []*model.Column{{Name: rID, Value: uint64(10)}}}}, - }, - }, - 4: { - { - Table: &model.TableName{Schema: "tidb_cdc", Table: "3"}, - StartTs: 3, - Rows: []*model.RowChangedEvent{{StartTs: 3, Columns: []*model.Column{{Name: rID, Value: uint64(10)}}}}, - }, - }, - }, - output: map[model.TableID][]*model.SingleTableTxn{}, - filterID: []uint64{10}, - replicaID: 1, - }, - { - input: map[model.TableID][]*model.SingleTableTxn{ - 1: {{Table: &model.TableName{Table: "a"}, StartTs: 1}}, - 2: {{Table: &model.TableName{Table: "b2"}, StartTs: 2}}, - 3: {{Table: &model.TableName{Table: "b2_1"}, StartTs: 2}}, - 4: { - { - Table: &model.TableName{Schema: "tidb_cdc", Table: "1"}, - StartTs: 1, - Rows: []*model.RowChangedEvent{{StartTs: 1, Columns: []*model.Column{{Name: rID, Value: uint64(10)}}}}, - }, - }, - }, - output: map[model.TableID][]*model.SingleTableTxn{ - 2: {{Table: &model.TableName{Table: "b2"}, StartTs: 2, ReplicaID: 1}}, - 3: {{Table: &model.TableName{Table: "b2_1"}, StartTs: 2, ReplicaID: 1}}, - }, - filterID: []uint64{10}, - replicaID: 1, - }, - { - input: map[model.TableID][]*model.SingleTableTxn{ - 1: {{Table: &model.TableName{Table: "a"}, StartTs: 1}}, - 2: {{Table: &model.TableName{Table: "b2"}, StartTs: 2}}, - 3: {{Table: &model.TableName{Table: "b2_1"}, StartTs: 2}}, - 4: {{Table: &model.TableName{Table: "b3"}, StartTs: 3}}, - 5: {{Table: &model.TableName{Table: "b3_1"}, StartTs: 3}}, - 6: { - { - Table: &model.TableName{Schema: "tidb_cdc", Table: "1"}, - StartTs: 2, - Rows: []*model.RowChangedEvent{{StartTs: 2, Columns: []*model.Column{{Name: rID, Value: uint64(10)}}}}, - }, - { - Table: &model.TableName{Schema: "tidb_cdc", Table: "1"}, - StartTs: 3, - Rows: []*model.RowChangedEvent{{StartTs: 3, Columns: []*model.Column{{Name: rID, Value: uint64(11)}}}}, - }, - }, - }, - output: map[model.TableID][]*model.SingleTableTxn{ - 1: {{Table: &model.TableName{Table: "a"}, StartTs: 1, ReplicaID: 1}}, - 4: {{Table: &model.TableName{Table: "b3"}, StartTs: 3, ReplicaID: 11}}, - 5: {{Table: &model.TableName{Table: "b3_1"}, StartTs: 3, ReplicaID: 11}}, - }, - filterID: []uint64{10}, // 10 -> 2, filter start ts 2 - replicaID: 1, - }, - { - input: map[model.TableID][]*model.SingleTableTxn{ - 2: {{Table: &model.TableName{Table: "b2"}, StartTs: 2, CommitTs: 2}}, - 3: { - {Table: &model.TableName{Table: "b3"}, StartTs: 2, CommitTs: 2}, - {Table: &model.TableName{Table: "b3"}, StartTs: 3, CommitTs: 3}, - {Table: &model.TableName{Table: "b3"}, StartTs: 3, CommitTs: 3}, - {Table: &model.TableName{Table: "b3"}, StartTs: 4, CommitTs: 4}, - }, - 6: { - { - Table: &model.TableName{Schema: "tidb_cdc", Table: "1"}, - StartTs: 2, - Rows: []*model.RowChangedEvent{{StartTs: 2, Columns: []*model.Column{{Name: rID, Value: uint64(10)}}}}, - }, - { - Table: &model.TableName{Schema: "tidb_cdc", Table: "1"}, - StartTs: 3, - Rows: []*model.RowChangedEvent{{StartTs: 3, Columns: []*model.Column{{Name: rID, Value: uint64(11)}}}}, - }, - }, - }, - output: map[model.TableID][]*model.SingleTableTxn{ - 3: { - {Table: &model.TableName{Table: "b3"}, StartTs: 3, CommitTs: 3, ReplicaID: 11}, - {Table: &model.TableName{Table: "b3"}, StartTs: 3, CommitTs: 3, ReplicaID: 11}, - {Table: &model.TableName{Table: "b3"}, StartTs: 4, CommitTs: 4, ReplicaID: 1}, - }, - }, - filterID: []uint64{10}, // 10 -> 2, filter start ts 2 - replicaID: 1, - }, - { - input: map[model.TableID][]*model.SingleTableTxn{ - 2: {{Table: &model.TableName{Table: "b2"}, StartTs: 2}}, - 6: { - { - Table: &model.TableName{Schema: "tidb_cdc", Table: "1"}, - StartTs: 2, - Rows: []*model.RowChangedEvent{{StartTs: 2, Columns: []*model.Column{{Name: rID, Value: uint64(10)}}}}, - }, - { - Table: &model.TableName{Schema: "tidb_cdc", Table: "1"}, - StartTs: 2, - Rows: []*model.RowChangedEvent{{StartTs: 2, Columns: []*model.Column{{Name: rID, Value: uint64(10)}}}}, - }, - }, - }, - output: map[model.TableID][]*model.SingleTableTxn{}, - filterID: []uint64{10}, // 10 -> 2, filter start ts 2 - replicaID: 1, - }, - } - - for i, tc := range testCases { - FilterAndReduceTxns(tc.input, tc.filterID, tc.replicaID) - require.Equal(t, tc.input, tc.output, "case %d %s\n", i, spew.Sdump(tc)) - } -} diff --git a/cdc/pkg/cyclic/main_test.go b/cdc/pkg/cyclic/main_test.go deleted file mode 100644 index 7fb0e993..00000000 --- a/cdc/pkg/cyclic/main_test.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package cyclic - -import ( - "testing" - - "github.com/tikv/migration/cdc/pkg/leakutil" -) - -func TestMain(m *testing.M) { - leakutil.SetUpLeakTest(m) -} diff --git a/cdc/pkg/cyclic/mark/main_test.go b/cdc/pkg/cyclic/mark/main_test.go deleted file mode 100644 index e66d1ae4..00000000 --- a/cdc/pkg/cyclic/mark/main_test.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2021 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package mark - -import ( - "testing" - - "github.com/tikv/migration/cdc/pkg/leakutil" -) - -func TestMain(m *testing.M) { - leakutil.SetUpLeakTest(m) -} diff --git a/cdc/pkg/cyclic/mark/mark.go b/cdc/pkg/cyclic/mark/mark.go deleted file mode 100644 index 4aef6cc0..00000000 --- a/cdc/pkg/cyclic/mark/mark.go +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package mark - -import ( - "context" - "database/sql" - "fmt" - "strings" - - "github.com/go-sql-driver/mysql" - "github.com/pingcap/errors" - "github.com/pingcap/log" - cerror "github.com/tikv/migration/cdc/pkg/errors" - "github.com/tikv/migration/cdc/pkg/quotes" - "github.com/tikv/migration/cdc/pkg/security" - "go.uber.org/zap" -) - -const ( - // SchemaName is the name of schema where all mark tables are created - SchemaName string = "tidb_cdc" - tableName string = "repl_mark" - - // CyclicReplicaIDCol is the name of replica ID in mark tables - CyclicReplicaIDCol string = "replica_id" - - // OptCyclicConfig is the key that adds to changefeed options - // automatically is cyclic replication is on. - OptCyclicConfig string = "_cyclic_relax_sql_mode" -) - -// GetMarkTableName returns mark table name regards to the tableID -func GetMarkTableName(sourceSchema, sourceTable string) (schema, table string) { // nolint:exported - // TODO(neil) better unquote or just crc32 the name. - table = strings.Join([]string{tableName, sourceSchema, sourceTable}, "_") - schema = SchemaName - return -} - -// IsMarkTable tells whether the table is a mark table or not. -func IsMarkTable(schema, table string) bool { - const quoteSchemaName = "`" + SchemaName + "`" - const quotetableName = "`" + tableName - - if schema == SchemaName || schema == quoteSchemaName { - return true - } - if strings.HasPrefix(table, quotetableName) { - return true - } - return strings.HasPrefix(table, tableName) -} - -// TableName is an interface gets schema and table name. -// Note it is only used for avoiding import model.TableName. -type TableName interface { - GetSchema() string - GetTable() string -} - -// CreateMarkTables creates mark table regard to the table name. -// -// Note table name is only for avoid write hotspot there is *NO* guarantee -// normal tables and mark tables are one:one map. -func CreateMarkTables(ctx context.Context, upstreamDSN string, upstreamCred *security.Credential, tables ...TableName) error { - tlsCfg, err := upstreamCred.ToTLSConfig() - if err != nil { - return cerror.WrapError(cerror.ErrCreateMarkTableFailed, - errors.Annotate(err, "fail to open upstream TiDB connection")) - } - if tlsCfg != nil { - tlsName := "cli-marktable" - err = mysql.RegisterTLSConfig(tlsName, tlsCfg) - if err != nil { - return cerror.WrapError(cerror.ErrCreateMarkTableFailed, - errors.Annotate(err, "fail to open upstream TiDB connection")) - } - if strings.Contains(upstreamDSN, "?") && strings.Contains(upstreamDSN, "=") { - upstreamDSN += ("&tls=" + tlsName) - } else { - upstreamDSN += ("?tls=" + tlsName) - } - } - db, err := sql.Open("mysql", upstreamDSN) - if err != nil { - return cerror.WrapError(cerror.ErrCreateMarkTableFailed, - errors.Annotate(err, "Open upstream database connection failed")) - } - err = db.PingContext(ctx) - if err != nil { - return cerror.WrapError(cerror.ErrCreateMarkTableFailed, - errors.Annotate(err, "fail to open upstream TiDB connection")) - } - - userTableCount := 0 - for _, name := range tables { - if IsMarkTable(name.GetSchema(), name.GetTable()) { - continue - } - userTableCount++ - schema, table := GetMarkTableName(name.GetSchema(), name.GetTable()) - _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s;", schema)) - if err != nil { - return cerror.WrapError(cerror.ErrCreateMarkTableFailed, - errors.Annotate(err, "fail to create mark database")) - } - _, err = db.ExecContext(ctx, fmt.Sprintf( - `CREATE TABLE IF NOT EXISTS %s - ( - bucket INT NOT NULL, - %s BIGINT UNSIGNED NOT NULL, - val BIGINT DEFAULT 0, - start_timestamp BIGINT DEFAULT 0, - PRIMARY KEY (bucket, %s) - );`, quotes.QuoteSchema(schema, table), CyclicReplicaIDCol, CyclicReplicaIDCol)) - if err != nil { - return cerror.WrapError(cerror.ErrCreateMarkTableFailed, - errors.Annotatef(err, "fail to create mark table %s", table)) - } - } - log.Info("create upstream mark done", zap.Int("count", userTableCount)) - return nil -} diff --git a/cdc/pkg/cyclic/mark/mark_test.go b/cdc/pkg/cyclic/mark/mark_test.go deleted file mode 100644 index 3b81b7f6..00000000 --- a/cdc/pkg/cyclic/mark/mark_test.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package mark - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestIsMarkTable(t *testing.T) { - t.Parallel() - tests := []struct { - schema, table string - isMarkTable bool - }{ - {"", "", false}, - {"a", "a", false}, - {"a", "", false}, - {"", "a", false}, - {SchemaName, "", true}, - {"", tableName, true}, - {"`" + SchemaName + "`", "", true}, - {"`" + SchemaName + "`", "repl_mark_1", true}, - {SchemaName, tableName, true}, - {SchemaName, "`repl_mark_1`", true}, - } - - for _, test := range tests { - require.Equal(t, IsMarkTable(test.schema, test.table), test.isMarkTable, - "%v", test) - } -} diff --git a/cdc/pkg/cyclic/replication.go b/cdc/pkg/cyclic/replication.go deleted file mode 100644 index 81723b15..00000000 --- a/cdc/pkg/cyclic/replication.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package cyclic contains scaffolds for implementing cyclic replication -// among multiple TiDB clusters/MySQL. It uses a mark table to identify and -// filter duplicate DMLs. -// CDC needs to watch DMLs to mark tables and ignore all DDLs to mark tables. -// -// Note for now, mark tables must be create manually. -package cyclic - -import ( - "fmt" - "strings" - - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/cyclic/mark" - "github.com/tikv/migration/cdc/pkg/quotes" -) - -// RelaxSQLMode returns relaxed SQL mode, "STRICT_TRANS_TABLES" is removed. -func RelaxSQLMode(oldMode string) string { - toRemove := "STRICT_TRANS_TABLES" - - if !strings.Contains(oldMode, toRemove) { - return oldMode - } - - // concatenated by "," like: mode1,mode2 - modes := strings.Split(oldMode, ",") - var newMode string - for idx := range modes { - m := modes[idx] - if strings.Contains(m, toRemove) { - continue - } - m = strings.TrimSpace(m) - if newMode == "" { - newMode = m - } else { - newMode = strings.Join([]string{newMode, modes[idx]}, ",") - } - } - return newMode -} - -// Cyclic wraps a cyclic config. -type Cyclic struct { - config config.CyclicConfig -} - -// UdpateSourceTableCyclicMark return a DML to update mark table regrad to -// the source table name, bucket and replicaID. -func (*Cyclic) UdpateSourceTableCyclicMark(sourceSchema, sourceTable string, bucket, replicaID uint64, startTs uint64) string { - schema, table := mark.GetMarkTableName(sourceSchema, sourceTable) - return fmt.Sprintf( - `INSERT INTO %s VALUES (%d, %d, 0, %d) ON DUPLICATE KEY UPDATE val = val + 1;`, - quotes.QuoteSchema(schema, table), bucket, replicaID, startTs) -} - -// Enabled returns whether cyclic replication is enabled -func (c *Cyclic) Enabled() bool { - return c.config.Enable -} - -// FilterReplicaID return a slice of replica IDs needs to be filtered. -func (c *Cyclic) FilterReplicaID() []uint64 { - return c.config.FilterReplicaID -} - -// ReplicaID return a replica ID of this cluster. -func (c *Cyclic) ReplicaID() uint64 { - return c.config.ReplicaID -} - -// NewCyclic creates a cyclic -func NewCyclic(config *config.CyclicConfig) *Cyclic { - if config == nil || config.ReplicaID == 0 { - return nil - } - return &Cyclic{ - config: *config, - } -} - -// IsTablesPaired checks if normal tables are paired with mark tables. -func IsTablesPaired(tables []model.TableName) bool { - normalTables := make([]model.TableName, 0, len(tables)/2) - markMap := make(map[model.TableName]struct{}, len(tables)/2) - for _, table := range tables { - if mark.IsMarkTable(table.Schema, table.Table) { - markMap[table] = struct{}{} - } else { - normalTables = append(normalTables, table) - } - } - for _, table := range normalTables { - markTable := model.TableName{} - markTable.Schema, markTable.Table = mark.GetMarkTableName(table.Schema, table.Table) - _, ok := markMap[markTable] - if !ok { - return false - } - } - return true -} diff --git a/cdc/pkg/cyclic/replication_test.go b/cdc/pkg/cyclic/replication_test.go deleted file mode 100644 index b34824f8..00000000 --- a/cdc/pkg/cyclic/replication_test.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package cyclic - -import ( - "testing" - - "github.com/stretchr/testify/require" - "github.com/tikv/migration/cdc/cdc/model" - "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/cyclic/mark" -) - -func TestCyclicConfig(t *testing.T) { - t.Parallel() - cfg := &config.CyclicConfig{ - Enable: true, - ReplicaID: 1, - FilterReplicaID: []uint64{2, 3}, - } - cyc := NewCyclic(cfg) - require.NotNil(t, cyc) - require.True(t, cyc.Enabled()) - require.Equal(t, cyc.ReplicaID(), uint64(1)) - require.Equal(t, cyc.FilterReplicaID(), []uint64{2, 3}) - - cyc = NewCyclic(nil) - require.Nil(t, cyc) - cyc = NewCyclic(&config.CyclicConfig{ReplicaID: 0}) - require.Nil(t, cyc) -} - -func TestRelaxSQLMode(t *testing.T) { - t.Parallel() - tests := []struct { - oldMode string - newMode string - }{ - {"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE", "ONLY_FULL_GROUP_BY,NO_ZERO_IN_DATE"}, - {"ONLY_FULL_GROUP_BY,NO_ZERO_IN_DATE,STRICT_TRANS_TABLES", "ONLY_FULL_GROUP_BY,NO_ZERO_IN_DATE"}, - {"STRICT_TRANS_TABLES", ""}, - {"ONLY_FULL_GROUP_BY,NO_ZERO_IN_DATE", "ONLY_FULL_GROUP_BY,NO_ZERO_IN_DATE"}, - } - - for _, test := range tests { - getNew := RelaxSQLMode(test.oldMode) - require.Equal(t, getNew, test.newMode) - } -} - -func TestIsTablePaired(t *testing.T) { - t.Parallel() - tests := []struct { - tables []model.TableName - isParied bool - }{ - {[]model.TableName{}, true}, - { - []model.TableName{{Schema: mark.SchemaName, Table: "repl_mark_1"}}, - true, - }, - { - []model.TableName{{Schema: "a", Table: "a"}}, - false, - }, - { - []model.TableName{ - {Schema: mark.SchemaName, Table: "repl_mark_a_a"}, - {Schema: "a", Table: "a"}, - }, - true, - }, - { - []model.TableName{ - {Schema: mark.SchemaName, Table: "repl_mark_a_a"}, - {Schema: mark.SchemaName, Table: "repl_mark_a_b"}, - {Schema: "a", Table: "a"}, - {Schema: "a", Table: "b"}, - }, - true, - }, - } - - for _, test := range tests { - require.Equal(t, IsTablesPaired(test.tables), test.isParied, - "%v", test) - } -} diff --git a/cdc/pkg/db/leveldb.go b/cdc/pkg/db/leveldb.go index 19341641..9d5343c0 100644 --- a/cdc/pkg/db/leveldb.go +++ b/cdc/pkg/db/leveldb.go @@ -25,7 +25,6 @@ import ( "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" "github.com/syndtr/goleveldb/leveldb/util" - "github.com/tikv/migration/cdc/cdc/sorter" "github.com/tikv/migration/cdc/pkg/config" cerrors "github.com/tikv/migration/cdc/pkg/errors" "github.com/tikv/migration/cdc/pkg/retry" @@ -116,12 +115,14 @@ func (p *levelDB) CollectMetrics(captureAddr string, i int) { log.Panic("leveldb error", zap.Error(err), zap.Int("db", i)) } id := strconv.Itoa(i) - sorter.OnDiskDataSizeGauge. - WithLabelValues(captureAddr, id).Set(float64(stats.LevelSizes.Sum())) - sorter.InMemoryDataSizeGauge. - WithLabelValues(captureAddr, id).Set(float64(stats.BlockCacheSize)) - sorter.OpenFileCountGauge. - WithLabelValues(captureAddr, id).Set(float64(stats.OpenedTablesCount)) + /* + sorter.OnDiskDataSizeGauge. + WithLabelValues(captureAddr, id).Set(float64(stats.LevelSizes.Sum())) + sorter.InMemoryDataSizeGauge. + WithLabelValues(captureAddr, id).Set(float64(stats.BlockCacheSize)) + sorter.OpenFileCountGauge. + WithLabelValues(captureAddr, id).Set(float64(stats.OpenedTablesCount)) + */ dbSnapshotGauge. WithLabelValues(captureAddr, id).Set(float64(stats.AliveSnapshots)) dbIteratorGauge. diff --git a/cdc/pkg/db/metrics.go b/cdc/pkg/db/metrics.go index c02b2edb..621bdb3c 100644 --- a/cdc/pkg/db/metrics.go +++ b/cdc/pkg/db/metrics.go @@ -19,49 +19,49 @@ import ( var ( dbWriteBytes = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "db", Name: "write_bytes_total", Help: "The total number of write bytes by the leveldb", }, []string{"capture", "id"}) dbReadBytes = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "db", Name: "read_bytes_total", Help: "The total number of read bytes by the leveldb", }, []string{"capture", "id"}) dbSnapshotGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "db", Name: "snapshot_count_gauge", Help: "The number of snapshot by the db", }, []string{"capture", "id"}) dbIteratorGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "db", Name: "iterator_count_gauge", Help: "The number of iterator by the db", }, []string{"capture", "id"}) dbLevelCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "db", Name: "level_count", Help: "The number of files in each level by the db", }, []string{"capture", "level", "id"}) dbWriteDelayDuration = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "db", Name: "write_delay_seconds", Help: "The duration of leveldb write delay seconds", }, []string{"capture", "id"}) dbWriteDelayCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "db", Name: "write_delay_total", Help: "The total number of leveldb delay", diff --git a/cdc/pkg/db/pebble.go b/cdc/pkg/db/pebble.go index 9fde1cde..59681db7 100644 --- a/cdc/pkg/db/pebble.go +++ b/cdc/pkg/db/pebble.go @@ -26,7 +26,6 @@ import ( "github.com/cockroachdb/pebble/bloom" "github.com/pingcap/errors" "github.com/pingcap/log" - "github.com/tikv/migration/cdc/cdc/sorter" "github.com/tikv/migration/cdc/pkg/config" "github.com/tikv/migration/cdc/pkg/retry" "go.uber.org/zap" @@ -169,10 +168,12 @@ func (p *pebbleDB) CollectMetrics(captureAddr string, i int) { for i := range stats.Levels { sum += int(stats.Levels[i].Size) } - sorter.OnDiskDataSizeGauge. - WithLabelValues(captureAddr, id).Set(float64(stats.DiskSpaceUsage())) - sorter.InMemoryDataSizeGauge. - WithLabelValues(captureAddr, id).Set(float64(stats.BlockCache.Size)) + /* + sorter.OnDiskDataSizeGauge. + WithLabelValues(captureAddr, id).Set(float64(stats.DiskSpaceUsage())) + sorter.InMemoryDataSizeGauge. + WithLabelValues(captureAddr, id).Set(float64(stats.BlockCache.Size)) + */ dbIteratorGauge. WithLabelValues(captureAddr, id).Set(float64(stats.TableIters)) dbWriteDelayCount. diff --git a/cdc/pkg/errors/errors.go b/cdc/pkg/errors/errors.go index 6b97fb1f..58e4b872 100644 --- a/cdc/pkg/errors/errors.go +++ b/cdc/pkg/errors/errors.go @@ -163,7 +163,7 @@ var ( ErrNewProcessorFailed = errors.Normalize("new processor failed", errors.RFCCodeText("CDC:ErrNewProcessorFailed")) ErrProcessorUnknown = errors.Normalize("processor running unknown error", errors.RFCCodeText("CDC:ErrProcessorUnknown")) ErrOwnerUnknown = errors.Normalize("owner running unknown error", errors.RFCCodeText("CDC:ErrOwnerUnknown")) - ErrProcessorTableNotFound = errors.Normalize("table not found in processor cache", errors.RFCCodeText("CDC:ErrProcessorTableNotFound")) + ErrProcessorKeySpanNotFound = errors.Normalize("keyspan not found in processor cache", errors.RFCCodeText("CDC:ErrProcessorKeySpanNotFound")) ErrProcessorEtcdWatch = errors.Normalize("etcd watch returns error", errors.RFCCodeText("CDC:ErrProcessorEtcdWatch")) ErrProcessorSortDir = errors.Normalize("sort dir error", errors.RFCCodeText("CDC:ErrProcessorSortDir")) ErrUnknownSortEngine = errors.Normalize("unknown sort engine %s", errors.RFCCodeText("CDC:ErrUnknownSortEngine")) @@ -194,7 +194,7 @@ var ( ErrGCTTLExceeded = errors.Normalize("the checkpoint-ts(%d) lag of the changefeed(%s) has exceeded the GC TTL", errors.RFCCodeText("CDC:ErrGCTTLExceeded")) ErrNotOwner = errors.Normalize("this capture is not a owner", errors.RFCCodeText("CDC:ErrNotOwner")) ErrOwnerNotFound = errors.Normalize("owner not found", errors.RFCCodeText("CDC:ErrOwnerNotFound")) - ErrTableListenReplicated = errors.Normalize("A table(%d) is being replicated by at least two processors(%s, %s), please report a bug", errors.RFCCodeText("CDC:ErrTableListenReplicated")) + ErrKeySpanListenReplicated = errors.Normalize("A keyspan(%d) is being replicated by at least two processors(%s, %s), please report a bug", errors.RFCCodeText("CDC:ErrKeySpanListenReplicated")) ErrTableIneligible = errors.Normalize("some tables are not eligible to replicate(%v), if you want to ignore these tables, please set ignore_ineligible_table to true", errors.RFCCodeText("CDC:ErrTableIneligible")) // EtcdWorker related errors. Internal use only. @@ -248,8 +248,8 @@ var ( ErrSorterClosed = errors.Normalize("sorter is closed", errors.RFCCodeText("CDC:ErrSorterClosed")) // processor errors - ErrTableProcessorStoppedSafely = errors.Normalize("table processor stopped safely", errors.RFCCodeText("CDC:ErrTableProcessorStoppedSafely")) - ErrProcessorDuplicateOperations = errors.Normalize("table processor duplicate operation, table-id: %d", errors.RFCCodeText("CDC:ErrProcessorDuplicateOperations")) + ErrKeySpanProcessorStoppedSafely = errors.Normalize("keyspan processor stopped safely", errors.RFCCodeText("CDC:ErrKeySpanProcessorStoppedSafely")) + ErrProcessorDuplicateOperations = errors.Normalize("keyspan processor duplicate operation, keyspan-id: %d", errors.RFCCodeText("CDC:ErrProcessorDuplicateOperations")) // owner errors ErrOwnerChangedUnexpectedly = errors.Normalize("owner changed unexpectedly", errors.RFCCodeText("CDC:ErrOwnerChangedUnexpectedly")) diff --git a/cdc/pkg/etcd/etcd_test.go b/cdc/pkg/etcd/etcd_test.go index 341dea91..848087b1 100644 --- a/cdc/pkg/etcd/etcd_test.go +++ b/cdc/pkg/etcd/etcd_test.go @@ -158,7 +158,7 @@ func (s *etcdSuite) TestGetPutTaskStatus(c *check.C) { defer s.TearDownTest(c) ctx := context.Background() info := &model.TaskStatus{ - Tables: map[model.TableID]*model.TableReplicaInfo{ + KeySpans: map[model.KeySpanID]*model.KeySpanReplicaInfo{ 1: {StartTs: 100}, }, } diff --git a/cdc/pkg/etcd/etcdkey.go b/cdc/pkg/etcd/etcdkey.go index 84589ab6..7cf11944 100644 --- a/cdc/pkg/etcd/etcdkey.go +++ b/cdc/pkg/etcd/etcdkey.go @@ -22,7 +22,7 @@ import ( const ( // EtcdKeyBase is the common prefix of the keys in CDC - EtcdKeyBase = "/tidb/cdc" + EtcdKeyBase = "/tikv/cdc" ownerKey = "/owner" captureKey = "/capture" @@ -56,7 +56,7 @@ const ( we can parse a raw etcd key: ``` k := new(CDCKey) - rawKey := "/tidb/cdc/changefeed/info/test/changefeed" + rawKey := "/tikv/cdc/changefeed/info/test/changefeed" err := k.Parse(rawKey) c.Assert(k, check.DeepEquals, &CDCKey{ Tp: CDCKeyTypeChangefeedInfo, @@ -70,7 +70,7 @@ const ( Tp: CDCKeyTypeChangefeedInfo, ChangefeedID: "test/changefeed", } - c.Assert(k.String(), check.Equals, "/tidb/cdc/changefeed/info/test/changefeed") + c.Assert(k.String(), check.Equals, "/tikv/cdc/changefeed/info/test/changefeed") ``` */ diff --git a/cdc/pkg/etcd/etcdkey_test.go b/cdc/pkg/etcd/etcdkey_test.go index 34a83add..529db604 100644 --- a/cdc/pkg/etcd/etcdkey_test.go +++ b/cdc/pkg/etcd/etcdkey_test.go @@ -28,64 +28,64 @@ func (s *etcdkeySuite) TestEtcdKey(c *check.C) { key string expected *CDCKey }{{ - key: "/tidb/cdc/owner/223176cb44d20a13", + key: "/tikv/cdc/owner/223176cb44d20a13", expected: &CDCKey{ Tp: CDCKeyTypeOwner, OwnerLeaseID: "223176cb44d20a13", }, }, { - key: "/tidb/cdc/owner", + key: "/tikv/cdc/owner", expected: &CDCKey{ Tp: CDCKeyTypeOwner, OwnerLeaseID: "", }, }, { - key: "/tidb/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", + key: "/tikv/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", expected: &CDCKey{ Tp: CDCKeyTypeCapture, CaptureID: "6bbc01c8-0605-4f86-a0f9-b3119109b225", }, }, { - key: "/tidb/cdc/changefeed/info/test-_@#$%changefeed", + key: "/tikv/cdc/changefeed/info/test-_@#$%changefeed", expected: &CDCKey{ Tp: CDCKeyTypeChangefeedInfo, ChangefeedID: "test-_@#$%changefeed", }, }, { - key: "/tidb/cdc/changefeed/info/test/changefeed", + key: "/tikv/cdc/changefeed/info/test/changefeed", expected: &CDCKey{ Tp: CDCKeyTypeChangefeedInfo, ChangefeedID: "test/changefeed", }, }, { - key: "/tidb/cdc/job/test-changefeed", + key: "/tikv/cdc/job/test-changefeed", expected: &CDCKey{ Tp: CDCKeyTypeChangeFeedStatus, ChangefeedID: "test-changefeed", }, }, { - key: "/tidb/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test-changefeed", + key: "/tikv/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test-changefeed", expected: &CDCKey{ Tp: CDCKeyTypeTaskPosition, ChangefeedID: "test-changefeed", CaptureID: "6bbc01c8-0605-4f86-a0f9-b3119109b225", }, }, { - key: "/tidb/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test/changefeed", + key: "/tikv/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test/changefeed", expected: &CDCKey{ Tp: CDCKeyTypeTaskPosition, ChangefeedID: "test/changefeed", CaptureID: "6bbc01c8-0605-4f86-a0f9-b3119109b225", }, }, { - key: "/tidb/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test-changefeed", + key: "/tikv/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test-changefeed", expected: &CDCKey{ Tp: CDCKeyTypeTaskStatus, ChangefeedID: "test-changefeed", CaptureID: "6bbc01c8-0605-4f86-a0f9-b3119109b225", }, }, { - key: "/tidb/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test-changefeed", + key: "/tikv/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test-changefeed", expected: &CDCKey{ Tp: CDCKeyTypeTaskWorkload, ChangefeedID: "test-changefeed", @@ -107,25 +107,25 @@ func (s *etcdkeySuite) TestEtcdKeyParseError(c *check.C) { key string error bool }{{ - key: "/tidb/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test/changefeed", + key: "/tikv/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test/changefeed", error: false, }, { - key: "/tidb/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/", + key: "/tikv/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/", error: false, }, { - key: "/tidb/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225", + key: "/tikv/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225", error: true, }, { - key: "/tidb/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225", + key: "/tikv/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225", error: true, }, { - key: "/tidb/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225", + key: "/tikv/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225", error: true, }, { - key: "/tidb/cd", + key: "/tikv/cd", error: true, }, { - key: "/tidb/cdc/", + key: "/tikv/cdc/", error: true, }} for _, tc := range testCases { diff --git a/cdc/pkg/etcd/metrics.go b/cdc/pkg/etcd/metrics.go index 4f95f4c5..a8300ed7 100644 --- a/cdc/pkg/etcd/metrics.go +++ b/cdc/pkg/etcd/metrics.go @@ -17,7 +17,7 @@ import "github.com/prometheus/client_golang/prometheus" var etcdRequestCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "etcd", Name: "request_count", Help: "request counter of etcd operation", diff --git a/cdc/pkg/filter/filter.go b/cdc/pkg/filter/filter.go index 9d9b0133..18106695 100644 --- a/cdc/pkg/filter/filter.go +++ b/cdc/pkg/filter/filter.go @@ -14,11 +14,12 @@ package filter import ( + "fmt" + filterV1 "github.com/pingcap/tidb-tools/pkg/filter" filterV2 "github.com/pingcap/tidb-tools/pkg/table-filter" "github.com/pingcap/tidb/parser/model" "github.com/tikv/migration/cdc/pkg/config" - "github.com/tikv/migration/cdc/pkg/cyclic/mark" cerror "github.com/tikv/migration/cdc/pkg/errors" ) @@ -82,12 +83,9 @@ func (f *Filter) shouldIgnoreStartTs(ts uint64) bool { // NOTICE: Set `tbl` to an empty string to test against the whole database. func (f *Filter) ShouldIgnoreTable(db, tbl string) bool { if isSysSchema(db) { + fmt.Println("is sys schema", db, tbl) return true } - if f.isCyclicEnabled && mark.IsMarkTable(db, tbl) { - // Always replicate mark tables. - return false - } return !f.filter.MatchTable(db, tbl) } diff --git a/cdc/pkg/filter/filter_test.go b/cdc/pkg/filter/filter_test.go index a4566c49..4f2ad54d 100644 --- a/cdc/pkg/filter/filter_test.go +++ b/cdc/pkg/filter/filter_test.go @@ -53,7 +53,6 @@ func TestShouldUseCustomRules(t *testing.T) { require.True(t, filter.ShouldIgnoreTable("ecom", "test")) require.True(t, filter.ShouldIgnoreTable("sns", "log")) require.True(t, filter.ShouldIgnoreTable("information_schema", "")) - require.False(t, filter.ShouldIgnoreTable("tidb_cdc", "repl_mark_a_a")) } func TestShouldIgnoreTxn(t *testing.T) { diff --git a/cdc/pkg/orchestrator/metrics.go b/cdc/pkg/orchestrator/metrics.go index efbb2428..9d519adb 100644 --- a/cdc/pkg/orchestrator/metrics.go +++ b/cdc/pkg/orchestrator/metrics.go @@ -18,7 +18,7 @@ import "github.com/prometheus/client_golang/prometheus" var ( etcdTxnSize = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "etcd_worker", Name: "etcd_txn_size_bytes", Help: "Bucketed histogram of a etcd txn size.", @@ -27,7 +27,7 @@ var ( etcdTxnExecDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "etcd_worker", Name: "etcd_txn_exec_duration", Help: "Bucketed histogram of processing time (s) of a etcd txn.", @@ -36,7 +36,7 @@ var ( etcdWorkerTickDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "etcd_worker", Name: "tick_reactor_duration", Help: "Bucketed histogram of etcdWorker tick reactor time (s).", diff --git a/cdc/pkg/orchestrator/reactor_state_test.go b/cdc/pkg/orchestrator/reactor_state_test.go index f83fe3c9..208edc15 100644 --- a/cdc/pkg/orchestrator/reactor_state_test.go +++ b/cdc/pkg/orchestrator/reactor_state_test.go @@ -37,7 +37,7 @@ func (s *stateSuite) TestCheckCaptureAlive(c *check.C) { stateTester := NewReactorStateTester(c, state, nil) state.CheckCaptureAlive("6bbc01c8-0605-4f86-a0f9-b3119109b225") c.Assert(stateTester.ApplyPatches(), check.ErrorMatches, ".*[CDC:ErrLeaseExpired].*") - err := stateTester.Update("/tidb/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", []byte(`{"id":"6bbc01c8-0605-4f86-a0f9-b3119109b225","address":"127.0.0.1:8300"}`)) + err := stateTester.Update("/tikv/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", []byte(`{"id":"6bbc01c8-0605-4f86-a0f9-b3119109b225","address":"127.0.0.1:8300"}`)) c.Assert(err, check.IsNil) state.CheckCaptureAlive("6bbc01c8-0605-4f86-a0f9-b3119109b225") stateTester.MustApplyPatches() @@ -56,18 +56,18 @@ func (s *stateSuite) TestChangefeedStateUpdate(c *check.C) { { // common case changefeedID: "test1", updateKey: []string{ - "/tidb/cdc/changefeed/info/test1", - "/tidb/cdc/job/test1", - "/tidb/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", + "/tikv/cdc/changefeed/info/test1", + "/tikv/cdc/job/test1", + "/tikv/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", }, updateValue: []string{ - `{"sink-uri":"blackhole://","opts":{},"create-time":"2020-02-02T00:00:00.000000+00:00","start-ts":421980685886554116,"target-ts":0,"admin-job-type":0,"sort-engine":"memory","sort-dir":"","config":{"case-sensitive":true,"enable-old-value":false,"force-replicate":false,"check-gc-safe-point":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"open-protocol"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"table-number","polling-time":-1},"consistent":{"level":"normal","storage":"local"}},"state":"normal","history":null,"error":null,"sync-point-enabled":false,"sync-point-interval":600000000000}`, + `{"sink-uri":"blackhole://","opts":{},"create-time":"2020-02-02T00:00:00.000000+00:00","start-ts":421980685886554116,"target-ts":0,"admin-job-type":0,"sort-engine":"memory","sort-dir":"","config":{"case-sensitive":true,"enable-old-value":false,"force-replicate":false,"check-gc-safe-point":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"open-protocol"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"keyspan-number","polling-time":-1},"consistent":{"level":"normal","storage":"local"}},"state":"normal","history":null,"error":null,"sync-point-enabled":false,"sync-point-interval":600000000000}`, `{"resolved-ts":421980720003809281,"checkpoint-ts":421980719742451713,"admin-job-type":0}`, `{"checkpoint-ts":421980720003809281,"resolved-ts":421980720003809281,"count":0,"error":null}`, - `{"tables":{"45":{"start-ts":421980685886554116,"mark-table-id":0}},"operation":null,"admin-job-type":0}`, + `{"keyspans":{"45":{"start-ts":421980685886554116,"mark-keyspan-id":0}},"operation":null,"admin-job-type":0}`, `{"45":{"workload":1}}`, `{"id":"6bbc01c8-0605-4f86-a0f9-b3119109b225","address":"127.0.0.1:8300"}`, }, @@ -88,14 +88,14 @@ func (s *stateSuite) TestChangefeedStateUpdate(c *check.C) { Mounter: &config.MounterConfig{WorkerNum: 16}, Sink: &config.SinkConfig{Protocol: "open-protocol"}, Cyclic: &config.CyclicConfig{}, - Scheduler: &config.SchedulerConfig{Tp: "table-number", PollingTime: -1}, + Scheduler: &config.SchedulerConfig{Tp: "keyspan-number", PollingTime: -1}, Consistent: &config.ConsistentConfig{Level: "normal", Storage: "local"}, }, }, Status: &model.ChangeFeedStatus{CheckpointTs: 421980719742451713, ResolvedTs: 421980720003809281}, TaskStatuses: map[model.CaptureID]*model.TaskStatus{ "6bbc01c8-0605-4f86-a0f9-b3119109b225": { - Tables: map[int64]*model.TableReplicaInfo{45: {StartTs: 421980685886554116}}, + KeySpans: map[uint64]*model.KeySpanReplicaInfo{45: {StartTs: 421980685886554116}}, }, }, TaskPositions: map[model.CaptureID]*model.TaskPosition{ @@ -109,26 +109,26 @@ func (s *stateSuite) TestChangefeedStateUpdate(c *check.C) { { // test multiple capture changefeedID: "test1", updateKey: []string{ - "/tidb/cdc/changefeed/info/test1", - "/tidb/cdc/job/test1", - "/tidb/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", - "/tidb/cdc/task/position/666777888/test1", - "/tidb/cdc/task/status/666777888/test1", - "/tidb/cdc/task/workload/666777888/test1", - "/tidb/cdc/capture/666777888", + "/tikv/cdc/changefeed/info/test1", + "/tikv/cdc/job/test1", + "/tikv/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", + "/tikv/cdc/task/position/666777888/test1", + "/tikv/cdc/task/status/666777888/test1", + "/tikv/cdc/task/workload/666777888/test1", + "/tikv/cdc/capture/666777888", }, updateValue: []string{ - `{"sink-uri":"blackhole://","opts":{},"create-time":"2020-02-02T00:00:00.000000+00:00","start-ts":421980685886554116,"target-ts":0,"admin-job-type":0,"sort-engine":"memory","sort-dir":"","config":{"case-sensitive":true,"enable-old-value":false,"force-replicate":false,"check-gc-safe-point":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"open-protocol"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"table-number","polling-time":-1},"consistent":{"level":"normal","storage":"local"}},"state":"normal","history":null,"error":null,"sync-point-enabled":false,"sync-point-interval":600000000000}`, + `{"sink-uri":"blackhole://","opts":{},"create-time":"2020-02-02T00:00:00.000000+00:00","start-ts":421980685886554116,"target-ts":0,"admin-job-type":0,"sort-engine":"memory","sort-dir":"","config":{"case-sensitive":true,"enable-old-value":false,"force-replicate":false,"check-gc-safe-point":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"open-protocol"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"keyspan-number","polling-time":-1},"consistent":{"level":"normal","storage":"local"}},"state":"normal","history":null,"error":null,"sync-point-enabled":false,"sync-point-interval":600000000000}`, `{"resolved-ts":421980720003809281,"checkpoint-ts":421980719742451713,"admin-job-type":0}`, `{"checkpoint-ts":421980720003809281,"resolved-ts":421980720003809281,"count":0,"error":null}`, - `{"tables":{"45":{"start-ts":421980685886554116,"mark-table-id":0}},"operation":null,"admin-job-type":0}`, + `{"keyspans":{"45":{"start-ts":421980685886554116,"mark-keyspan-id":0}},"operation":null,"admin-job-type":0}`, `{"45":{"workload":1}}`, `{"id":"6bbc01c8-0605-4f86-a0f9-b3119109b225","address":"127.0.0.1:8300"}`, `{"checkpoint-ts":11332244,"resolved-ts":312321,"count":8,"error":null}`, - `{"tables":{"46":{"start-ts":412341234,"mark-table-id":0}},"operation":null,"admin-job-type":0}`, + `{"keyspans":{"46":{"start-ts":412341234,"mark-keyspan-id":0}},"operation":null,"admin-job-type":0}`, `{"46":{"workload":3}}`, `{"id":"666777888","address":"127.0.0.1:8300"}`, }, @@ -149,17 +149,17 @@ func (s *stateSuite) TestChangefeedStateUpdate(c *check.C) { Mounter: &config.MounterConfig{WorkerNum: 16}, Sink: &config.SinkConfig{Protocol: "open-protocol"}, Cyclic: &config.CyclicConfig{}, - Scheduler: &config.SchedulerConfig{Tp: "table-number", PollingTime: -1}, + Scheduler: &config.SchedulerConfig{Tp: "keyspan-number", PollingTime: -1}, Consistent: &config.ConsistentConfig{Level: "normal", Storage: "local"}, }, }, Status: &model.ChangeFeedStatus{CheckpointTs: 421980719742451713, ResolvedTs: 421980720003809281}, TaskStatuses: map[model.CaptureID]*model.TaskStatus{ "6bbc01c8-0605-4f86-a0f9-b3119109b225": { - Tables: map[int64]*model.TableReplicaInfo{45: {StartTs: 421980685886554116}}, + KeySpans: map[uint64]*model.KeySpanReplicaInfo{45: {StartTs: 421980685886554116}}, }, "666777888": { - Tables: map[int64]*model.TableReplicaInfo{46: {StartTs: 412341234}}, + KeySpans: map[uint64]*model.KeySpanReplicaInfo{46: {StartTs: 412341234}}, }, }, TaskPositions: map[model.CaptureID]*model.TaskPosition{ @@ -175,23 +175,23 @@ func (s *stateSuite) TestChangefeedStateUpdate(c *check.C) { { // testing changefeedID not match changefeedID: "test1", updateKey: []string{ - "/tidb/cdc/changefeed/info/test1", - "/tidb/cdc/job/test1", - "/tidb/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", - "/tidb/cdc/changefeed/info/test-fake", - "/tidb/cdc/job/test-fake", - "/tidb/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test-fake", - "/tidb/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test-fake", - "/tidb/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test-fake", + "/tikv/cdc/changefeed/info/test1", + "/tikv/cdc/job/test1", + "/tikv/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", + "/tikv/cdc/changefeed/info/test-fake", + "/tikv/cdc/job/test-fake", + "/tikv/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test-fake", + "/tikv/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test-fake", + "/tikv/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test-fake", }, updateValue: []string{ - `{"sink-uri":"blackhole://","opts":{},"create-time":"2020-02-02T00:00:00.000000+00:00","start-ts":421980685886554116,"target-ts":0,"admin-job-type":0,"sort-engine":"memory","sort-dir":"","config":{"case-sensitive":true,"enable-old-value":false,"force-replicate":false,"check-gc-safe-point":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"open-protocol"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"table-number","polling-time":-1},"consistent":{"level":"normal","storage":"local"}},"state":"normal","history":null,"error":null,"sync-point-enabled":false,"sync-point-interval":600000000000}`, + `{"sink-uri":"blackhole://","opts":{},"create-time":"2020-02-02T00:00:00.000000+00:00","start-ts":421980685886554116,"target-ts":0,"admin-job-type":0,"sort-engine":"memory","sort-dir":"","config":{"case-sensitive":true,"enable-old-value":false,"force-replicate":false,"check-gc-safe-point":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"open-protocol"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"keyspan-number","polling-time":-1},"consistent":{"level":"normal","storage":"local"}},"state":"normal","history":null,"error":null,"sync-point-enabled":false,"sync-point-interval":600000000000}`, `{"resolved-ts":421980720003809281,"checkpoint-ts":421980719742451713,"admin-job-type":0}`, `{"checkpoint-ts":421980720003809281,"resolved-ts":421980720003809281,"count":0,"error":null}`, - `{"tables":{"45":{"start-ts":421980685886554116,"mark-table-id":0}},"operation":null,"admin-job-type":0}`, + `{"keyspans":{"45":{"start-ts":421980685886554116,"mark-keyspan-id":0}},"operation":null,"admin-job-type":0}`, `{"45":{"workload":1}}`, `{"id":"6bbc01c8-0605-4f86-a0f9-b3119109b225","address":"127.0.0.1:8300"}`, `fake value`, @@ -217,14 +217,14 @@ func (s *stateSuite) TestChangefeedStateUpdate(c *check.C) { Mounter: &config.MounterConfig{WorkerNum: 16}, Sink: &config.SinkConfig{Protocol: "open-protocol"}, Cyclic: &config.CyclicConfig{}, - Scheduler: &config.SchedulerConfig{Tp: "table-number", PollingTime: -1}, + Scheduler: &config.SchedulerConfig{Tp: "keyspan-number", PollingTime: -1}, Consistent: &config.ConsistentConfig{Level: "normal", Storage: "local"}, }, }, Status: &model.ChangeFeedStatus{CheckpointTs: 421980719742451713, ResolvedTs: 421980720003809281}, TaskStatuses: map[model.CaptureID]*model.TaskStatus{ "6bbc01c8-0605-4f86-a0f9-b3119109b225": { - Tables: map[int64]*model.TableReplicaInfo{45: {StartTs: 421980685886554116}}, + KeySpans: map[uint64]*model.KeySpanReplicaInfo{45: {StartTs: 421980685886554116}}, }, }, TaskPositions: map[model.CaptureID]*model.TaskPosition{ @@ -238,33 +238,33 @@ func (s *stateSuite) TestChangefeedStateUpdate(c *check.C) { { // testing value is nil changefeedID: "test1", updateKey: []string{ - "/tidb/cdc/changefeed/info/test1", - "/tidb/cdc/job/test1", - "/tidb/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", - "/tidb/cdc/task/position/666777888/test1", - "/tidb/cdc/task/status/666777888/test1", - "/tidb/cdc/task/workload/666777888/test1", - "/tidb/cdc/changefeed/info/test1", - "/tidb/cdc/job/test1", - "/tidb/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", - "/tidb/cdc/task/workload/666777888/test1", - "/tidb/cdc/task/status/666777888/test1", + "/tikv/cdc/changefeed/info/test1", + "/tikv/cdc/job/test1", + "/tikv/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", + "/tikv/cdc/task/position/666777888/test1", + "/tikv/cdc/task/status/666777888/test1", + "/tikv/cdc/task/workload/666777888/test1", + "/tikv/cdc/changefeed/info/test1", + "/tikv/cdc/job/test1", + "/tikv/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", + "/tikv/cdc/task/workload/666777888/test1", + "/tikv/cdc/task/status/666777888/test1", }, updateValue: []string{ - `{"sink-uri":"blackhole://","opts":{},"create-time":"2020-02-02T00:00:00.000000+00:00","start-ts":421980685886554116,"target-ts":0,"admin-job-type":0,"sort-engine":"memory","sort-dir":"","config":{"case-sensitive":true,"enable-old-value":false,"force-replicate":false,"check-gc-safe-point":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"open-protocol"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"table-number","polling-time":-1},"consistent":{"level":"normal","storage":"local"}},"state":"normal","history":null,"error":null,"sync-point-enabled":false,"sync-point-interval":600000000000}`, + `{"sink-uri":"blackhole://","opts":{},"create-time":"2020-02-02T00:00:00.000000+00:00","start-ts":421980685886554116,"target-ts":0,"admin-job-type":0,"sort-engine":"memory","sort-dir":"","config":{"case-sensitive":true,"enable-old-value":false,"force-replicate":false,"check-gc-safe-point":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"open-protocol"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"keyspan-number","polling-time":-1},"consistent":{"level":"normal","storage":"local"}},"state":"normal","history":null,"error":null,"sync-point-enabled":false,"sync-point-interval":600000000000}`, `{"resolved-ts":421980720003809281,"checkpoint-ts":421980719742451713,"admin-job-type":0}`, `{"checkpoint-ts":421980720003809281,"resolved-ts":421980720003809281,"count":0,"error":null}`, - `{"tables":{"45":{"start-ts":421980685886554116,"mark-table-id":0}},"operation":null,"admin-job-type":0}`, + `{"keyspans":{"45":{"start-ts":421980685886554116,"mark-keyspan-id":0}},"operation":null,"admin-job-type":0}`, `{"45":{"workload":1}}`, `{"id":"6bbc01c8-0605-4f86-a0f9-b3119109b225","address":"127.0.0.1:8300"}`, `{"checkpoint-ts":11332244,"resolved-ts":312321,"count":8,"error":null}`, - `{"tables":{"46":{"start-ts":412341234,"mark-table-id":0}},"operation":null,"admin-job-type":0}`, + `{"keyspans":{"46":{"start-ts":412341234,"mark-keyspan-id":0}},"operation":null,"admin-job-type":0}`, `{"46":{"workload":3}}`, ``, ``, @@ -289,22 +289,22 @@ func (s *stateSuite) TestChangefeedStateUpdate(c *check.C) { { // testing the same key case changefeedID: "test1", updateKey: []string{ - "/tidb/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/task/status/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", }, updateValue: []string{ - `{"tables":{"45":{"start-ts":421980685886554116,"mark-table-id":0}},"operation":null,"admin-job-type":0}`, - `{"tables":{"46":{"start-ts":421980685886554116,"mark-table-id":0}},"operation":null,"admin-job-type":0}`, + `{"keyspans":{"45":{"start-ts":421980685886554116,"mark-keyspan-id":0}},"operation":null,"admin-job-type":0}`, + `{"keyspans":{"46":{"start-ts":421980685886554116,"mark-keyspan-id":0}},"operation":null,"admin-job-type":0}`, ``, - `{"tables":{"47":{"start-ts":421980685886554116,"mark-table-id":0}},"operation":null,"admin-job-type":0}`, + `{"keyspans":{"47":{"start-ts":421980685886554116,"mark-keyspan-id":0}},"operation":null,"admin-job-type":0}`, }, expected: ChangefeedReactorState{ ID: "test1", TaskStatuses: map[model.CaptureID]*model.TaskStatus{ "6bbc01c8-0605-4f86-a0f9-b3119109b225": { - Tables: map[int64]*model.TableReplicaInfo{47: {StartTs: 421980685886554116}}, + KeySpans: map[uint64]*model.KeySpanReplicaInfo{47: {StartTs: 421980685886554116}}, }, }, TaskPositions: map[model.CaptureID]*model.TaskPosition{}, @@ -470,39 +470,39 @@ func (s *stateSuite) TestPatchTaskStatus(c *check.C) { state.PatchTaskStatus(captureID1, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { c.Assert(status, check.IsNil) return &model.TaskStatus{ - Tables: map[model.TableID]*model.TableReplicaInfo{45: {StartTs: 1}}, + KeySpans: map[model.KeySpanID]*model.KeySpanReplicaInfo{45: {StartTs: 1}}, }, true, nil }) state.PatchTaskStatus(captureID2, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { c.Assert(status, check.IsNil) return &model.TaskStatus{ - Tables: map[model.TableID]*model.TableReplicaInfo{46: {StartTs: 1}}, + KeySpans: map[model.KeySpanID]*model.KeySpanReplicaInfo{46: {StartTs: 1}}, }, true, nil }) stateTester.MustApplyPatches() c.Assert(state.TaskStatuses, check.DeepEquals, map[model.CaptureID]*model.TaskStatus{ - captureID1: {Tables: map[model.TableID]*model.TableReplicaInfo{45: {StartTs: 1}}}, - captureID2: {Tables: map[model.TableID]*model.TableReplicaInfo{46: {StartTs: 1}}}, + captureID1: {KeySpans: map[model.KeySpanID]*model.KeySpanReplicaInfo{45: {StartTs: 1}}}, + captureID2: {KeySpans: map[model.KeySpanID]*model.KeySpanReplicaInfo{46: {StartTs: 1}}}, }) state.PatchTaskStatus(captureID1, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { - status.Tables[46] = &model.TableReplicaInfo{StartTs: 2} + status.KeySpans[46] = &model.KeySpanReplicaInfo{StartTs: 2} return status, true, nil }) state.PatchTaskStatus(captureID2, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { - status.Tables[46].StartTs++ + status.KeySpans[46].StartTs++ return status, true, nil }) stateTester.MustApplyPatches() c.Assert(state.TaskStatuses, check.DeepEquals, map[model.CaptureID]*model.TaskStatus{ - captureID1: {Tables: map[model.TableID]*model.TableReplicaInfo{45: {StartTs: 1}, 46: {StartTs: 2}}}, - captureID2: {Tables: map[model.TableID]*model.TableReplicaInfo{46: {StartTs: 2}}}, + captureID1: {KeySpans: map[model.KeySpanID]*model.KeySpanReplicaInfo{45: {StartTs: 1}, 46: {StartTs: 2}}}, + captureID2: {KeySpans: map[model.KeySpanID]*model.KeySpanReplicaInfo{46: {StartTs: 2}}}, }) state.PatchTaskStatus(captureID2, func(status *model.TaskStatus) (*model.TaskStatus, bool, error) { return nil, true, nil }) stateTester.MustApplyPatches() c.Assert(state.TaskStatuses, check.DeepEquals, map[model.CaptureID]*model.TaskStatus{ - captureID1: {Tables: map[model.TableID]*model.TableReplicaInfo{45: {StartTs: 1}, 46: {StartTs: 2}}}, + captureID1: {KeySpans: map[model.KeySpanID]*model.KeySpanReplicaInfo{45: {StartTs: 1}, 46: {StartTs: 2}}}, }) } @@ -556,12 +556,12 @@ func (s *stateSuite) TestGlobalStateUpdate(c *check.C) { }{ { // common case updateKey: []string{ - "/tidb/cdc/owner/22317526c4fc9a37", - "/tidb/cdc/owner/22317526c4fc9a38", - "/tidb/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", - "/tidb/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test2", - "/tidb/cdc/task/workload/55551111/test2", + "/tikv/cdc/owner/22317526c4fc9a37", + "/tikv/cdc/owner/22317526c4fc9a38", + "/tikv/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", + "/tikv/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test2", + "/tikv/cdc/task/workload/55551111/test2", }, updateValue: []string{ `6bbc01c8-0605-4f86-a0f9-b3119109b225`, @@ -600,16 +600,16 @@ func (s *stateSuite) TestGlobalStateUpdate(c *check.C) { }, { // testing remove changefeed updateKey: []string{ - "/tidb/cdc/owner/22317526c4fc9a37", - "/tidb/cdc/owner/22317526c4fc9a38", - "/tidb/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", - "/tidb/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test2", - "/tidb/cdc/task/workload/55551111/test2", - "/tidb/cdc/owner/22317526c4fc9a37", - "/tidb/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", - "/tidb/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test2", - "/tidb/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", + "/tikv/cdc/owner/22317526c4fc9a37", + "/tikv/cdc/owner/22317526c4fc9a38", + "/tikv/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", + "/tikv/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test2", + "/tikv/cdc/task/workload/55551111/test2", + "/tikv/cdc/owner/22317526c4fc9a37", + "/tikv/cdc/task/position/6bbc01c8-0605-4f86-a0f9-b3119109b225/test1", + "/tikv/cdc/task/workload/6bbc01c8-0605-4f86-a0f9-b3119109b225/test2", + "/tikv/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225", }, updateValue: []string{ `6bbc01c8-0605-4f86-a0f9-b3119109b225`, diff --git a/cdc/pkg/p2p/client_test.go b/cdc/pkg/p2p/client_test.go index a3f6aa14..52b62c08 100644 --- a/cdc/pkg/p2p/client_test.go +++ b/cdc/pkg/p2p/client_test.go @@ -180,8 +180,7 @@ func TestMessageClientBasics(t *testing.T) { sender.AssertExpectations(t) // Test point 7: Interrupt the connection - grpcStream.ExpectedCalls = nil - grpcStream.Calls = nil + grpcStream.ResetMock() sender.ExpectedCalls = nil sender.Calls = nil @@ -320,6 +319,8 @@ func TestClientSendAnomalies(t *testing.T) { }) grpcStream.On("Recv").Return(nil, nil) + sender.On("Flush").Return(nil) + sender.On("Append", mock.Anything).Return(nil) var wg sync.WaitGroup wg.Add(1) @@ -340,9 +341,11 @@ func TestClientSendAnomalies(t *testing.T) { time.Sleep(100 * time.Millisecond) closeClient() }() - _, err = client.SendMessage(ctx, "test-topic", &testMessage{Value: 1}) - require.Error(t, err) - require.Regexp(t, ".*ErrPeerMessageClientClosed.*", err.Error()) + _, _ = client.SendMessage(ctx, "test-topic", &testMessage{Value: 1}) + // There is no need to check for error here, because when a client is closing, + // message loss is expected because sending the message is fully asynchronous. + // The client implementation is considered correct if `SendMessage` does not + // block infinitely. wg.Wait() diff --git a/cdc/pkg/p2p/metrics.go b/cdc/pkg/p2p/metrics.go index d5a53824..79be251d 100644 --- a/cdc/pkg/p2p/metrics.go +++ b/cdc/pkg/p2p/metrics.go @@ -22,21 +22,21 @@ const unknownPeerLabel = "unknown" var ( serverStreamCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "message_server", Name: "cur_stream_count", Help: "count of concurrent streams handled by the message server", }, []string{"from"}) serverMessageCount = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "message_server", Name: "message_count", Help: "count of messages received", }, []string{"from"}) serverMessageBatchHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "message_server", Name: "message_batch_size", Help: "size in number of messages of message batches received", @@ -45,7 +45,7 @@ var ( // serverMessageBatchBytesHistogram records the wire sizes as reported by protobuf. serverMessageBatchBytesHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "message_server", Name: "message_batch_bytes", Help: "size in bytes of message batches received", @@ -54,7 +54,7 @@ var ( // serverMessageBytesHistogram records the wire sizes as reported by protobuf. serverMessageBytesHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "message_server", Name: "message_bytes", Help: "size in bytes of messages received", @@ -62,40 +62,40 @@ var ( }, []string{"from"}) serverAckCount = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "message_server", Name: "ack_count", Help: "count of ack messages sent", }, []string{"to"}) serverRepeatedMessageCount = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "message_server", Name: "repeated_count", Help: "count of received repeated messages", }, []string{"from", "topic"}) grpcClientMetrics = grpc_prometheus.NewClientMetrics(func(opts *prometheus.CounterOpts) { - opts.Namespace = "ticdc" + opts.Namespace = "tikv_cdc" opts.Subsystem = "message_client" }) clientCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "message_client", Name: "client_count", Help: "count of messaging clients", }, []string{"to"}) clientMessageCount = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "message_client", Name: "message_count", Help: "count of messages sent", }, []string{"to"}) clientAckCount = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "ticdc", + Namespace: "tikv_cdc", Subsystem: "message_client", Name: "ack_count", Help: "count of ack messages received", diff --git a/cdc/pkg/p2p/mock_grpc_client.go b/cdc/pkg/p2p/mock_grpc_client.go index 8232d65f..c563c402 100644 --- a/cdc/pkg/p2p/mock_grpc_client.go +++ b/cdc/pkg/p2p/mock_grpc_client.go @@ -15,6 +15,7 @@ package p2p import ( "context" + "sync" "sync/atomic" "github.com/stretchr/testify/mock" @@ -24,6 +25,7 @@ import ( //nolint:unused type mockSendMessageClient struct { + mu sync.Mutex mock.Mock // embeds an empty interface p2p.CDCPeerToPeer_SendMessageClient @@ -41,16 +43,28 @@ func newMockSendMessageClient(ctx context.Context) *mockSendMessageClient { } func (s *mockSendMessageClient) Send(packet *p2p.MessagePacket) error { + s.mu.Lock() + defer s.mu.Unlock() + args := s.Called(packet) atomic.AddInt32(&s.msgCount, 1) return args.Error(0) } func (s *mockSendMessageClient) Recv() (*p2p.SendMessageResponse, error) { - args := s.Called() + var args mock.Arguments + func() { + // We use a deferred Unlock in case `s.Called()` panics. + s.mu.Lock() + defer s.mu.Unlock() + + args = s.MethodCalled("Recv") + }() + if err := args.Error(1); err != nil { return nil, err } + if args.Get(0) != nil { return args.Get(0).(*p2p.SendMessageResponse), nil } @@ -66,6 +80,14 @@ func (s *mockSendMessageClient) Context() context.Context { return s.ctx } +func (s *mockSendMessageClient) ResetMock() { + s.mu.Lock() + defer s.mu.Unlock() + + s.ExpectedCalls = nil + s.Calls = nil +} + //nolint:unused type mockCDCPeerToPeerClient struct { mock.Mock diff --git a/cdc/pkg/pipeline/pipeline.go b/cdc/pkg/pipeline/pipeline.go index b7c44358..5d55a1ab 100644 --- a/cdc/pkg/pipeline/pipeline.go +++ b/cdc/pkg/pipeline/pipeline.go @@ -96,7 +96,7 @@ func (p *Pipeline) driveRunner(ctx context.Context, previousRunner, runner runne err := runner.run(ctx) if err != nil { ctx.Throw(err) - if cerror.ErrTableProcessorStoppedSafely.NotEqual(err) { + if cerror.ErrKeySpanProcessorStoppedSafely.NotEqual(err) { log.Error("found error when running the node", zap.String("name", runner.getName()), zap.Error(err)) } } diff --git a/cdc/pkg/regionspan/span.go b/cdc/pkg/regionspan/span.go index 3324c52d..f5ee6601 100644 --- a/cdc/pkg/regionspan/span.go +++ b/cdc/pkg/regionspan/span.go @@ -23,9 +23,15 @@ import ( "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/util/codec" cerror "github.com/tikv/migration/cdc/pkg/errors" + "github.com/twmb/murmur3" "go.uber.org/zap" ) +const ( + RawKvStartKey = byte('r') + RawKvEndKey = byte('s') +) + // Span represents an arbitrary kv range type Span struct { Start []byte @@ -37,6 +43,15 @@ func (s Span) String() string { return fmt.Sprintf("[%s, %s)", hex.EncodeToString(s.Start), hex.EncodeToString(s.End)) } +func (s Span) ID() uint64 { + buf := make([]byte, 0, len(s.Start)+len(s.End)) + buf = append(buf, s.Start...) + buf = append(buf, s.End...) + h := murmur3.New64() + h.Write(buf) + return h.Sum64() +} + // UpperBoundKey represents the maximum value. var UpperBoundKey = []byte{255, 255, 255, 255, 255} diff --git a/cdc/pkg/scheduler/interface.go b/cdc/pkg/scheduler/interface.go index 11e3789e..229fad0a 100644 --- a/cdc/pkg/scheduler/interface.go +++ b/cdc/pkg/scheduler/interface.go @@ -18,7 +18,7 @@ import ( "github.com/tikv/migration/cdc/cdc/model" ) -// Scheduler is an abstraction for anything that provide the schedule table feature +// Scheduler is an abstraction for anything that provide the schedule keyspan feature type Scheduler interface { // ResetWorkloads resets the workloads info of the capture ResetWorkloads(captureID model.CaptureID, workloads model.TaskWorkload) @@ -30,19 +30,19 @@ type Scheduler interface { // returns * the skewness after rebalance // * the move jobs need by rebalance CalRebalanceOperates(targetSkewness float64) ( - skewness float64, moveTableJobs map[model.TableID]*model.MoveTableJob) - // DistributeTables distributes the new tables to the captures - // returns the operations of the new tables - DistributeTables(tableIDs map[model.TableID]model.Ts) map[model.CaptureID]map[model.TableID]*model.TableOperation + skewness float64, moveKeySpanJobs map[model.KeySpanID]*model.MoveKeySpanJob) + // DistributeKeySpans distributes the new keyspans to the captures + // returns the operations of the new keyspans + DistributeKeySpans(keyspanIDs map[model.KeySpanID]model.Ts) map[model.CaptureID]map[model.KeySpanID]*model.KeySpanOperation } // NewScheduler creates a new Scheduler func NewScheduler(tp string) Scheduler { switch tp { - case "table-number": - return newTableNumberScheduler() + case "keyspan-number": + return newKeySpanNumberScheduler() default: log.Info("invalid scheduler type, using default scheduler") - return newTableNumberScheduler() + return newKeySpanNumberScheduler() } } diff --git a/cdc/pkg/scheduler/keyspan_number.go b/cdc/pkg/scheduler/keyspan_number.go new file mode 100644 index 00000000..a1f83465 --- /dev/null +++ b/cdc/pkg/scheduler/keyspan_number.go @@ -0,0 +1,100 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package scheduler + +import "github.com/tikv/migration/cdc/cdc/model" + +// KeySpanNumberScheduler provides a feature that scheduling by the keyspan number +type KeySpanNumberScheduler struct { + workloads workloads +} + +// newKeySpanNumberScheduler creates a new keyspan number scheduler +func newKeySpanNumberScheduler() *KeySpanNumberScheduler { + return &KeySpanNumberScheduler{ + workloads: make(workloads), + } +} + +// ResetWorkloads implements the Scheduler interface +func (t *KeySpanNumberScheduler) ResetWorkloads(captureID model.CaptureID, workloads model.TaskWorkload) { + t.workloads.SetCapture(captureID, workloads) +} + +// AlignCapture implements the Scheduler interface +func (t *KeySpanNumberScheduler) AlignCapture(captureIDs map[model.CaptureID]struct{}) { + t.workloads.AlignCapture(captureIDs) +} + +// Skewness implements the Scheduler interface +func (t *KeySpanNumberScheduler) Skewness() float64 { + return t.workloads.Skewness() +} + +// CalRebalanceOperates implements the Scheduler interface +func (t *KeySpanNumberScheduler) CalRebalanceOperates(targetSkewness float64) ( + skewness float64, moveKeySpanJobs map[model.KeySpanID]*model.MoveKeySpanJob) { + var totalKeySpanNumber uint64 + for _, captureWorkloads := range t.workloads { + totalKeySpanNumber += uint64(len(captureWorkloads)) + } + limitKeySpanNumber := (float64(totalKeySpanNumber) / float64(len(t.workloads))) + 1 + appendKeySpans := make(map[model.KeySpanID]model.Ts) + moveKeySpanJobs = make(map[model.KeySpanID]*model.MoveKeySpanJob) + + for captureID, captureWorkloads := range t.workloads { + for float64(len(captureWorkloads)) >= limitKeySpanNumber { + for keyspanID := range captureWorkloads { + // find a keyspan in this capture + appendKeySpans[keyspanID] = 0 + moveKeySpanJobs[keyspanID] = &model.MoveKeySpanJob{ + From: captureID, + KeySpanID: keyspanID, + } + t.workloads.RemoveKeySpan(captureID, keyspanID) + break + } + } + } + addOperations := t.DistributeKeySpans(appendKeySpans) + for captureID, keyspanOperations := range addOperations { + for keyspanID := range keyspanOperations { + job := moveKeySpanJobs[keyspanID] + job.To = captureID + if job.From == job.To { + delete(moveKeySpanJobs, keyspanID) + } + } + } + skewness = t.Skewness() + return +} + +// DistributeKeySpans implements the Scheduler interface +func (t *KeySpanNumberScheduler) DistributeKeySpans(keyspanIDs map[model.KeySpanID]model.Ts) map[model.CaptureID]map[model.KeySpanID]*model.KeySpanOperation { + result := make(map[model.CaptureID]map[model.KeySpanID]*model.KeySpanOperation, len(t.workloads)) + for keyspanID, boundaryTs := range keyspanIDs { + captureID := t.workloads.SelectIdleCapture() + operations := result[captureID] + if operations == nil { + operations = make(map[model.KeySpanID]*model.KeySpanOperation) + result[captureID] = operations + } + operations[keyspanID] = &model.KeySpanOperation{ + BoundaryTs: boundaryTs, + } + t.workloads.SetKeySpan(captureID, keyspanID, model.WorkloadInfo{Workload: 1}) + } + return result +} diff --git a/cdc/pkg/scheduler/table_number_test.go b/cdc/pkg/scheduler/keyspan_number_test.go similarity index 80% rename from cdc/pkg/scheduler/table_number_test.go rename to cdc/pkg/scheduler/keyspan_number_test.go index 2ed6adb8..b5f0f72e 100644 --- a/cdc/pkg/scheduler/table_number_test.go +++ b/cdc/pkg/scheduler/keyspan_number_test.go @@ -22,9 +22,9 @@ import ( "github.com/stretchr/testify/require" ) -func TestDistributeTables(t *testing.T) { +func TestDistributeKeySpans(t *testing.T) { t.Parallel() - scheduler := newTableNumberScheduler() + scheduler := newKeySpanNumberScheduler() scheduler.ResetWorkloads("capture1", model.TaskWorkload{ 1: model.WorkloadInfo{Workload: 1}, 2: model.WorkloadInfo{Workload: 1}, @@ -40,26 +40,26 @@ func TestDistributeTables(t *testing.T) { 8: model.WorkloadInfo{Workload: 1}, }) require.Equal(t, fmt.Sprintf("%.2f%%", scheduler.Skewness()*100), "35.36%") - tableToAdd := map[model.TableID]model.Ts{10: 1, 11: 2, 12: 3, 13: 4, 14: 5, 15: 6, 16: 7, 17: 8} - result := scheduler.DistributeTables(tableToAdd) + keyspanToAdd := map[model.KeySpanID]model.Ts{10: 1, 11: 2, 12: 3, 13: 4, 14: 5, 15: 6, 16: 7, 17: 8} + result := scheduler.DistributeKeySpans(keyspanToAdd) require.Equal(t, len(result), 3) - totalTableNum := 0 + totalKeySpanNum := 0 for _, ops := range result { - for tableID, op := range ops { - ts, exist := tableToAdd[tableID] + for keyspanID, op := range ops { + ts, exist := keyspanToAdd[keyspanID] require.True(t, exist) require.False(t, op.Delete) require.Equal(t, op.BoundaryTs, ts) - totalTableNum++ + totalKeySpanNum++ } } - require.Equal(t, totalTableNum, 8) + require.Equal(t, totalKeySpanNum, 8) require.Equal(t, fmt.Sprintf("%.2f%%", scheduler.Skewness()*100), "8.84%") } func TestCalRebalanceOperates(t *testing.T) { t.Parallel() - scheduler := newTableNumberScheduler() + scheduler := newKeySpanNumberScheduler() scheduler.ResetWorkloads("capture1", model.TaskWorkload{ 1: model.WorkloadInfo{Workload: 1}, 2: model.WorkloadInfo{Workload: 1}, @@ -79,12 +79,12 @@ func TestCalRebalanceOperates(t *testing.T) { require.Equal(t, fmt.Sprintf("%.2f%%", scheduler.Skewness()*100), "56.57%") skewness, moveJobs := scheduler.CalRebalanceOperates(0) - for tableID, job := range moveJobs { + for keyspanID, job := range moveJobs { require.Greater(t, len(job.From), 0) require.Greater(t, len(job.To), 0) - require.Equal(t, job.TableID, tableID) + require.Equal(t, job.KeySpanID, keyspanID) require.NotEqual(t, job.From, job.To) - require.Equal(t, job.Status, model.MoveTableStatusNone) + require.Equal(t, job.Status, model.MoveKeySpanStatusNone) } require.Equal(t, fmt.Sprintf("%.2f%%", skewness*100), "14.14%") @@ -99,12 +99,12 @@ func TestCalRebalanceOperates(t *testing.T) { require.Equal(t, fmt.Sprintf("%.2f%%", scheduler.Skewness()*100), "141.42%") skewness, moveJobs = scheduler.CalRebalanceOperates(0) - for tableID, job := range moveJobs { + for keyspanID, job := range moveJobs { require.Greater(t, len(job.From), 0) require.Greater(t, len(job.To), 0) - require.Equal(t, job.TableID, tableID) + require.Equal(t, job.KeySpanID, keyspanID) require.NotEqual(t, job.From, job.To) - require.Equal(t, job.Status, model.MoveTableStatusNone) + require.Equal(t, job.Status, model.MoveKeySpanStatusNone) } require.Equal(t, fmt.Sprintf("%.2f%%", skewness*100), "0.00%") } diff --git a/cdc/pkg/scheduler/table_number.go b/cdc/pkg/scheduler/table_number.go deleted file mode 100644 index 980bfbe7..00000000 --- a/cdc/pkg/scheduler/table_number.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package scheduler - -import "github.com/tikv/migration/cdc/cdc/model" - -// TableNumberScheduler provides a feature that scheduling by the table number -type TableNumberScheduler struct { - workloads workloads -} - -// newTableNumberScheduler creates a new table number scheduler -func newTableNumberScheduler() *TableNumberScheduler { - return &TableNumberScheduler{ - workloads: make(workloads), - } -} - -// ResetWorkloads implements the Scheduler interface -func (t *TableNumberScheduler) ResetWorkloads(captureID model.CaptureID, workloads model.TaskWorkload) { - t.workloads.SetCapture(captureID, workloads) -} - -// AlignCapture implements the Scheduler interface -func (t *TableNumberScheduler) AlignCapture(captureIDs map[model.CaptureID]struct{}) { - t.workloads.AlignCapture(captureIDs) -} - -// Skewness implements the Scheduler interface -func (t *TableNumberScheduler) Skewness() float64 { - return t.workloads.Skewness() -} - -// CalRebalanceOperates implements the Scheduler interface -func (t *TableNumberScheduler) CalRebalanceOperates(targetSkewness float64) ( - skewness float64, moveTableJobs map[model.TableID]*model.MoveTableJob) { - var totalTableNumber uint64 - for _, captureWorkloads := range t.workloads { - totalTableNumber += uint64(len(captureWorkloads)) - } - limitTableNumber := (float64(totalTableNumber) / float64(len(t.workloads))) + 1 - appendTables := make(map[model.TableID]model.Ts) - moveTableJobs = make(map[model.TableID]*model.MoveTableJob) - - for captureID, captureWorkloads := range t.workloads { - for float64(len(captureWorkloads)) >= limitTableNumber { - for tableID := range captureWorkloads { - // find a table in this capture - appendTables[tableID] = 0 - moveTableJobs[tableID] = &model.MoveTableJob{ - From: captureID, - TableID: tableID, - } - t.workloads.RemoveTable(captureID, tableID) - break - } - } - } - addOperations := t.DistributeTables(appendTables) - for captureID, tableOperations := range addOperations { - for tableID := range tableOperations { - job := moveTableJobs[tableID] - job.To = captureID - if job.From == job.To { - delete(moveTableJobs, tableID) - } - } - } - skewness = t.Skewness() - return -} - -// DistributeTables implements the Scheduler interface -func (t *TableNumberScheduler) DistributeTables(tableIDs map[model.TableID]model.Ts) map[model.CaptureID]map[model.TableID]*model.TableOperation { - result := make(map[model.CaptureID]map[model.TableID]*model.TableOperation, len(t.workloads)) - for tableID, boundaryTs := range tableIDs { - captureID := t.workloads.SelectIdleCapture() - operations := result[captureID] - if operations == nil { - operations = make(map[model.TableID]*model.TableOperation) - result[captureID] = operations - } - operations[tableID] = &model.TableOperation{ - BoundaryTs: boundaryTs, - } - t.workloads.SetTable(captureID, tableID, model.WorkloadInfo{Workload: 1}) - } - return result -} diff --git a/cdc/pkg/scheduler/workload.go b/cdc/pkg/scheduler/workload.go index c414892f..f33f2ca7 100644 --- a/cdc/pkg/scheduler/workload.go +++ b/cdc/pkg/scheduler/workload.go @@ -38,33 +38,33 @@ func (w workloads) AlignCapture(captureIDs map[model.CaptureID]struct{}) { } } -func (w workloads) SetTable(captureID model.CaptureID, tableID model.TableID, workload model.WorkloadInfo) { +func (w workloads) SetKeySpan(captureID model.CaptureID, keyspanID model.KeySpanID, workload model.WorkloadInfo) { captureWorkloads, exist := w[captureID] if !exist { captureWorkloads = make(model.TaskWorkload) w[captureID] = captureWorkloads } - captureWorkloads[tableID] = workload + captureWorkloads[keyspanID] = workload } -func (w workloads) RemoveTable(captureID model.CaptureID, tableID model.TableID) { +func (w workloads) RemoveKeySpan(captureID model.CaptureID, keyspanID model.KeySpanID) { captureWorkloads, exist := w[captureID] if !exist { return } - delete(captureWorkloads, tableID) + delete(captureWorkloads, keyspanID) } -func (w workloads) AvgEachTable() uint64 { +func (w workloads) AvgEachKeySpan() uint64 { var totalWorkload uint64 - var totalTable uint64 + var totalKeySpan uint64 for _, captureWorkloads := range w { for _, workload := range captureWorkloads { totalWorkload += workload.Workload } - totalTable += uint64(len(captureWorkloads)) + totalKeySpan += uint64(len(captureWorkloads)) } - return totalWorkload / totalTable + return totalWorkload / totalKeySpan } func (w workloads) Skewness() float64 { @@ -107,8 +107,8 @@ func (w workloads) Clone() workloads { cloneWorkloads := make(map[model.CaptureID]model.TaskWorkload, len(w)) for captureID, captureWorkloads := range w { cloneCaptureWorkloads := make(model.TaskWorkload, len(captureWorkloads)) - for tableID, workload := range captureWorkloads { - cloneCaptureWorkloads[tableID] = workload + for keyspanID, workload := range captureWorkloads { + cloneCaptureWorkloads[keyspanID] = workload } cloneWorkloads[captureID] = cloneCaptureWorkloads } diff --git a/cdc/pkg/scheduler/workload_test.go b/cdc/pkg/scheduler/workload_test.go index bd674883..4869ad03 100644 --- a/cdc/pkg/scheduler/workload_test.go +++ b/cdc/pkg/scheduler/workload_test.go @@ -33,17 +33,17 @@ func TestWorkloads(t *testing.T) { 4: model.WorkloadInfo{Workload: 1}, 3: model.WorkloadInfo{Workload: 2}, }) - w.SetTable("capture2", 5, model.WorkloadInfo{Workload: 8}) - w.SetTable("capture3", 6, model.WorkloadInfo{Workload: 1}) - w.RemoveTable("capture1", 4) - w.RemoveTable("capture5", 4) - w.RemoveTable("capture1", 1) + w.SetKeySpan("capture2", 5, model.WorkloadInfo{Workload: 8}) + w.SetKeySpan("capture3", 6, model.WorkloadInfo{Workload: 1}) + w.RemoveKeySpan("capture1", 4) + w.RemoveKeySpan("capture5", 4) + w.RemoveKeySpan("capture1", 1) require.Equal(t, w, workloads{ "capture1": {2: model.WorkloadInfo{Workload: 2}}, "capture2": {4: model.WorkloadInfo{Workload: 1}, 3: model.WorkloadInfo{Workload: 2}, 5: model.WorkloadInfo{Workload: 8}}, "capture3": {6: model.WorkloadInfo{Workload: 1}}, }) - require.Equal(t, w.AvgEachTable(), uint64(2+1+2+8+1)/5) + require.Equal(t, w.AvgEachKeySpan(), uint64(2+1+2+8+1)/5) require.Equal(t, w.SelectIdleCapture(), "capture3") require.Equal(t, fmt.Sprintf("%.2f%%", w.Skewness()*100), "96.36%") diff --git a/cdc/pkg/util/ctx.go b/cdc/pkg/util/ctx.go index 492b00f8..d86d3730 100644 --- a/cdc/pkg/util/ctx.go +++ b/cdc/pkg/util/ctx.go @@ -25,7 +25,7 @@ import ( type ctxKey string const ( - ctxKeyTableID = ctxKey("tableID") + ctxKeyKeySpanID = ctxKey("keyspanID") ctxKeyCaptureAddr = ctxKey("captureAddr") ctxKeyChangefeedID = ctxKey("changefeedID") ctxKeyIsOwner = ctxKey("isOwner") @@ -58,19 +58,24 @@ func PutKVStorageInCtx(ctx context.Context, store kv.Storage) context.Context { return context.WithValue(ctx, ctxKeyKVStorage, store) } -type tableinfo struct { +type keyspaninfo struct { id int64 name string } -// PutTableInfoInCtx returns a new child context with the specified table ID and name stored. -func PutTableInfoInCtx(ctx context.Context, tableID int64, tableName string) context.Context { - return context.WithValue(ctx, ctxKeyTableID, tableinfo{id: tableID, name: tableName}) +// PutKeySpanInfoInCtx returns a new child context with the specified keyspan ID and name stored. +func PutKeySpanInfoInCtx(ctx context.Context, keyspanID int64, keyspanName string) context.Context { + return context.WithValue(ctx, ctxKeyKeySpanID, keyspaninfo{id: keyspanID, name: keyspanName}) } -// TableIDFromCtx returns a table ID -func TableIDFromCtx(ctx context.Context) (int64, string) { - info, ok := ctx.Value(ctxKeyTableID).(tableinfo) +// PutKeySpanInfoInCtx returns a new child context with the specified keyspan ID and name stored. +func PutKeySpanIDInCtx(ctx context.Context, keyspanID uint64) context.Context { + return context.WithValue(ctx, ctxKeyKeySpanID, keyspanID) +} + +// KeySpanIDFromCtx returns a kyspan ID +func KeySpanIDFromCtx(ctx context.Context) (int64, string) { + info, ok := ctx.Value(ctxKeyKeySpanID).(keyspaninfo) if !ok { return 0, "" } diff --git a/cdc/pkg/util/ctx_test.go b/cdc/pkg/util/ctx_test.go index e892f38f..06497a9f 100644 --- a/cdc/pkg/util/ctx_test.go +++ b/cdc/pkg/util/ctx_test.go @@ -83,23 +83,23 @@ func (s *ctxValueSuite) TestTimezoneNotSet(c *check.C) { c.Assert(tz, check.IsNil) } -func (s *ctxValueSuite) TestShouldReturnTableInfo(c *check.C) { +func (s *ctxValueSuite) TestShouldReturnKeySpanInfo(c *check.C) { defer testleak.AfterTest(c)() - ctx := PutTableInfoInCtx(context.Background(), 1321, "ello") - tableID, tableName := TableIDFromCtx(ctx) - c.Assert(tableID, check.Equals, int64(1321)) - c.Assert(tableName, check.Equals, "ello") + ctx := PutKeySpanInfoInCtx(context.Background(), 1321, "ello") + keyspanID, keyspanName := KeySpanIDFromCtx(ctx) + c.Assert(keyspanID, check.Equals, int64(1321)) + c.Assert(keyspanName, check.Equals, "ello") } -func (s *ctxValueSuite) TestTableInfoNotSet(c *check.C) { +func (s *ctxValueSuite) TestKeySpanInfoNotSet(c *check.C) { defer testleak.AfterTest(c)() - tableID, tableName := TableIDFromCtx(context.Background()) - c.Assert(tableID, check.Equals, int64(0)) - c.Assert(tableName, check.Equals, "") - ctx := context.WithValue(context.Background(), ctxKeyTableID, 1321) - tableID, tableName = TableIDFromCtx(ctx) - c.Assert(tableID, check.Equals, int64(0)) - c.Assert(tableName, check.Equals, "") + keyspanID, keyspanName := KeySpanIDFromCtx(context.Background()) + c.Assert(keyspanID, check.Equals, int64(0)) + c.Assert(keyspanName, check.Equals, "") + ctx := context.WithValue(context.Background(), ctxKeyKeySpanID, 1321) + keyspanID, keyspanName = KeySpanIDFromCtx(ctx) + c.Assert(keyspanID, check.Equals, int64(0)) + c.Assert(keyspanName, check.Equals, "") } func (s *ctxValueSuite) TestShouldReturnKVStorage(c *check.C) {