From 40ac22e402a828320043b43249254595fb7b3a03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BE=99=E6=96=B9=E6=B7=9E?= Date: Fri, 26 Nov 2021 09:48:56 +0800 Subject: [PATCH 1/5] Create 0081-flashback.md Add flashback Signed-off-by: longfangsong --- text/0081-flashback.md | 104 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 text/0081-flashback.md diff --git a/text/0081-flashback.md b/text/0081-flashback.md new file mode 100644 index 00000000..251ad91c --- /dev/null +++ b/text/0081-flashback.md @@ -0,0 +1,104 @@ +# RFC: Return TiKV to some certain version + +## Summary + +This proposal introduces a method to return TiKV to some certain version, which could be useful when doing some lossy recovery. + +## Motivation + +The initial motivation is to do lossy recovery on TiKV when deploying TiDB cluster across DCs with DR Auto-sync method. + +When use DR Auto-sync method to deploy TiDB cluster in two DC and + +- there is a network isolation between primary and DR DC +- PD make the replica state of primary DC to be Async after a while +- the primary DC is down + +Then the sync state of the cluster (from the DR DC's PD) is not dependable. In this case, data can be partially sent to the TiKV nodes in DR, and break the ACID consistency. + +Though it is impossible to do lossless recovery in this situation, we can recover the data on TiKV nodes to a certain version (`resolved_ts` in this case), which is ACID consistency. + +## Detailed design + +Basically, we need a method to return TiKV to a certain version (timestamp), we should add a new RPC to kvproto: + +```protobuf +message FlashBackRequest { + Context context = 1; + uint64 checkpoint_ts = 2; +} + +message FlashBackResponse { +} + +service Tikv { + // … + rpc FlashBack(kvrpcpb.FlashBackRequest) returns (kvrpcpb.FlashBackResponse) {} +} +``` + +When a TiKV recieve `FlashBackRequest`, it will create a `FlashBackWorker` instance and start the work. + +There are several steps to follow when doing the flash back: + +### 1. Wait all committed raft log are applied + +We can just polling each region's `RegionInfo` in a method similar with [`Debugger::region_info`](https://github.com/tikv/tikv/blob/789c99666f2f9faaa6c6e5b021ac0cf7a76ae24e/src/server/debug.rs#L188) does, and wait for `commit_index` to be equal to `apply_index`. + +### 2. Clean WriteCF and DefaultCF + W +Then we can remove all data created by transactions which commited after `checkpoint_ts`, this can be done by remove all `Write` records which `commit_ts > checkpoint_ts` and corresponding data in `DEFAULT_CF` from KVEngine, ie. + +```rust +fn scan_next_batch(&mut self, batch_size: usize) -> Option, Write)>> { + let mut writes = None; + for _ in 0..batch_size { + if let Some((key, write)) = self.next_write()? { + let commit_ts = Key::decode_ts_from(keys::origin_key(&key)); + let mut writes = writes.get_or_insert(Vec::new()); + if commit_ts > self.ts { + writes.push((key, write)); + } + } else { + return writes; + } + } + writes +} + +pub fn process_next_batch( + &mut self, + batch_size: usize, + wb: &mut RocksWriteBatch, +) -> bool { + let writes = if let Some(writes) = self.scan_next_batch(batch_size) { + writes + } else { + return false; + } + for (key, write) in writes { + let default_key = Key::append_ts(Key::from_raw(&key), write.start_ts).to_raw().unwrap(); + box_try!(wb.delete_cf(CF_WRITE, &key)); + box_try!(wb.delete_cf(CF_DEFAULT, &default_key)); + } + true +} +``` + +Note we scan and process data in batches, so we can report the progress to the client. + +### 3. Resolve locks once + +After remove all the writes, we can promise we won't give the user ACID inconsistency data when processing read, however, there might be still locks in lock cf which block some user's operation. + +So we should do resolve locks on all data once. + +This lock resolving process is just like the one used by GC, with safepoint ts equals to current ts. + +## Drawbacks + +This is a **very** unsafe operation, we should prevent the user call this on normal cluster. + +## Alternatives + +- We can also clean the lock cf in step2 instead of do resolve locks. From 9dc2a778f74bbed2402d0782fdec172a71679421 Mon Sep 17 00:00:00 2001 From: longfangsong Date: Fri, 3 Dec 2021 10:06:55 +0800 Subject: [PATCH 2/5] rename flashback into truncate Signed-off-by: longfangsong --- text/{0081-flashback.md => 0081-truncate.md} | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) rename text/{0081-flashback.md => 0081-truncate.md} (87%) diff --git a/text/0081-flashback.md b/text/0081-truncate.md similarity index 87% rename from text/0081-flashback.md rename to text/0081-truncate.md index 251ad91c..e5dc0f67 100644 --- a/text/0081-flashback.md +++ b/text/0081-truncate.md @@ -23,31 +23,32 @@ Though it is impossible to do lossless recovery in this situation, we can recove Basically, we need a method to return TiKV to a certain version (timestamp), we should add a new RPC to kvproto: ```protobuf -message FlashBackRequest { +message TruncateRequest { Context context = 1; uint64 checkpoint_ts = 2; } -message FlashBackResponse { +message TruncateResponse { + string error = 1; } service Tikv { // … - rpc FlashBack(kvrpcpb.FlashBackRequest) returns (kvrpcpb.FlashBackResponse) {} + rpc Truncate(kvrpcpb.TruncateRequest) returns (kvrpcpb.TruncateResponse) {} } ``` -When a TiKV recieve `FlashBackRequest`, it will create a `FlashBackWorker` instance and start the work. +When a TiKV receive `TruncateRequest`, it will create a `TruncateWorker` instance and start the work. -There are several steps to follow when doing the flash back: +There are several steps to follow when doing truncate: ### 1. Wait all committed raft log are applied We can just polling each region's `RegionInfo` in a method similar with [`Debugger::region_info`](https://github.com/tikv/tikv/blob/789c99666f2f9faaa6c6e5b021ac0cf7a76ae24e/src/server/debug.rs#L188) does, and wait for `commit_index` to be equal to `apply_index`. ### 2. Clean WriteCF and DefaultCF - W -Then we can remove all data created by transactions which commited after `checkpoint_ts`, this can be done by remove all `Write` records which `commit_ts > checkpoint_ts` and corresponding data in `DEFAULT_CF` from KVEngine, ie. + +Then we can remove all data created by transactions which committed after `checkpoint_ts`, this can be done by remove all `Write` records which `commit_ts > checkpoint_ts` and corresponding data in `DEFAULT_CF` from KVEngine, ie. ```rust fn scan_next_batch(&mut self, batch_size: usize) -> Option, Write)>> { From a2cc55c1d045f29f7b37dafdda61768af4a1882e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BE=99=E6=96=B9=E6=B7=9E?= Date: Mon, 6 Dec 2021 11:37:03 +0800 Subject: [PATCH 3/5] Update text/0081-truncate.md Co-authored-by: Lei Zhao Update text/0081-truncate.md Co-authored-by: Lei Zhao fix indent Signed-off-by: longfangsong --- text/0081-truncate.md | 56 +++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/text/0081-truncate.md b/text/0081-truncate.md index e5dc0f67..17b4154e 100644 --- a/text/0081-truncate.md +++ b/text/0081-truncate.md @@ -33,8 +33,8 @@ message TruncateResponse { } service Tikv { - // … - rpc Truncate(kvrpcpb.TruncateRequest) returns (kvrpcpb.TruncateResponse) {} + // … + rpc Truncate(kvrpcpb.TruncateRequest) returns (kvrpcpb.TruncateResponse) {} } ``` @@ -44,7 +44,7 @@ There are several steps to follow when doing truncate: ### 1. Wait all committed raft log are applied -We can just polling each region's `RegionInfo` in a method similar with [`Debugger::region_info`](https://github.com/tikv/tikv/blob/789c99666f2f9faaa6c6e5b021ac0cf7a76ae24e/src/server/debug.rs#L188) does, and wait for `commit_index` to be equal to `apply_index`. +We can just polling each region's `RegionInfo` in a method similar with [`Debugger::region_info`](https://github.com/tikv/tikv/blob/789c99666f2f9faaa6c6e5b021ac0cf7a76ae24e/src/server/debug.rs#L188) does, and wait for `apply_index` to be equal to `commit_index`. ### 2. Clean WriteCF and DefaultCF @@ -52,37 +52,37 @@ Then we can remove all data created by transactions which committed after `check ```rust fn scan_next_batch(&mut self, batch_size: usize) -> Option, Write)>> { - let mut writes = None; - for _ in 0..batch_size { - if let Some((key, write)) = self.next_write()? { - let commit_ts = Key::decode_ts_from(keys::origin_key(&key)); - let mut writes = writes.get_or_insert(Vec::new()); - if commit_ts > self.ts { - writes.push((key, write)); - } - } else { - return writes; + let mut writes = None; + for _ in 0..batch_size { + if let Some((key, write)) = self.next_write()? { + let commit_ts = Key::decode_ts_from(keys::origin_key(&key)); + let mut writes = writes.get_or_insert(Vec::new()); + if commit_ts > self.ts { + writes.push((key, write)); } + } else { + return writes; } - writes + } + writes } pub fn process_next_batch( - &mut self, - batch_size: usize, - wb: &mut RocksWriteBatch, + &mut self, + batch_size: usize, + wb: &mut RocksWriteBatch, ) -> bool { - let writes = if let Some(writes) = self.scan_next_batch(batch_size) { - writes - } else { - return false; - } - for (key, write) in writes { - let default_key = Key::append_ts(Key::from_raw(&key), write.start_ts).to_raw().unwrap(); - box_try!(wb.delete_cf(CF_WRITE, &key)); - box_try!(wb.delete_cf(CF_DEFAULT, &default_key)); - } - true + let writes = if let Some(writes) = self.scan_next_batch(batch_size) { + writes + } else { + return false; + } + for (key, write) in writes { + let default_key = Key::append_ts(Key::from_raw(&key), write.start_ts).to_raw().unwrap(); + box_try!(wb.delete_cf(CF_WRITE, &key)); + box_try!(wb.delete_cf(CF_DEFAULT, &default_key)); + } + true } ``` From 95b955cae84e6e3dc374a85405c6f10e7280d729 Mon Sep 17 00:00:00 2001 From: longfangsong Date: Mon, 6 Dec 2021 12:42:25 +0800 Subject: [PATCH 4/5] add some note about no more write requests should be sent to TiKV during the truncate process Signed-off-by: longfangsong --- text/0081-truncate.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/text/0081-truncate.md b/text/0081-truncate.md index 17b4154e..a086250d 100644 --- a/text/0081-truncate.md +++ b/text/0081-truncate.md @@ -40,7 +40,7 @@ service Tikv { When a TiKV receive `TruncateRequest`, it will create a `TruncateWorker` instance and start the work. -There are several steps to follow when doing truncate: +There are several steps to follow when doing truncate, note it is expected that no more write requests will be sent to TiKV during the truncate process: ### 1. Wait all committed raft log are applied From a5c7eb9dfb0fc3208a54ca142b82f77f317c26cc Mon Sep 17 00:00:00 2001 From: longfangsong Date: Fri, 10 Dec 2021 10:05:47 +0800 Subject: [PATCH 5/5] address comments Signed-off-by: longfangsong --- ...1-truncate.md => 0081-reset-to-version.md} | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) rename text/{0081-truncate.md => 0081-reset-to-version.md} (86%) diff --git a/text/0081-truncate.md b/text/0081-reset-to-version.md similarity index 86% rename from text/0081-truncate.md rename to text/0081-reset-to-version.md index a086250d..65b082aa 100644 --- a/text/0081-truncate.md +++ b/text/0081-reset-to-version.md @@ -23,22 +23,21 @@ Though it is impossible to do lossless recovery in this situation, we can recove Basically, we need a method to return TiKV to a certain version (timestamp), we should add a new RPC to kvproto: ```protobuf -message TruncateRequest { - Context context = 1; - uint64 checkpoint_ts = 2; +message ResetToVersionRequest { + uint64 checkpoint_ts = 1; } -message TruncateResponse { - string error = 1; +message ResetToVersionResponse { } service Tikv { // … - rpc Truncate(kvrpcpb.TruncateRequest) returns (kvrpcpb.TruncateResponse) {} + rpc ResetToVersion(ResetToVersionRequest) returns (ResetToVersionResponse) {} + rpc ResetToVersionStatus(ResetToVersionStatusRequest) returns (ResetToVersionStatus) {} } ``` -When a TiKV receive `TruncateRequest`, it will create a `TruncateWorker` instance and start the work. +When a TiKV receive `ResetToVersionRequest`, it will create a `ResetToVersionWorker` instance and start the work. There are several steps to follow when doing truncate, note it is expected that no more write requests will be sent to TiKV during the truncate process: @@ -94,12 +93,11 @@ After remove all the writes, we can promise we won't give the user ACID inconsis So we should do resolve locks on all data once. -This lock resolving process is just like the one used by GC, with safepoint ts equals to current ts. +We can do this by simply remove all locks in lockcf. + + ## Drawbacks This is a **very** unsafe operation, we should prevent the user call this on normal cluster. -## Alternatives - -- We can also clean the lock cf in step2 instead of do resolve locks.