diff --git a/src/WriterPipeline.h b/src/WriterPipeline.h index c2db357f..cefb91e3 100644 --- a/src/WriterPipeline.h +++ b/src/WriterPipeline.h @@ -145,6 +145,7 @@ struct WriterPipeline { continue; } + event.preChecked = true; // pre-filter above already ran lookupEventById in ro txn newEventsToProc.emplace_back(std::move(event)); } } diff --git a/src/events.cpp b/src/events.cpp index c969b6f9..fbf02f90 100644 --- a/src/events.cpp +++ b/src/events.cpp @@ -266,7 +266,10 @@ void writeEvents(lmdb::txn &txn, NegentropyFilterCache &neFilterCache, std::vect PackedEventView packed(ev.packedStr); - if (lookupEventById(txn, packed.id()) || (i != 0 && ev.id() == evs[i-1].id())) { + bool isDup = (i != 0 && ev.id() == evs[i-1].id()); + if (!isDup && !ev.preChecked) isDup = (bool) lookupEventById(txn, packed.id()); + + if (isDup) { ev.status = EventWriteStatus::Duplicate; continue; } diff --git a/src/events.h b/src/events.h index 17296fc4..c7af9712 100644 --- a/src/events.h +++ b/src/events.h @@ -70,6 +70,10 @@ struct EventToWrite { void *userData = nullptr; EventWriteStatus status = EventWriteStatus::Pending; uint64_t levId = 0; + // Set when the caller already probed this id via lookupEventById() from the same + // thread that will open writeEvents()'s txn_rw (e.g. WriterPipeline). Do not set + // from cross-thread paths (e.g. RelayIngester) where the probe snapshot may be stale. + bool preChecked = false; EventToWrite() {}