Skip to content

Commit 68fbad4

Browse files
apkardongxinEric
apkar
authored and
dongxinEric
committed
Make UnboundCollectionContext versioned. (#141)
1 parent 5f5c290 commit 68fbad4

5 files changed

Lines changed: 56 additions & 40 deletions

File tree

‎src/Constants.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ const uint64_t FDB_KEY_LENGTH_LIMIT = (uint64_t)1e4;
2828
const uint64_t FDB_VALUE_LENGTH_LIMIT = (uint64_t)1e5;
2929

3030
const uint64_t METADATA_CACHE_SIZE = (uint64_t)100;
31+
const uint64_t METADATA_INVALID_VERSION = (uint64_t)-1;
3132

3233
const std::string METADATA = "metadata";
3334
const std::string VERSION_KEY = "version";

‎src/Constants.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ extern const uint64_t FDB_VALUE_LENGTH_LIMIT;
3636

3737
// Size of metadata cache in entries, number of collections
3838
extern const uint64_t METADATA_CACHE_SIZE;
39+
extern const uint64_t METADATA_INVALID_VERSION;
3940

4041
// KVS DocLayer internal keys
4142
extern const std::string METADATA;

‎src/MetadataManager.actor.cpp‎

Lines changed: 36 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,10 @@ IndexInfo MetadataManager::indexInfoFromObj(const bson::BSONObj& indexObj, Refer
8686
}
8787
}
8888

89-
ACTOR static Future<std::pair<Reference<UnboundCollectionContext>, uint64_t>>
90-
constructContext(Namespace ns, Reference<DocTransaction> tr, DocumentLayer* docLayer, bool createCollectionIfAbsent) {
89+
ACTOR static Future<Reference<UnboundCollectionContext>> constructContext(Namespace ns,
90+
Reference<DocTransaction> tr,
91+
DocumentLayer* docLayer,
92+
bool createCollectionIfAbsent) {
9193
try {
9294
// The initial set of directory reads take place in a separate transaction with the same read version as `tr'.
9395
// This hopefully prevents us from accidentally RYWing a directory that `tr' itself created, and then adding it
@@ -105,26 +107,22 @@ constructContext(Namespace ns, Reference<DocTransaction> tr, DocumentLayer* docL
105107
state Future<uint64_t> fv = getMetadataVersion(tr, metadataDirectory);
106108
state Reference<DirectorySubspace> collectionDirectory = wait(fcollectionDirectory);
107109
state Reference<DirectorySubspace> indexDirectory = wait(findexDirectory);
108-
state Reference<UnboundCollectionContext> cx =
109-
Reference<UnboundCollectionContext>(new UnboundCollectionContext(collectionDirectory, metadataDirectory));
110-
111110
state Reference<UnboundCollectionContext> indexCx = Reference<UnboundCollectionContext>(
112111
new UnboundCollectionContext(indexDirectory, Reference<DirectorySubspace>()));
113112
state Reference<Plan> indexesPlan = getIndexesForCollectionPlan(indexCx, ns);
114-
std::vector<bson::BSONObj> allIndexes = wait(getIndexesTransactionally(indexesPlan, tr));
113+
state std::vector<bson::BSONObj> allIndexes = wait(getIndexesTransactionally(indexesPlan, tr));
114+
115+
uint64_t version = wait(fv);
116+
state Reference<UnboundCollectionContext> cx = Reference<UnboundCollectionContext>(
117+
new UnboundCollectionContext(version, collectionDirectory, metadataDirectory));
115118

116119
for (const auto& indexObj : allIndexes) {
117120
IndexInfo index = MetadataManager::indexInfoFromObj(indexObj, cx);
118121
if (index.status != IndexInfo::IndexStatus::INVALID) {
119122
cx->addIndex(index);
120123
}
121124
}
122-
123-
// fprintf(stderr, "%s.%s Reading: Collection dir: %s Metadata dir:%s Caller:%s\n", dbName.c_str(),
124-
// collectionName.c_str(), printable(collectionDirectory->key()).c_str(),
125-
// printable(metadataDirectory->key()).c_str(), "");
126-
uint64_t version = wait(fv);
127-
return std::make_pair(cx, version);
125+
return cx;
128126
} catch (Error& e) {
129127
if (e.code() != error_code_directory_does_not_exist && e.code() != error_code_parent_directory_does_not_exist)
130128
throw;
@@ -148,69 +146,71 @@ constructContext(Namespace ns, Reference<DocTransaction> tr, DocumentLayer* docL
148146
tr->tr, {StringRef(ns.first), StringRef(ns.second), StringRef(DocLayerConstants::METADATA)}));
149147
state Reference<UnboundCollectionContext> tcx =
150148
Reference<UnboundCollectionContext>(new UnboundCollectionContext(tcollectionDirectory, tmetadataDirectory));
151-
// fprintf(stderr, "%s.%s Creating: Collection dir: %s Metadata dir:%s Caller:%s\n", dbName.c_str(),
152-
// collectionName.c_str(), printable(tcollectionDirectory->key()).c_str(),
153-
// printable(tmetadataDirectory->key()).c_str(), "");
149+
154150
tcx->bindCollectionContext(tr)->bumpMetadataVersion(); // We start at version 1.
155151
TraceEvent(SevInfo, "BumpMetadataVersion")
156152
.detail("reason", "createCollection")
157153
.detail("ns", fullCollNameToString(ns));
158154

159-
return std::make_pair(tcx, -1); // So we don't pollute the cache in case this transaction never commits
155+
return tcx;
160156
}
161157
}
162158

163159
ACTOR static Future<Reference<UnboundCollectionContext>> assembleCollectionContext(Reference<DocTransaction> tr,
164160
Namespace ns,
165161
Reference<MetadataManager> self,
166162
bool createCollectionIfAbsent) {
167-
if (self->contexts.size() > DocLayerConstants::METADATA_CACHE_SIZE)
168-
self->contexts.clear();
163+
if (self->metadataCache.size() > DocLayerConstants::METADATA_CACHE_SIZE)
164+
self->metadataCache.clear();
169165

170-
auto match = self->contexts.find(ns);
166+
auto match = self->metadataCache.find(ns);
171167

172-
if (match == self->contexts.end()) {
173-
std::pair<Reference<UnboundCollectionContext>, uint64_t> unboundPair =
168+
if (match == self->metadataCache.end()) {
169+
Reference<UnboundCollectionContext> cx =
174170
wait(constructContext(ns, tr, self->docLayer, createCollectionIfAbsent));
175171

176172
// Here and below don't pollute the cache if we just created the directory, since this transaction might
177173
// not commit.
178-
if (unboundPair.second != -1) {
174+
if (cx->isVersioned()) {
179175
TraceEvent(SevInfo, "MetadataCacheAdd")
180176
.detail("ns", fullCollNameToString(ns))
181-
.detail("version", unboundPair.second);
182-
auto insert_result = self->contexts.insert(std::make_pair(ns, unboundPair));
177+
.detail("version", cx->metadataVersion);
178+
auto insert_result = self->metadataCache.insert(std::make_pair(ns, cx));
183179
// Somebody else may have done the lookup and finished ahead of us. Either way, replace it with ours (can no
184180
// longer optimize this by only replacing if ours is newer, because the directory may have moved or
185181
// vanished.
186182
if (!insert_result.second) {
187-
insert_result.first->second = unboundPair;
183+
insert_result.first->second = cx;
188184
}
189185
}
190-
return unboundPair.first;
186+
return cx;
191187
} else {
192-
state uint64_t oldVersion = (*match).second.second;
193-
state Reference<UnboundCollectionContext> oldUnbound = (*match).second.first;
188+
state Reference<UnboundCollectionContext> oldUnbound = (*match).second;
189+
state uint64_t oldVersion = oldUnbound->metadataVersion;
190+
191+
// We would never cache a collection without valid version
192+
ASSERT(oldUnbound->isVersioned());
193+
194194
uint64_t version = wait(getMetadataVersion(tr, oldUnbound->metadataDirectory));
195195
if (version != oldVersion) {
196-
std::pair<Reference<UnboundCollectionContext>, uint64_t> unboundPair =
196+
Reference<UnboundCollectionContext> cx =
197197
wait(constructContext(ns, tr, self->docLayer, createCollectionIfAbsent));
198-
if (unboundPair.second != -1) {
198+
if (cx->isVersioned()) {
199199
// Create the iterator again instead of making the previous value state, because the map could have
200200
// changed during the previous wait. Either way, replace it with ours (can no longer optimize this by
201201
// only replacing if ours is newer, because the directory may have moved or vanished.
202-
auto match = self->contexts.find(ns);
203-
if (match != self->contexts.end())
204-
match->second = unboundPair;
202+
auto it = self->metadataCache.find(ns);
203+
if (it != self->metadataCache.end())
204+
it->second = cx;
205205
else
206-
self->contexts.insert(std::make_pair(ns, unboundPair));
206+
self->metadataCache.insert(std::make_pair(ns, cx));
207207

208208
TraceEvent(SevInfo, "MetadataCacheUpdate")
209209
.detail("ns", fullCollNameToString(ns))
210210
.detail("oldVersion", oldVersion)
211-
.detail("newVersion", unboundPair.second);
211+
.detail("newVersion", cx->metadataVersion);
212212
}
213-
return unboundPair.first;
213+
return cx;
214214
} else {
215215
return oldUnbound;
216216
}

‎src/MetadataManager.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ struct MetadataManager : ReferenceCounted<MetadataManager>, NonCopyable {
5757
UID build_id);
5858
static IndexInfo indexInfoFromObj(const bson::BSONObj& indexObj, Reference<UnboundCollectionContext> cx);
5959

60-
std::map<Namespace, std::pair<Reference<UnboundCollectionContext>, uint64_t>> contexts;
60+
std::map<Namespace, Reference<UnboundCollectionContext>> metadataCache;
6161
DocumentLayer* docLayer;
6262
};
6363

‎src/QLContext.h‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -372,14 +372,24 @@ struct IndexComparator {
372372
};
373373

374374
struct UnboundCollectionContext : ReferenceCounted<UnboundCollectionContext>, FastAllocated<UnboundCollectionContext> {
375-
UnboundCollectionContext(Reference<DirectorySubspace> collectionDirectory,
375+
UnboundCollectionContext(uint64_t metadataVersion,
376+
Reference<DirectorySubspace> collectionDirectory,
376377
Reference<DirectorySubspace> metadataDirectory)
377-
: collectionDirectory(collectionDirectory), metadataDirectory(metadataDirectory) {
378+
: metadataVersion(metadataVersion),
379+
collectionDirectory(collectionDirectory),
380+
metadataDirectory(metadataDirectory) {
378381
cx = Reference<UnboundQueryContext>(new UnboundQueryContext())->getSubContext(collectionDirectory->key());
379382
}
380383

384+
UnboundCollectionContext(Reference<DirectorySubspace> collectionDirectory,
385+
Reference<DirectorySubspace> metadataDirectory)
386+
: UnboundCollectionContext(DocLayerConstants::METADATA_INVALID_VERSION,
387+
collectionDirectory,
388+
metadataDirectory) {}
389+
381390
UnboundCollectionContext(const UnboundCollectionContext& other)
382-
: collectionDirectory(other.collectionDirectory),
391+
: metadataVersion(other.metadataVersion),
392+
collectionDirectory(other.collectionDirectory),
383393
metadataDirectory(other.metadataDirectory),
384394
simpleIndexMap(other.simpleIndexMap),
385395
knownIndexes(other.knownIndexes),
@@ -399,6 +409,8 @@ struct UnboundCollectionContext : ReferenceCounted<UnboundCollectionContext>, Fa
399409
std::string databaseName();
400410
std::string collectionName();
401411

412+
bool isVersioned() { return metadataVersion != DocLayerConstants::METADATA_INVALID_VERSION; }
413+
402414
Reference<DirectorySubspace> collectionDirectory;
403415
Reference<DirectorySubspace> metadataDirectory;
404416

@@ -411,6 +423,8 @@ struct UnboundCollectionContext : ReferenceCounted<UnboundCollectionContext>, Fa
411423
// include indexes that are still building
412424
std::vector<IndexInfo> knownIndexes;
413425

426+
const uint64_t metadataVersion;
427+
414428
private:
415429
std::set<std::string> bannedFieldNames;
416430
};

0 commit comments

Comments
 (0)