Skip to content

Commit 3b81e47

Browse files
committed
fix: Hold back an override file that is emptied during an in-place rewrite
Applications and editors rewrite a file in place: they truncate it and then write the new content. A reload in between found the override file empty, chose YAML for it because it did not start with '{', and applied a document with no entries, so every override was briefly removed. The override loader now chooses the parser by the file's extension (.yaml and .yml are YAML, any other file is JSON), detects the text encoding from a byte order mark and reads UTF-8 otherwise, and rejects an empty JSON file and a JSON document that is not an object. A file whose content is in effect and that a load finds empty, white space, or YAML with no document is held back until it has stayed so, without being written, for the retry delay; the hold's retry is armed for the time left, and each file's hold is bounded at five retry delays from its first hold, starting over at a load that holds nothing back. A load reads every file, also after a failure, fails with the first failure, and merges the entries only after every file is read. The skip-unchanged digest covers each file's path and presence. The file data source keeps choosing the parser by content.
1 parent 0b4c68c commit 3b81e47

8 files changed

Lines changed: 2751 additions & 135 deletions

File tree

‎lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileDataPoller.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,14 @@ static List<FileState> observeAll(List<Path> paths) {
104104
return states;
105105
}
106106

107-
private static FileState observe(Path path) {
107+
/**
108+
* Observes one file. The attributes are read through any symbolic link, so they describe the file
109+
* that the link points to.
110+
*
111+
* @param path the file
112+
* @return its state, or {@link FileState#ABSENT} if its attributes cannot be read
113+
*/
114+
static FileState observe(Path path) {
108115
try {
109116
BasicFileAttributes attributes = Files.readAttributes(path, BasicFileAttributes.class);
110117
return new FileState(true, attributes.lastModifiedTime().toMillis(), attributes.size());

‎lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileDataReloader.java‎

Lines changed: 131 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,12 @@
66
import com.launchdarkly.sdk.server.integrations.OverrideFileLoader.LoadResult;
77

88
import java.io.Closeable;
9+
import java.nio.file.Path;
910
import java.time.Duration;
11+
import java.util.ArrayList;
1012
import java.util.Arrays;
13+
import java.util.Collections;
14+
import java.util.List;
1115
import java.util.concurrent.ScheduledFuture;
1216
import java.util.concurrent.ScheduledThreadPoolExecutor;
1317
import java.util.concurrent.ThreadFactory;
@@ -23,6 +27,10 @@
2327
* and returns the merged result, or throws {@link FileDataException} when a file cannot be read or
2428
* parsed or when the files cannot be merged.
2529
* <p>
30+
* A file whose content is in effect and is found empty may be in the middle of an in-place rewrite.
31+
* The loader can hold such a file back with {@link EmptiedFileException}: the reloader keeps the
32+
* current data, reports no failure, and reads the files again when the exception says.
33+
* <p>
2634
* The file data source does not use this class. It keeps its own reload behavior.
2735
*/
2836
final class FileDataReloader implements Closeable {
@@ -43,7 +51,55 @@ final class FileDataReloader implements Closeable {
4351
* Performs one full load of all configured files.
4452
*/
4553
interface Loader {
46-
LoadResult load() throws FileDataException;
54+
/**
55+
* Reads every configured file and merges the entries.
56+
*
57+
* @param retryDelay the reloader's retry delay, or zero if it does not retry. A loader may hold
58+
* an empty file back for a time based on it, because a retry reads the files again. With zero,
59+
* nothing would read them again, so it must not hold any back.
60+
* @return the merged result
61+
* @throws FileDataException if a file cannot be read or parsed, or the files cannot be merged
62+
* @throws EmptiedFileException if a file was found empty and is held back
63+
*/
64+
LoadResult load(Duration retryDelay) throws FileDataException, EmptiedFileException;
65+
66+
/**
67+
* Called with the result of a load that is now the data in effect: it was applied, or it was
68+
* skipped because it is unchanged. It is called with the reload lock held.
69+
*
70+
* @param result the result
71+
*/
72+
void accepted(LoadResult result);
73+
}
74+
75+
/**
76+
* Thrown by a {@link Loader} that holds back files it found empty. A file may be in the middle of an
77+
* in-place rewrite: a writer that truncates the file and then writes the new content.
78+
*/
79+
@SuppressWarnings("serial")
80+
static final class EmptiedFileException extends Exception {
81+
private final List<Path> paths;
82+
private final long retryAfterNanos;
83+
84+
EmptiedFileException(List<Path> paths, long retryAfterNanos) {
85+
super("files " + paths + " are empty");
86+
this.paths = Collections.unmodifiableList(new ArrayList<>(paths));
87+
this.retryAfterNanos = retryAfterNanos;
88+
}
89+
90+
/**
91+
* The files held back, in configuration order.
92+
*/
93+
List<Path> getPaths() {
94+
return paths;
95+
}
96+
97+
/**
98+
* How long to wait before loading again, in nanoseconds.
99+
*/
100+
long getRetryAfterNanos() {
101+
return retryAfterNanos;
102+
}
47103
}
48104

49105
/**
@@ -66,7 +122,8 @@ interface Handler {
66122
* Called when a reload fails, including when {@link #apply(LoadResult)} throws, once per
67123
* distinct failure. With automatic retries, repeats of an identical failure do not call this
68124
* again. A success re-arms it. The reloader logs failures itself, so implementations only need
69-
* to update their own state.
125+
* to update their own state. A load that defers an empty file and reads it again later is not a
126+
* failure and does not call this.
70127
*
71128
* @param e the failure
72129
*/
@@ -78,35 +135,44 @@ interface Handler {
78135
private final LDLogger logger;
79136
private final long debounceDelayMillis;
80137
private final long retryDelayMillis;
138+
// What reload() returns when no retry is needed.
139+
private static final long NO_RETRY = -1;
81140
private final boolean skipUnchanged;
82141

83142
// Serializes the load work between reloadNow and the worker thread.
84143
private final Object reloadLock = new Object();
85144
private byte[] lastGoodHash = null;
86145
private String lastErrorMessage = null;
146+
// True from a load that found an emptied file until the next load starts. Written inside the
147+
// reload lock. It only chooses the log message of the retry that such a load arms.
148+
private volatile boolean readAgainPending = false;
87149

88150
// Guards the worker and the two timers.
89151
private final Object timerLock = new Object();
90152
private ScheduledThreadPoolExecutor worker = null;
91153
private ScheduledFuture<?> debounceFuture = null;
92154
private ScheduledFuture<?> retryFuture = null;
155+
// The delay of the latest request to arm a retry, in nanoseconds, or -1 before the first. Visible for
156+
// tests through getLastRequestedRetryNanos.
157+
private volatile long lastRequestedRetryNanos = -1;
93158

94159
private final AtomicBoolean closed = new AtomicBoolean(false);
95160

96161
/**
97162
* Creates a reloader. The caller performs the initial load with {@link #reloadNow()}, routes
98163
* change signals to {@link #trigger()}, and calls {@link #close()} when finished.
99164
* <p>
100-
* The worker thread starts on the first {@link #trigger()} call or the first failed load that
101-
* arms a retry, not here. Construction alone does not start a thread.
165+
* The worker thread starts on the first {@link #trigger()} call or the first load that arms a
166+
* retry, not here. Construction alone does not start a thread.
102167
*
103168
* @param loader performs each full load
104169
* @param handler receives results and failures
105170
* @param logger receives log output about reloads and failures
106171
* @param debounceDelay how long to wait after a trigger for further triggers to settle before
107172
* reloading. Zero or negative reloads on every trigger without waiting.
108-
* @param retryDelay how long to wait after a failed reload before retrying without a trigger.
109-
* Zero or negative disables the automatic retry.
173+
* @param retryDelay how long to wait after a failed reload before retrying without a trigger. A
174+
* load that holds back an empty file says itself when to load again. Zero or negative disables
175+
* the automatic retry, and the loader then parses an empty file as it is.
110176
* @param skipUnchanged if true, a successful load whose raw file contents are identical to the
111177
* last applied contents does not call the apply callback
112178
*/
@@ -131,8 +197,9 @@ interface Handler {
131197
* initial load. A failure here arms the same automatic retry as a failed triggered reload.
132198
*/
133199
void reloadNow() {
134-
if (!reload() && retryDelayMillis > 0) {
135-
armRetry();
200+
long retryAfterNanos = reload();
201+
if (retryAfterNanos != NO_RETRY) {
202+
armRetry(retryAfterNanos);
136203
}
137204
}
138205

@@ -181,6 +248,16 @@ public void close() {
181248
}
182249
}
183250

251+
/**
252+
* Returns the delay of the latest request to arm a retry, whether or not it armed one. Visible for
253+
* tests.
254+
*
255+
* @return the delay in nanoseconds, or -1 if no retry was requested
256+
*/
257+
long getLastRequestedRetryNanos() {
258+
return lastRequestedRetryNanos;
259+
}
260+
184261
/**
185262
* Reports whether the worker thread has been started. Visible for tests.
186263
*
@@ -228,7 +305,8 @@ private ScheduledThreadPoolExecutor ensureWorker() {
228305
return worker;
229306
}
230307

231-
private void armRetry() {
308+
private void armRetry(long delayNanos) {
309+
lastRequestedRetryNanos = delayNanos;
232310
synchronized (timerLock) {
233311
if (closed.get()) {
234312
return;
@@ -237,15 +315,16 @@ private void armRetry() {
237315
if (retryFuture != null && !retryFuture.isDone()) {
238316
return;
239317
}
240-
retryFuture = ensureWorker().schedule(() -> runReload(true), retryDelayMillis, TimeUnit.MILLISECONDS);
318+
retryFuture = ensureWorker().schedule(() -> runReload(true), delayNanos, TimeUnit.NANOSECONDS);
241319
}
242320
}
243321

244322
// Runs on the worker thread for a debounced or a retried reload.
245323
private void runReload(boolean isRetry) {
246324
try {
247325
if (isRetry) {
248-
logger.debug("Retrying flag data load after earlier failure");
326+
logger.debug(readAgainPending ? "Reading flag data again after a file was found empty"
327+
: "Retrying flag data load after earlier failure");
249328
} else {
250329
logger.info("Reloading flag data after detecting a change");
251330
}
@@ -257,8 +336,9 @@ private void runReload(boolean isRetry) {
257336
retryFuture = null;
258337
}
259338
}
260-
if (!reload() && retryDelayMillis > 0) {
261-
armRetry();
339+
long retryAfterNanos = reload();
340+
if (retryAfterNanos != NO_RETRY) {
341+
armRetry(retryAfterNanos);
262342
}
263343
} catch (RuntimeException e) {
264344
// The executor would otherwise swallow the exception and the worker would go quiet.
@@ -267,27 +347,31 @@ private void runReload(boolean isRetry) {
267347
}
268348
}
269349

270-
// Performs one full load of all configured files. It reports whether the load succeeded, which
271-
// decides whether a retry gets armed. A skipped no-op application counts as success. The whole
272-
// set is re-read on every reload because entries are combined across files in order, so a
273-
// change to one file can alter which file wins for a key.
274-
private boolean reload() {
350+
// Performs one full load of all configured files. It returns when to load again: NO_RETRY after a
351+
// success, including a skipped no-op application, the retry delay after a failure, or the time that
352+
// a load that holds back an empty file asks for. The whole set is re-read on every reload because
353+
// entries are combined across files in order, so a change to one file can alter which file wins
354+
// for a key.
355+
private long reload() {
275356
synchronized (reloadLock) {
276357
// A trigger already queued when close was called can still reach here.
277358
if (closed.get()) {
278-
return true;
359+
return NO_RETRY;
279360
}
361+
readAgainPending = false;
280362

281363
LoadResult result;
282364
try {
283-
result = loader.load();
365+
result = loader.load(retryDelayMillis > 0 ? Duration.ofMillis(retryDelayMillis) : Duration.ZERO);
366+
} catch (EmptiedFileException e) {
367+
return readAgainLater(e);
284368
} catch (FileDataException e) {
285369
return fail(e);
286370
}
287371

288372
// Close may have happened while the files were being read. Deliver nothing in that case.
289373
if (closed.get()) {
290-
return true;
374+
return NO_RETRY;
291375
}
292376

293377
// A success right after a failure must apply even when the content is unchanged since the
@@ -296,7 +380,8 @@ private boolean reload() {
296380
boolean recovering = lastErrorMessage != null;
297381
byte[] hash = result.getContentHash();
298382
if (skipUnchanged && !recovering && hash != null && Arrays.equals(hash, lastGoodHash)) {
299-
return true;
383+
loader.accepted(result);
384+
return NO_RETRY;
300385
}
301386
// Nothing is remembered until the consumer has accepted the result. A result that the
302387
// consumer rejects must not become the baseline that skip-unchanged compares against, and
@@ -308,27 +393,44 @@ private boolean reload() {
308393
}
309394
lastErrorMessage = null;
310395
lastGoodHash = hash;
311-
return true;
396+
loader.accepted(result);
397+
return NO_RETRY;
398+
}
399+
}
400+
401+
// Called inside the reload lock. The loader held back files that it found empty: a file whose
402+
// content is in effect may be in the middle of an in-place rewrite, a writer that truncates the file
403+
// and then writes the new content. Whatever the files' format, the load keeps the current data and
404+
// arms the retry without reporting a failure, for the time that the loader asks for. The change
405+
// signal for the write starts a load sooner. The loader decides how long a file can be held back, and
406+
// holds nothing back when retries are off, because nothing would read the file again. A closed
407+
// reloader arms no retry.
408+
private long readAgainLater(EmptiedFileException e) {
409+
for (Path path : e.getPaths()) {
410+
logger.debug("File {} is empty, possibly while it is being rewritten; reading it again before applying it", path);
312411
}
412+
readAgainPending = true;
413+
return e.getRetryAfterNanos();
313414
}
314415

315-
private boolean fail(FileDataException e) {
316-
// Close may have happened while the files were being read. Deliver nothing and report success
317-
// so that no retry is armed.
416+
// Reports a failure and returns when to retry.
417+
private long fail(FileDataException e) {
418+
// Close may have happened while the files were being read. Deliver nothing and arm no retry.
318419
if (closed.get()) {
319-
return true;
420+
return NO_RETRY;
320421
}
422+
long retryAfterNanos = retryDelayMillis > 0 ? TimeUnit.MILLISECONDS.toNanos(retryDelayMillis) : NO_RETRY;
321423
// With automatic retries, a persistent failure would repeat the same log entry and the same
322424
// callback on every attempt. Repeats of an identical failure are logged at debug level and do
323425
// not call onError again. A consumer therefore sees one report per distinct failure.
324426
String description = describe(e);
325427
if (description.equals(lastErrorMessage)) {
326428
logger.debug("Unable to load flags: {}", description);
327-
return false;
429+
return retryAfterNanos;
328430
}
329431
lastErrorMessage = description;
330432
logger.error("Unable to load flags: {}", description);
331433
handler.onError(e);
332-
return false;
434+
return retryAfterNanos;
333435
}
334436
}

0 commit comments

Comments
 (0)