Skip to content

Fix two causes of flaky nightly builds #2311

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kounelisagis merged 2 commits into from
Sep 25, 2026
Merged

Fix two causes of flaky nightly builds #2311

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f56d3a0
Drop the wall-clock limit from the bytes round-trip test
kounelisagis Sep 25, 2026
962ffa1
Retry the TileDB release list download, and fail plainly if it can't …
kounelisagis Sep 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 24 additions & 7 deletions cmake/DownloadTileDB.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,20 +38,37 @@ function(fetch_tiledb_release_list VERSION)
set(VERSION latest)
endif()

set(RELLIST_URL ${UPSTREAM_URL}/${VERSION}/releases.csv)
if(EXPECTED_HASH)
file(DOWNLOAD
${UPSTREAM_URL}/${VERSION}/releases.csv
${CMAKE_CURRENT_BINARY_DIR}/releases.csv
SHOW_PROGRESS
EXPECTED_HASH ${EXPECTED_HASH}
)
set(HASH_ARGS EXPECTED_HASH ${EXPECTED_HASH})
else()
message(WARNING "Downloading release list without SHA checksum!")
endif()

# A failed download leaves an empty file and no error, which surfaces
# later as an unrelated one, and a CI matrix fetches this from dozens of
# jobs at once. So a failure is retried, then reported as what it is.
set(ATTEMPTS 3)
foreach(ATTEMPT RANGE 1 ${ATTEMPTS})
file(DOWNLOAD
${UPSTREAM_URL}/${VERSION}/releases.csv
${RELLIST_URL}
${CMAKE_CURRENT_BINARY_DIR}/releases.csv
SHOW_PROGRESS
${HASH_ARGS}
STATUS DOWNLOAD_STATUS
)
list(GET DOWNLOAD_STATUS 0 DOWNLOAD_CODE)
list(GET DOWNLOAD_STATUS 1 DOWNLOAD_MESSAGE)
if(DOWNLOAD_CODE EQUAL 0)
break()
endif()
message(WARNING "Downloading ${RELLIST_URL} failed (attempt ${ATTEMPT} of ${ATTEMPTS}): ${DOWNLOAD_MESSAGE}")
if(ATTEMPT LESS ATTEMPTS)
execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 5)
endif()
endforeach()
Comment on lines +52 to +69

@teo-tsirpanis teo-tsirpanis Sep 25, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not a fan of adding retry logic in our own build system code. I wish CMake supported that itself.

if(NOT DOWNLOAD_CODE EQUAL 0)
message(FATAL_ERROR "Could not download the TileDB release list from ${RELLIST_URL}: ${DOWNLOAD_MESSAGE}")
endif()

file(STRINGS
Expand Down
27 changes: 0 additions & 27 deletions tiledb/tests/test_hypothesis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
import time

import hypothesis as hp
import hypothesis.strategies as st
import numpy as np
Expand All @@ -18,14 +16,9 @@
@hp.settings(deadline=None, verbosity=hp.Verbosity.verbose)
@hp.given(st.binary())
def test_bytes_npdf(checked_path, mode, data):
start = time.time()

uri = "mem://" + checked_path.path()
hp.note(f"!!! path '{uri}' time: {time.time() - start}")

array = np.array([data], dtype="S0")

start_ingest = time.time()
if mode == "np":
with tiledb.from_numpy(uri, array) as A:
pass
Expand All @@ -35,28 +28,8 @@ def test_bytes_npdf(checked_path, mode, data):
# NOTE: ctx required here for mem://
tiledb.from_pandas(uri, df, sparse=False, ctx=tiledb.default_ctx())

hp.note(f"{mode} ingest time: {time.time() - start_ingest}")

# DEBUG
tiledb.stats_enable()
tiledb.stats_reset()
# END DEBUG

with tiledb.open(uri) as A:
if mode == "np":
np.testing.assert_array_equal(A.multi_index[:][""], array)
else:
tm.assert_frame_equal(A.df[:], df)

hp.note(tiledb.stats_dump(print_out=False))

# DEBUG
tiledb.stats_disable()

duration = time.time() - start
hp.note(f"!!! test_bytes_{mode} duration: {duration}")
if duration > 2:
# Hypothesis setup is (maybe) causing deadline exceeded errors
# https://github.com/TileDB-Inc/TileDB-Py/issues/1194
# Set deadline=None and use internal timing instead.
pytest.fail(f"!!! {mode} function body duration exceeded 2s: {duration}")
Loading