diff --git a/git-reader/README.md b/git-reader/README.md index ac8184bf..6935de82 100644 --- a/git-reader/README.md +++ b/git-reader/README.md @@ -129,3 +129,11 @@ For example, every 5 minutes in a cronjob: ```bash */5 * * * * docker run ... ``` + +## Git Source Settings + +With ``SELF_CONTAINED=true``, the attachments files are served by the application, and must be fetched using LFS. The following settings control disk usage and transfer speed. + +- ``LFS_CONCURRENT_TRANSFERS`` (default: 8): increase number of parallel requests for LFS downloads. +- ``LFS_FETCH_EXCLUDE`` (default: none): exclude certain collections from LFS (eg. `"attachments/main-workspace/translation-dictionaries/*,attachments/main-workspace/quicksuggest-amp/*"`) +- ``LFS_KEEP_DAYS`` (default: unset, see [LFS defaults](https://github.com/git-lfs/git-lfs/blob/main/docs/man/git-lfs-config.adoc)): a disk-space knob for the LFS cache that controls how many days of LFS history is retained beyond the current checkout. Objects referenced by ``HEAD`` are always kept. In Remote Settings terms, this means that clients trying to pull attachments of records that have been obsolete for more than `LFS_KEEP_DAYS` days will be served an error response. This can be mitigated using a caching layer on the reverse proxy, which would continue to serve obsolete attachments as long as they remained cached. diff --git a/git-reader/run.sh b/git-reader/run.sh index c7fb9f22..d612784c 100644 --- a/git-reader/run.sh +++ b/git-reader/run.sh @@ -30,6 +30,28 @@ cmd_gitupdate() { start_total=$(date +%s) log "Repository path is $repo_path" + # Apply LFS tuning from environment variables. If a variable is not set, + # leave the corresponding Git config untouched. + # We set them globally because they affect cloning. + if [ -n "${LFS_CONCURRENT_TRANSFERS:-}" ]; then + log "Setting lfs.concurrenttransfers to ${LFS_CONCURRENT_TRANSFERS}." + git config --global lfs.concurrenttransfers "${LFS_CONCURRENT_TRANSFERS}" + fi + if [ -n "${LFS_FETCH_EXCLUDE:-}" ]; then + log "Setting lfs.fetchexclude to ${LFS_FETCH_EXCLUDE}." + git config --global lfs.fetchexclude "${LFS_FETCH_EXCLUDE}" + fi + if [ -n "${LFS_KEEP_DAYS:-}" ]; then + # Control how much LFS history `git lfs prune` keeps on disk. Set to `0` for HEAD only. + log "Keeping ${LFS_KEEP_DAYS} extra day(s) of LFS history (HEAD is always kept)." + # recent refs: any branch/tags whose last commit is within these days. + git config --global lfs.fetchrecentrefsdays "${LFS_KEEP_DAYS}" + # recent commits: walks these ref tips to look for commits within these days. + git config --global lfs.fetchrecentcommitsdays "${LFS_KEEP_DAYS}" + # Extra safety margin added on top of above two. Set to 0 for above settings to be exact. + git config --global lfs.pruneoffsetdays 0 + fi + # Check if latest symlink exists. if [ ! -L "$repo_path/latest" ]; then LOG_PREFIX="init" @@ -125,7 +147,6 @@ cmd_gitupdate() { log "Fetch completed in $(fmt_duration $(($(date +%s) - start_total)))." } - git_fetch_lfs() { local repo_path="$1" @@ -149,7 +170,7 @@ git_fetch_lfs() { log "Local HEAD: $local_head" log "Remote HEAD: $origin_head" if [ "$local_head" = "$origin_head" ]; then - log "No LFS updates..." + log "No update found." return 0 fi