Merge pull request #459 from buildkite-plugins/SUP-2585

SUP-2585: Docker Cache Support

Author: tomowatt

README.md

@@ -174,6 +174,12 @@ A list of images to attempt pulling before building in the format `service:CACHE
 
 They will be mapped directly to `cache-from` elements in the build according to the spec so any valid format there should be allowed.
 
+#### `cache-to` (build only, string or array)
+
+A list of export locations to be used to share build cache with future builds in the format `service:CACHE-SPEC` to allow for layer re-use. Unsupported caches are ignored and do not prevent building images.
+
+They will be mapped directly to `cache-to` elements in the build according to the spec so any valid format there should be allowed.
+
 #### `target` (build only)
 
 Allow for intermediate builds as if building with docker's `--target VALUE` options.
@@ -343,7 +349,7 @@ The default is `false`.
 To run the tests:
 
 ```bash
-docker-compose run --rm tests bats tests tests/v1
+docker compose run --rm tests bats tests tests/v1
 ```
 
 ## License

commands/build.sh

@@ -35,8 +35,24 @@ get_caches_for_service() {
   fi
 }
 
+get_caches_to_service() {
+  local service="$1"
+
+  # Read any cache-to parameters provided
+  for line in $(plugin_read_list CACHE_TO) ; do
+    IFS=':' read -r -a tokens <<< "$line"
+    service_name=${tokens[0]}
+    service_image=$(IFS=':'; echo "${tokens[*]:1}")
+
+    if [ "${service_name}" == "${service}" ]; then
+      echo "$service_image"
+    fi
+  done
+}
+
+
 # Run through all images in the build property, either a single item or a list
-# and build up a list of service name, image name and optional cache-froms to
+# and build up a list of service name, image name and optional cache-froms and cache-tos to
 # write into a docker-compose override file
 for service_name in $(plugin_read_list BUILD) ; do
   target="$(plugin_read_config TARGET "")"
@@ -47,19 +63,29 @@ for service_name in $(plugin_read_list BUILD) ; do
     cache_from+=("$cache_line")
   done
 
+  cache_to=()
+  for cache_line in $(get_caches_to_service "$service_name"); do
+    cache_to+=("$cache_line")
+  done
+
   labels=()
   while read -r label ; do
     [[ -n "${label:-}" ]] && labels+=("${label}")
   done <<< "$(plugin_read_list BUILD_LABELS)"
 
-  if [[ -n "${target}" ]] || [[ "${#labels[@]}" -gt 0 ]] || [[ "${#cache_from[@]}" -gt 0 ]]; then
+  if [[ -n "${target}" ]] || [[ "${#labels[@]}" -gt 0 ]] || [[ "${#cache_to[@]}" -gt 0 ]] || [[ "${#cache_from[@]}" -gt 0 ]]; then
     build_images+=("$service_name" "${image_name}" "${target}")
-    
+
     build_images+=("${#cache_from[@]}")
     if [[ "${#cache_from[@]}" -gt 0 ]]; then
       build_images+=("${cache_from[@]}")
     fi
-    
+
+    build_images+=("${#cache_to[@]}")
+    if [[ "${#cache_to[@]}" -gt 0 ]]; then
+      build_images+=("${cache_to[@]}")
+    fi
+
     build_images+=("${#labels[@]}")
     if [[ "${#labels[@]}" -gt 0 ]]; then
       build_images+=("${labels[@]}")

commands/run.sh

@@ -52,7 +52,7 @@ for service_name in "${prebuilt_candidates[@]}" ; do
   fi
 
   if [[ -n "$prebuilt_image" ]] ; then
-    prebuilt_service_overrides+=("$service_name" "$prebuilt_image" "" 0 0)
+    prebuilt_service_overrides+=("$service_name" "$prebuilt_image" "" 0 0 0)
     prebuilt_services+=("$service_name")
 
     # If it's prebuilt, we need to pull it down

lib/shared.bash

@@ -133,11 +133,15 @@ function build_image_override_file() {
     "$(docker_compose_config_version)" "$@"
 }
 
-# Checks that a specific version of docker-compose supports cache_from
-function docker_compose_supports_cache_from() {
+# Checks that a specific version of docker-compose supports cache_from and cache_to
+function docker_compose_supports_cache() {
   local version="$1"
   if [[ "$version" == 1* || "$version" =~ ^(2|3)(\.[01])?$ ]] ; then
-    return 1
+    echo "Unsupported Docker Compose config file version: $version"
+    echo "The 'cache_from' option can only be used with Compose file versions 2.2 or 3.2 and above."
+    echo "For more information on Docker Compose configuration file versions, see:"
+    echo "https://docs.docker.com/compose/compose-file/compose-versioning/#versioning"
+    exit 1
   fi
 }
 
@@ -176,6 +180,16 @@ function build_image_override_file_with_version() {
       done
     fi
 
+    # load cache_to array
+    cache_to_amt="${1:-0}"
+    [[ -n "${1:-}" ]] && shift; # remove the value if not empty
+    if [[ "${cache_to_amt}" -gt 0 ]]; then
+      cache_to=()
+      for _ in $(seq 1 "$cache_to_amt"); do
+        cache_to+=( "$1" ); shift
+      done
+    fi
+
     # load labels array
     labels_amt="${1:-0}"
     [[ -n "${1:-}" ]] && shift; # remove the value if not empty
@@ -186,7 +200,7 @@ function build_image_override_file_with_version() {
       done
     fi
 
-    if [[ -z "$image_name" ]] && [[ -z "$target" ]] && [[ "$cache_from_amt" -eq 0 ]] && [[ "$labels_amt" -eq 0 ]]; then
+    if [[ -z "$image_name" ]] && [[ -z "$target" ]] && [[ "$cache_from_amt" -eq 0 ]] && [[ "$cache_to_amt" -eq 0 ]] && [[ "$labels_amt" -eq 0 ]]; then
       # should not print out an empty service
       continue
     fi
@@ -197,7 +211,7 @@ function build_image_override_file_with_version() {
       printf "    image: %s\\n" "$image_name"
     fi
 
-    if [[ "$cache_from_amt" -gt 0 ]] || [[ -n "$target" ]] || [[ "$labels_amt" -gt 0 ]]; then
+    if [[ "$cache_from_amt" -gt 0 ]] || [[ "$cache_to_amt" -gt 0 ]] || [[ -n "$target" ]] || [[ "$labels_amt" -gt 0 ]]; then
       printf "    build:\\n"
     fi
 
@@ -206,20 +220,23 @@ function build_image_override_file_with_version() {
     fi
 
     if [[ "$cache_from_amt" -gt 0 ]] ; then
-      if ! docker_compose_supports_cache_from "$version" ; then
-        echo "Unsupported Docker Compose config file version: $version"
-        echo "The 'cache_from' option can only be used with Compose file versions 2.2 or 3.2 and above."
-        echo "For more information on Docker Compose configuration file versions, see:"
-        echo "https://docs.docker.com/compose/compose-file/compose-versioning/#versioning"
-        exit 1
-      fi
+      docker_compose_supports_cache "$version"
 
       printf "      cache_from:\\n"
       for cache_from_i in "${cache_from[@]}"; do
         printf "        - %s\\n" "${cache_from_i}"
       done
     fi
 
+    if [[ "$cache_to_amt" -gt 0 ]] ; then
+      docker_compose_supports_cache "$version"
+
+      printf "      cache_to:\\n"
+      for cache_to_i in "${cache_to[@]}"; do
+        printf "        - %s\\n" "${cache_to_i}"
+      done
+    fi
+
     if [[ "$labels_amt" -gt 0 ]] ; then
       printf "      labels:\\n"
       for label in "${labels[@]}"; do

plugin.yml

@@ -9,10 +9,10 @@ configuration:
     run:
       type: string
     build:
-      type: [ string, array ]
+      type: [string, array]
       minimum: 1
     push:
-      type: [ string, array ]
+      type: [string, array]
       minimum: 1
 
     ansi:
@@ -35,6 +35,9 @@ configuration:
     cache-from:
       type: [ string, array ]
       minimum: 1
+    cache-to:
+      type: [ string, array ]
+      minimum: 1
     cli-version:
       oneOf:
         - type: string
@@ -150,6 +153,7 @@ configuration:
     buildkit: [ build ]
     buildkit-inline-cache: [ build ]
     cache-from: [ build ]
+    cache-to: [ build ]
     command: [ run ]
     dependencies: [ run ]
     entrypoint: [ run ]

tests/docker-compose-config.bats

@@ -81,34 +81,34 @@ load '../lib/shared'
   assert_output "2.0"
 }
 
-@test "Whether docker-compose supports cache_from directive" {
-  run docker_compose_supports_cache_from ""
+@test "Whether docker-compose supports cache_from and cache_to directive" {
+  run docker_compose_supports_cache ""
   assert_success
 
-  run docker_compose_supports_cache_from "1.0"
+  run docker_compose_supports_cache "1.0"
   assert_failure
 
-  run docker_compose_supports_cache_from "2"
+  run docker_compose_supports_cache "2"
   assert_failure
 
-  run docker_compose_supports_cache_from "2.1"
+  run docker_compose_supports_cache "2.1"
   assert_failure
 
-  run docker_compose_supports_cache_from "2.2"
+  run docker_compose_supports_cache "2.2"
   assert_success
 
-  run docker_compose_supports_cache_from "2.3"
+  run docker_compose_supports_cache "2.3"
   assert_success
 
-  run docker_compose_supports_cache_from "3"
+  run docker_compose_supports_cache "3"
   assert_failure
 
-  run docker_compose_supports_cache_from "3.1"
+  run docker_compose_supports_cache "3.1"
   assert_failure
 
-  run docker_compose_supports_cache_from "3.2"
+  run docker_compose_supports_cache "3.2"
   assert_success
 
-  run docker_compose_supports_cache_from "3.3"
+  run docker_compose_supports_cache "3.3"
   assert_success
 }