From 660c368a415eb33d6dc1270e5f7f41f4932d377a Mon Sep 17 00:00:00 2001 From: Tomoki Kobayashi Date: Sat, 26 Sep 2026 18:27:04 +0900 Subject: [PATCH] Add a rig that makes the website's teaser video MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The landing page's video was an iPad screen recording made by hand. This makes its successor from a script instead, so it can be shot again whenever the app changes: `ruby Tools/teaser/teaser.rb` records a UI test on the 11-inch iPad simulator and cuts it into a silent two-minute 1080p film, ready for music. TeaserTests is the script: one line, a corner, a square the long way and then with a repeat, 120° and 144° until a star turns up, colour, and a box that grows it into a spiral, ending on the generated Swift. It presses by coordinates and logs every press, drag, caption and camera move with its time. teaser.rb is the editor. It cuts the still stretches the simulator leaves between frames, zooms toward the part of the screen that matters, draws the touches the simulator does not show, lays the captions on, and puts the icon on either end. What it took to make that reliable is in Tools/teaser/README.md: - Coordinate taps do not scroll, so the palette is scrolled by a finger. - Number fields need a hardware keyboard. - Caret blinks would otherwise count as motion. - "Finished drawing" cannot mean "the scrubber moved". - A drop that does not land is retried and cut out of the film. --- .claude/skills/screenshots/SKILL.md | 19 +- CLAUDE.md | 4 + Tools/teaser/README.md | 107 +++++ Tools/teaser/teaser.rb | 544 ++++++++++++++++++++++++ Tools/teaser/text.swift | 46 ++ TortoiseBlocksUITests/TeaserTests.swift | 529 +++++++++++++++++++++++ 6 files changed, 1247 insertions(+), 2 deletions(-) create mode 100644 Tools/teaser/README.md create mode 100644 Tools/teaser/teaser.rb create mode 100644 Tools/teaser/text.swift create mode 100644 TortoiseBlocksUITests/TeaserTests.swift diff --git a/.claude/skills/screenshots/SKILL.md b/.claude/skills/screenshots/SKILL.md index 2aaaacb..b058651 100644 --- a/.claude/skills/screenshots/SKILL.md +++ b/.claude/skills/screenshots/SKILL.md @@ -7,8 +7,9 @@ description: >- (Tools/ipad-shots.rb, Tools/macos-shots.rb, Tools/visionos-shots.rb), the flatten-and-optimise pass (Tools/screenshots.rb), what a sendable capture has to be, and the traps that - make a screenshot tool fail silently. Load this before reshooting, before - adding a shot or a platform, and whenever a capture looks wrong. + make a screenshot tool fail silently. Also the website's teaser video + (Tools/teaser/). Load this before reshooting, before adding a shot or a + platform, and whenever a capture looks wrong. --- # Screenshots @@ -258,6 +259,20 @@ Seven ways the Mac differs from the iPad, all handled but all worth knowing: what makes 1280×800pt — 2560×1600px — reproducible. Keep that `defaultSize`: a capture at any other size would need cropping or resampling. +## The teaser video + +The website's two-minute video is made the same way: `ruby Tools/teaser/teaser.rb` +records `TortoiseBlocksUITests/TeaserTests.swift` on the 11-inch iPad simulator +and cuts it into a silent 1080p film; music goes on by hand afterwards. The +test is the script, and it logs every press, caption and camera move for the +editor to work from. `Tools/teaser/README.md` has why each piece is as it is. +It shares DerivedData with the rigs above, so it counts as one of them: one at +a time. + +Judge it the way the stills are judged, but on the film: a contact sheet +shows what is in it (`ffmpeg -i teaser.mp4 -vf fps=1/3,scale=320:-1,tile=6x9`), +and only playing it shows whether it moves well. + ## Judging the result Look at the pictures. `metadata_check` proves a capture is *sendable*, not that diff --git a/CLAUDE.md b/CLAUDE.md index 7068b2e..8c80a96 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -38,6 +38,10 @@ xcrun simctl launch space.hiraku.tortoiseblocks \ xcrun simctl io screenshot shot.png # 3840x2160, with an alpha channel xcrun simctl io recordVideo walk.mov # Ctrl-C to stop +# The website's teaser video: records TeaserTests on the iPad simulator, then +# cuts it to a silent 1080p film (Tools/teaser/README.md). ~15 minutes. +ruby Tools/teaser/teaser.rb # --compose re-cuts the last recording + # The App Store listing (appstore/). The check needs no key and no bundle; # the other two need ASC_ISSUER_ID / ASC_KEY_ID / ASC_PRIVATE_KEY_PATH. ruby Tools/screenshots.rb # after ANY reshoot: strip alpha, optimise, rebuild site/shots and docs/ diff --git a/Tools/teaser/README.md b/Tools/teaser/README.md new file mode 100644 index 0000000..84b372b --- /dev/null +++ b/Tools/teaser/README.md @@ -0,0 +1,107 @@ +# The teaser video + +The website's video — a program built by trial and error, from one line to a +spiral of stars — is made here, not filmed by hand. + +```bash +ruby Tools/teaser/teaser.rb # record on the iPad simulator, then compose +ruby Tools/teaser/teaser.rb --compose # compose the last recording again +``` + +The film comes out silent, 1920×1080 at 30fps, about two minutes, in the +work directory the script prints (`$TMPDIR/tortoise-teaser/teaser.mp4`), with +the raw recording beside it. Music is added afterwards, by hand. + +Three pieces: + +- **`TortoiseBlocksUITests/TeaserTests.swift` is the script.** Every press, + when each caption comes up, and where the camera looks, in the order they + happen. Changing the story is editing this file. The test writes a log of + what it did and when (`events.jsonl`); it does not decide anything about the + film. +- **`teaser.rb` is the crew and the editor.** It records the simulator while + the test runs, then cuts, frames, zooms, captions and draws the touches from + the log. +- **`text.swift`** renders captions and titles in SF Pro Rounded, because + ImageMagick cannot ask the system's variable font for a weight. + +It needs Xcode, `ffmpeg` and ImageMagick (`magick`). A full run takes about a +quarter of an hour, nearly all of it the recording; composing again takes two +minutes. Don't run it while a screenshot rig is +running: they share DerivedData (see the `screenshots` skill). + +## Things that look like mistakes and are not + +**The shoot is on the 11-inch iPad**, not the 13-inch one the App Store +captures use. It has the same layout in fewer points, so every block is about +a quarter larger in a 1080p frame. At 13 inches the palette's text came out a +few pixels tall. + +**Most of the recording is thrown away.** A UI test is slow: every press +waits for the app to go idle, finding an element takes a snapshot, and a +typed digit takes seconds. The raw run is almost four minutes. The simulator +writes a frame only when the screen changes, so the gaps between frames are +exactly the still stretches. Each one is cut down to `HEAD` seconds, except +where the log asks for time: a caption to read, a camera move to finish, the +moment around a touch, and `linger` (a finished drawing, the code at the end). +The test's own `pause`s therefore don't set the pace. They only give the app +time to settle. Two refinements: while a number is being typed the field's +caret blinks, and every blink is a frame, so the test logs the typing span and +only the keys inside it are kept. And a drawing longer than `LONG_DRAWING` +plays at `FAST`×: the spiral takes nine seconds at the app's own tempo. + +**A drop that did not land is cut out.** A drag in the simulator now and then +parts the rows and inserts nothing. The test checks that the block arrived, +tries again if not, and logs the failed take as a `cut`, which the film leaves +out. + +**Touches are drawn afterwards.** The simulator's recording shows no finger, +and the log already knows where each press went. A tap is logged just after it +returns, and the screen reacts 0.1–0.26s before that (measured against the +first changed frame), so the finger is drawn landing `TAP_LEAD` earlier. A drag +is worked back from when it returned: its hold, its travel at the velocity it +was given, and its linger. + +**The camera is part of the log, not the edit.** `camera(.blocks)` in the test +records a rectangle in screen points. The film eases to it, zooming so that +the rectangle fills the frame. The zoom is `zoompan` on the full-resolution +source, so the text stays sharp at 2×. + +**Presses are coordinates, so nothing scrolls for them.** `XCUIElement.tap()` +scrolls its element into view first; a coordinate tap below the fold presses +whatever is there. On the 11-inch iPad that includes Repeat, and the first run +put a Start Fill where the repeat should have been. `palette(_:)` scrolls the +palette with a finger until the entry is on screen. Coordinates are used anyway +because the log needs the point that was pressed. + +**The palette entry is the leftmost button with that label.** "Forward" is +also the transport's step button. It is a separate element with the same name, +and a query that walks its matches one by one lost one between counting and +fetching. + +**A hardware keyboard is attached for the run**, by switching the Simulator's +`ConnectHardwareKeyboard` preference on and restarting the device to pick it +up. Xcode 27 has no Simulator.app to attach one by hand. The +preferences are exported first and imported back when the run ends. Without +it, a number field raises the full on-screen keyboard over half the screen. The maintainer's own recording +used a pointer and a keyboard, so none shows there either. The value is typed +after a tap in the middle of the field, which puts the caret at the end, so +the old digits go by backspace. `⌘A` is not used: selecting brought iPadOS +27's small number keypad up over the field, and sometimes the full keyboard +under it. + +**"Drawing finished" means the scrubber settled after it moved or had time +to.** It does not require a new end value: changing an angle leaves the step +count alone, so the run ends on exactly the value the previous one did, and +waiting for a different value waited forever. It does not require seeing the +scrubber move either: a press returns only once the app is idle, which once +took nine seconds, and an eight-step triangle is drawn in 0.8. + +**The recording is sideways and variable-rate.** `simctl io recordVideo` +writes the framebuffer as it is held (portrait, the landscape app on its side) +and only when something changes. `transpose=2` stands it up, and `fps=30` +fills the still stretches before anything is cut. + +**The system language is English for the run** as well as the app, because +the status bar writes its date in the system's language. It is put back +afterwards, as `ipad-shots.rb` does. diff --git a/Tools/teaser/teaser.rb b/Tools/teaser/teaser.rb new file mode 100644 index 0000000..c431a28 --- /dev/null +++ b/Tools/teaser/teaser.rb @@ -0,0 +1,544 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Makes the teaser video for the website: a child's-eye walk from one line to +# a spiral of stars, shot on the iPad simulator and cut to 1920×1080. +# +# ruby Tools/teaser/teaser.rb # record, then compose +# ruby Tools/teaser/teaser.rb --compose # compose the last recording again +# +# Silent on purpose: the music is laid on afterwards, by hand. The film lands +# in the work directory printed at the end, with the raw recording beside it. +# +# `TortoiseBlocksUITests/TeaserTests.swift` is the script — every press, when +# each caption comes up and where the camera looks. This is the camera crew +# and the editor. The README beside this file says why each piece is the way +# it is; the short version of the four things that are not tidiness: +# +# - **The recording is mostly waiting**, and the cut is what removes it. The +# simulator writes a frame only when the screen changes, so the gaps between +# frames *are* the still stretches; each is cut down to HEAD seconds, except +# where the script asked for time (a caption to read, a camera move, a +# finished drawing). +# - **The simulator records the framebuffer as it is held**: portrait, with the +# landscape app on its side. `transpose=2` stands it up. +# - **Typing needs a hardware keyboard**, or a number field brings up half a +# screen of keys. It is the Simulator's preference, so it is switched on for +# the run, the device restarted to pick it up, and the preferences put back. +# - **Touches are drawn afterwards**, from the script's own log. The simulator +# shows none, and the log knows exactly where each press went. + +require "fileutils" +require "json" +require "open3" +require "pathname" +require "tmpdir" + +ROOT = Pathname.new(__dir__).parent.parent +WORK = Pathname.new(Dir.tmpdir) / "tortoise-teaser" +RAW = WORK / "raw" +BUNDLE_ID = "space.hiraku.tortoiseblocks" +# The 11-inch rather than the 13-inch the App Store captures use: the same +# layout in fewer points, so every block is a quarter larger in the frame. +DEVICE_NAME = "iPad Pro 11-inch (M5)" + +W = 1920 +H = 1080 +FPS = 30 +# The current video's backdrop, left to right. +GREEN = "#39FE92" +CYAN = "#42D2F7" +INK = "1c1c1e" + +# The iPad's screen in the frame: clear of the top and bottom by a little. +MARGIN = 40 +RADIUS = 26 + +# Time, in seconds. A still screen is cut down to HEAD, except around these. +HEAD = 0.22 +TAP_BEFORE = 0.35 +TAP_AFTER = 0.25 +# The press lands this long before the test logs it: measured against the +# first changed frame, 0.1–0.26s. +TAP_LEAD = 0.22 +# A typed key has landed by the time it is logged. +KEY_BEFORE = 0.35 +KEY_AFTER = 0.15 +CAPTION_HOLD = 1.0 +CAMERA_HOLD = 0.8 +CAMERA_EASE = 0.7 +# A drawing longer than this plays at FAST× — the spiral takes nine seconds. +LONG_DRAWING = 3.0 +FAST = 2 +DRAG_AFTER = 0.3 +TITLE = 3.5 +ENDING = 5.0 +FADE = 0.6 + +def run(*command) + system(*command) || abort("failed: #{command.join(' ')[0, 400]}") +end + +def simctl(*arguments) + IO.popen(["xcrun", "simctl", *arguments], err: %i[child out], &:read) +end + +# `a+b+c+…` as a balanced tree of sums. ffmpeg's expression parser recurses +# once per `+`, and a flat sum of the ~120 pieces a cut has fails to parse +# ("Cannot allocate memory"); balanced, it is seven deep. +def sum(terms) + return terms.first if terms.size == 1 + + half = terms.size / 2 + "(#{sum(terms[0, half])}+#{sum(terms[half..-1])})" +end + +def dims(file) + `magick identify -format '%w %h' '#{file}'`.split.map(&:to_i) +end + +# --------------------------------------------------------------------------- +# Recording. + +# The newest iOS runtime that has the device — 27.0 or later, the same floor +# as the screenshot rig: 26.5 does not open a document handed to it by URL. +def device + json = JSON.parse(simctl("list", "devices", "available", "-j")) + candidates = json["devices"].flat_map do |runtime, list| + runtime.include?("iOS") ? list.select { |d| d["name"] == DEVICE_NAME }.map { |d| [runtime, d] } : [] + end + abort("No #{DEVICE_NAME} simulator.") if candidates.empty? + runtime, chosen = candidates.max_by { |r, _| r } + version = runtime[/iOS-(\d+)-(\d+)/, 0].to_s.sub("iOS-", "").tr("-", ".") + abort("#{DEVICE_NAME} is only on iOS #{version}; 27.0 or newer is needed.") if version < "27" + [chosen["udid"], chosen["state"]] +end + +def restart(udid) + simctl("shutdown", udid) + simctl("boot", udid) + simctl("bootstatus", udid) +end + +def record + udid, state = device + puts "device #{udid}" + if state != "Booted" + simctl("boot", udid) + simctl("bootstatus", udid) + end + + # A hardware keyboard, so a number field raises no on-screen keyboard. It is + # the Simulator's preference — Xcode 27 has no Simulator.app to attach one + # from — so it is written first, the device restarted below to pick it up, + # and the preferences put back afterwards. + preferences = WORK / "simulator-preferences.plist" + run("defaults", "export", "com.apple.iphonesimulator", preferences.to_s) + at_exit { system("defaults", "import", "com.apple.iphonesimulator", preferences.to_s) } + run("defaults", "write", "com.apple.iphonesimulator", "ConnectHardwareKeyboard", "-bool", "true") + + # English, said to the system as well as the app: the status bar writes the + # date in the system's language. Whatever it was goes back afterwards. + languages = simctl("spawn", udid, "defaults", "read", "-g", "AppleLanguages").scan(/[A-Za-z]{2,3}(?:-[A-Za-z0-9]+)*/) + locale = simctl("spawn", udid, "defaults", "read", "-g", "AppleLocale").strip + unless languages.first == "en-US" && locale == "en_US" + at_exit do + simctl("spawn", udid, "defaults", "write", "-g", "AppleLanguages", "-array", *languages) + simctl("spawn", udid, "defaults", "write", "-g", "AppleLocale", locale) + end + simctl("spawn", udid, "defaults", "write", "-g", "AppleLanguages", "-array", "en-US") + simctl("spawn", udid, "defaults", "write", "-g", "AppleLocale", "en_US") + end + restart(udid) + simctl("status_bar", udid, "override", + "--time", "9:41", "--batteryState", "charged", "--batteryLevel", "100", + "--wifiMode", "active", "--wifiBars", "3", "--cellularMode", "notSupported") + simctl("ui", udid, "appearance", "light") + + # An empty document to build in, where the test can hand it to the app: the + # device's tmp survives the reinstall that preparing a test run does. + seed = Pathname.new(Dir.home) / "Library/Developer/CoreSimulator/Devices" / udid / "data/tmp/tbteaser" + FileUtils.rm_rf(seed) + FileUtils.mkdir_p(seed) + (seed / "My Drawing.tortoise").write( + JSON.pretty_generate({ "blocks" => [], "schemaVersion" => 1, "title" => "My Drawing" }) + ) + events = seed / "events.jsonl" + simctl("uninstall", udid, BUNDLE_ID) + + common = [ + "-project", (ROOT / "TortoiseBlocks.xcodeproj").to_s, + "-scheme", "TortoiseBlocks", + "-destination", "id=#{udid}", + "-only-testing:TortoiseBlocksUITests/TeaserTests", + "-parallel-testing-enabled", "NO", + # A failed UI test otherwise spends ten minutes in `simctl diagnose`. + "-collect-test-diagnostics", "never" + ] + # Built first, so the recording is not four minutes of a compiler. + run("xcodebuild", "build-for-testing", *common, "-quiet") + + FileUtils.rm_rf(RAW) + FileUtils.mkdir_p(RAW) + movie = RAW / "raw.mov" + _, output, recorder = Open3.popen2e("xcrun", "simctl", "io", udid, "recordVideo", "--codec=h264", "--force", movie.to_s) + started = nil + while (line = output.gets) + next unless line.include?("Recording started") + + # The recording's own clock starts here, and the test's log is in wall + # time, so this is what lines the two up. + started = Time.now.to_f + break + end + abort("the recorder never started") unless started + + result = RAW / "teaser.xcresult" + environment = { "TEST_RUNNER_TB_DOCUMENTS" => seed.to_s, "TEST_RUNNER_TB_EVENTS" => events.to_s } + passed = system(environment, "xcodebuild", "test-without-building", *common, + "-resultBundlePath", result.to_s, "-quiet") + Process.kill("INT", recorder.pid) + recorder.value + simctl("status_bar", udid, "clear") + + FileUtils.cp(events, RAW / "events.jsonl") + failure = Pathname.new("#{events}.failure.txt") + FileUtils.cp(failure, RAW / "failure.txt") if failure.exist? + (RAW / "recording.json").write(JSON.generate({ "started" => started })) + abort("The script stopped part-way; #{RAW / 'failure.txt'} has the screen it was looking at.") unless passed +end + +# --------------------------------------------------------------------------- +# Composing. + +def compose + text_tool = WORK / "text" + run("xcrun", "swiftc", "-O", (Pathname.new(__dir__) / "text.swift").to_s, "-o", text_tool.to_s) + text = lambda do |file, string, size, weight, colour| + run(text_tool.to_s, file.to_s, string, size.to_s, weight, colour) + end + layers = WORK / "layers" + FileUtils.mkdir_p(layers) + + events = RAW.join("events.jsonl").readlines.map { |line| JSON.parse(line) } + started = JSON.parse(RAW.join("recording.json").read)["started"] + at = ->(event) { event["t"] - started } + screen = events.find { |e| e["type"] == "screen" } + points = [screen["w"].to_f, screen["h"].to_f] + t_start = at.call(events.find { |e| e["type"] == "start" }) + t_end = at.call(events.find { |e| e["type"] == "end" }) + + # The source the right way up, at a constant rate. + cfr = WORK / "upright.mp4" + unless cfr.exist? && cfr.mtime > RAW.join("raw.mov").mtime + run("ffmpeg", "-v", "error", "-y", "-i", RAW.join("raw.mov").to_s, "-vf", "transpose=2,fps=#{FPS}", + "-c:v", "libx264", "-preset", "fast", "-crf", "10", "-pix_fmt", "yuv420p", cfr.to_s) + end + # While a number is being typed the caret blinks, and every blink is a + # frame; those stretches count as still, and only the keys are kept. + typing = events.select { |e| e["type"] == "typing" }.map { |e| [e["from"] - started, at.call(e)] } + changes = `ffprobe -v error -select_streams v -show_entries frame=pts_time -of csv=p=0 '#{RAW / 'raw.mov'}'` + .split.map { |s| s.delete(",").to_f }.select { |t| t > t_start && t < t_end } + .reject { |t| typing.any? { |a, b| t > a && t < b } } + + # A drag's phases, in recording time, worked back from when it returned. + gesture = lambda do |e| + distance = Math.hypot(e["x2"] - e["x"], e["y2"] - e["y"]) + release = at.call(e) - 0.05 + move_end = release - e["linger"] + move_start = move_end - distance / e["velocity"] + { press: move_start - e["hold"], move_start: move_start, move_end: move_end, release: release } + end + + # What stays: the first HEAD of every still stretch, all of every moving + # one, and whatever each event needs around it. + windows = [] + ([t_start] + changes + [t_end]).each_cons(2) { |x, y| windows << [x, [y, x + HEAD].min] } + events.each do |e| + t = at.call(e) + case e["type"] + when "caption" then windows << [t, t + CAPTION_HOLD] + when "camera" then windows << [t, t + CAMERA_HOLD] + when "tap" then windows << [t - TAP_BEFORE, t + TAP_AFTER] + when "key" then windows << [t - KEY_BEFORE, t + KEY_AFTER] + when "drag" + g = gesture.call(e) + windows << [g[:press] - 0.15, g[:release] + DRAG_AFTER] + when "linger" then windows << [t, t + e["seconds"]] + end + end + windows = windows.map { |a, b| [a.clamp(t_start, t_end), b.clamp(t_start, t_end)] }.reject { |a, b| b <= a }.sort + kept = [] + windows.each do |a, b| + if kept.any? && a <= kept.last[1] + 1.5 / FPS + kept.last[1] = [kept.last[1], b].max + else + kept << [a, b] + end + end + # Takes the script threw away — a drop that did not land, tried again. + events.select { |e| e["type"] == "cut" }.each do |e| + a = e["from"] - started + b = at.call(e) + kept = kept.flat_map do |x, y| + next [[x, y]] if y <= a || x >= b + + [[x, [y, a].min], [[x, b].max, y]].reject { |p, q| q - p < 1e-3 } + end + end + # Long drawings, from the press of play to the moment they finished (the + # linger `playDrawing` logs), are sped up; each piece carries its speed. + drawings = [] + events.each_with_index do |e, i| + next unless e["type"] == "linger" + + press = events[0...i].reverse.find { |p| p["type"] == "tap" } + next unless press && at.call(e) - at.call(press) > LONG_DRAWING + + drawings << [at.call(press) + TAP_AFTER, at.call(e)] + end + kept = kept.flat_map do |a, b| + cuts = drawings.flatten.select { |t| t > a && t < b }.sort + ([a] + cuts + [b]).each_cons(2).map do |x, y| + fast = drawings.any? { |p, q| x >= p && y <= q } + [x, y, fast ? FAST : 1] + end + end + offsets = [] + total = 0.0 + kept.each do |a, b, speed| + offsets << total + total += (b - a) / speed + end + # Recording time → film time. A moment that was cut lands where the cut is. + map = lambda do |t| + kept.each_with_index do |(a, b, speed), i| + return offsets[i] if t < a + return offsets[i] + (t - a) / speed if t <= b + end + total + end + footage = (total * FPS).floor / FPS.to_f + puts format("kept %.1fs of %.1fs, in %d pieces", total, t_end - t_start, kept.size) + + # The screen in the frame, and the camera over it. + screen_h = H - 2 * MARGIN + screen_w = (screen_h * points[0] / points[1]).round + screen_w += 1 if screen_w.odd? + screen_x = (W - screen_w) / 2 + screen_y = MARGIN + scale = screen_h / points[1] + + cues = events.select { |e| e["type"] == "camera" }.map do |e| + x, y, w, h = e["rect"] + zoom = [[points[0] / w, points[1] / h].min, 1.0].max + { at: map.call(at.call(e)), to: [x + w / 2.0, y + h / 2.0, zoom] } + end + cues = [{ at: -10.0, to: [points[0] / 2, points[1] / 2, 1.0] }] + cues + smooth = ->(u) { u = u.clamp(0.0, 1.0); u * u * (3 - 2 * u) } + camera_at = lambda do |t| + value = cues.first[:to].dup + cues.each_cons(2) do |a, b| + k = smooth.call((t - b[:at]) / CAMERA_EASE) + 3.times { |i| value[i] += (b[:to][i] - a[:to][i]) * k } + end + value + end + # The same curve, for zoompan, as a function of the film's time. + camera_expression = lambda do |i| + terms = ["(#{cues.first[:to][i].round(4)})"] + cues.each_cons(2) do |a, b| + delta = b[:to][i] - a[:to][i] + next if delta.abs < 1e-6 + + u = "clip((it-#{b[:at].round(3)})/#{CAMERA_EASE},0,1)" + terms << "(#{delta.round(5)})*#{u}*#{u}*(3-2*#{u})" + end + sum(terms) + end + to_frame = lambda do |x, y, t| + cx, cy, z = camera_at.call(t) + left = (cx - points[0] / z / 2).clamp(0, points[0] - points[0] / z) + top = (cy - points[1] / z / 2).clamp(0, points[1] - points[1] / z) + [screen_x + (x - left) * z * scale, screen_y + (y - top) * z * scale] + end + + # Stills: the backdrop, the screen's corners and shadow, the finger. + background = layers / "background.png" + run("magick", "-size", "#{H}x#{W}", "gradient:#{GREEN}-#{CYAN}", "-rotate", "-90", background.to_s) + mask = layers / "mask.png" + run("magick", "-size", "#{screen_w}x#{screen_h}", "xc:black", "-fill", "white", + "-draw", "roundrectangle 0,0 #{screen_w - 1},#{screen_h - 1} #{RADIUS},#{RADIUS}", mask.to_s) + shadow = layers / "shadow.png" + run("magick", "-size", "#{W}x#{H}", "xc:none", "-fill", "rgba(0,70,50,0.30)", + "-draw", "roundrectangle #{screen_x},#{screen_y + 12} #{screen_x + screen_w},#{screen_y + screen_h + 12} #{RADIUS},#{RADIUS}", + "-blur", "0x20", shadow.to_s) + + finger = 120 + c = finger / 2 + frames = layers / "finger" + FileUtils.rm_rf(frames) + FileUtils.mkdir_p(frames) + # A tap: the finger lands, then a ring spreads as it lifts. PNG32 on every + # frame, the empty last one included — written as greyscale it has no alpha + # and overlays as a black square. + 12.times do |i| + p = i / 11.0 + dot_alpha = i < 7 ? 0.6 : 0.6 * (1 - (i - 6) / 5.0) + ring_r = 24 + 30 * p + ring_alpha = 0.9 * (1 - p) + dot_r = 24 - 4 * [p * 2, 1].min + run("magick", "-size", "#{finger}x#{finger}", "xc:none", + "-fill", "rgba(255,255,255,#{dot_alpha.round(3)})", + "-stroke", "rgba(60,60,67,#{(dot_alpha * 0.8).round(3)})", "-strokewidth", "2", + "-draw", "circle #{c},#{c} #{c + dot_r},#{c}", + "-fill", "none", "-stroke", "rgba(255,255,255,#{ring_alpha.round(3)})", "-strokewidth", "4", + "-draw", "circle #{c},#{c} #{c + ring_r},#{c}", "PNG32:#{frames / format('%02d.png', i)}") + end + run("magick", "-size", "#{finger}x#{finger}", "xc:none", "PNG32:#{frames / '12.png'}") + tap_movie = layers / "tap.mov" + run("ffmpeg", "-v", "error", "-y", "-framerate", FPS.to_s, "-i", (frames / "%02d.png").to_s, + "-c:v", "png", tap_movie.to_s) + held = layers / "held.png" + run("magick", "-size", "#{finger}x#{finger}", "xc:none", + "-fill", "rgba(255,255,255,0.6)", "-stroke", "rgba(60,60,67,0.5)", "-strokewidth", "2", + "-draw", "circle #{c},#{c} #{c + 24},#{c}", "PNG32:#{held}") + + # Captions: dark text on a white pill, low over the screen, each up until + # the next one. + captions = events.select { |e| e["type"] == "caption" } + caption_layers = captions.each_with_index.map do |caption, i| + words = layers / "caption-text-#{i}.png" + text.call(words, caption["text"], 46, "bold", INK) + tw, th = dims(words) + pill_w = tw + 24 + pill_h = th - 4 + pad = 24 + file = layers / "caption-#{i}.png" + run("magick", "-size", "#{pill_w + 2 * pad}x#{pill_h + 2 * pad}", "xc:none", + "-fill", "rgba(0,0,0,0.22)", + "-draw", "roundrectangle #{pad},#{pad + 4} #{pad + pill_w - 1},#{pad + pill_h + 3} #{pill_h / 2},#{pill_h / 2}", + "-blur", "0x9", + "-fill", "rgba(255,255,255,0.96)", + "-draw", "roundrectangle #{pad},#{pad} #{pad + pill_w - 1},#{pad + pill_h - 1} #{pill_h / 2},#{pill_h / 2}", + words.to_s, "-gravity", "center", "-geometry", "+0-2", "-composite", "PNG32:#{file}") + from = map.call(at.call(caption)) + to = i + 1 < captions.size ? map.call(at.call(captions[i + 1])) : footage + puts format(" %5.1fs %4.1fs %s", TITLE - FADE + from, to - from, caption["text"]) + { file: file, from: from, to: to } + end + + taps = events.select { |e| e["type"] == "tap" }.map do |e| + moment = map.call(at.call(e) - TAP_LEAD) + x, y = to_frame.call(e["x"], e["y"], moment) + { at: moment, x: (x - c).round, y: (y - c).round } + end + drags = events.select { |e| e["type"] == "drag" }.map do |e| + g = gesture.call(e).transform_values { |t| map.call(t) } + x1, y1 = to_frame.call(e["x"], e["y"], g[:press]) + x2, y2 = to_frame.call(e["x2"], e["y2"], g[:release]) + g.merge(x1: x1 - c, y1: y1 - c, x2: x2 - c, y2: y2 - c) + end + + # The footage: cut, zoomed, framed, captioned, touched. + select = sum(kept.map do |a, b, speed| + piece = "between(t,#{a.round(4)},#{b.round(4)})" + speed == 1 ? piece : "#{piece}*not(mod(n,#{speed}))" + end) + zx = camera_expression.call(0) + zy = camera_expression.call(1) + zz = camera_expression.call(2) + inputs = ["-loop", "1", "-i", background.to_s, "-i", cfr.to_s, + "-loop", "1", "-i", mask.to_s, "-loop", "1", "-i", shadow.to_s] + filters = [] + filters << "[1:v]select='#{select}',setpts=N/(#{FPS}*TB)," \ + "zoompan=z='#{zz}':x='clip((#{zx})*iw/#{points[0]}-iw/zoom/2,0,iw-iw/zoom)':" \ + "y='clip((#{zy})*ih/#{points[1]}-ih/zoom/2,0,ih-ih/zoom)':d=1:s=#{screen_w}x#{screen_h}:fps=#{FPS}," \ + "format=rgba[scr0]" + filters << "[2:v]format=gray[mask]" + filters << "[scr0][mask]alphamerge[scr]" + filters << "[0:v][3:v]overlay=0:0[bg]" + filters << "[bg][scr]overlay=#{screen_x}:#{screen_y}:shortest=1[v0]" + last = "v0" + index = 4 + caption_layers.each_with_index do |layer, i| + inputs += ["-loop", "1", "-i", layer[:file].to_s] + w, h = dims(layer[:file]) + filters << "[#{index}:v]format=rgba,fade=t=in:st=#{layer[:from].round(3)}:d=0.25:alpha=1," \ + "fade=t=out:st=#{(layer[:to] - 0.25).round(3)}:d=0.25:alpha=1[c#{i}]" + filters << "[#{last}][c#{i}]overlay=#{(W - w) / 2}:#{screen_y + screen_h - h - 18}:shortest=1[vc#{i}]" + last = "vc#{i}" + index += 1 + end + taps.each_with_index do |tap, i| + inputs += ["-i", tap_movie.to_s] + filters << "[#{index}:v]format=rgba,setpts=PTS-STARTPTS+#{tap[:at].round(3)}/TB[t#{i}]" + filters << "[#{last}][t#{i}]overlay=#{tap[:x]}:#{tap[:y]}:eof_action=pass[vt#{i}]" + last = "vt#{i}" + index += 1 + end + drags.each_with_index do |d, i| + inputs += ["-loop", "1", "-i", held.to_s] + span = [d[:move_end] - d[:move_start], 0.01].max + u = "clip((t-#{d[:move_start].round(3)})/#{span.round(3)},0,1)" + x = "#{d[:x1].round(1)}+(#{(d[:x2] - d[:x1]).round(1)})*#{u}" + y = "#{d[:y1].round(1)}+(#{(d[:y2] - d[:y1]).round(1)})*#{u}" + filters << "[#{index}:v]format=rgba[h#{i}]" + filters << "[#{last}][h#{i}]overlay=x='#{x}':y='#{y}':eval=frame:shortest=1:" \ + "enable='between(t,#{d[:press].round(3)},#{d[:release].round(3)})'[vd#{i}]" + last = "vd#{i}" + index += 1 + # The lift, from the tap's own animation: its second half. + inputs += ["-i", tap_movie.to_s] + filters << "[#{index}:v]format=rgba,trim=start=#{(6.0 / FPS).round(3)}," \ + "setpts=PTS-STARTPTS+#{d[:release].round(3)}/TB[r#{i}]" + filters << "[#{last}][r#{i}]overlay=#{d[:x2].round}:#{d[:y2].round}:eof_action=pass[vr#{i}]" + last = "vr#{i}" + index += 1 + end + footage_file = WORK / "footage.mp4" + script = WORK / "filters.txt" + script.write(filters.join(";\n")) + run("ffmpeg", "-v", "error", "-y", *inputs, "-/filter_complex", script.to_s, "-map", "[#{last}]", + "-t", footage.round(3).to_s, "-r", FPS.to_s, + "-c:v", "libx264", "-preset", "medium", "-crf", "16", "-pix_fmt", "yuv420p", footage_file.to_s) + + # The cards: the icon and the name first; the icon, the name and where it + # runs at the end. + icon = ROOT / "docs" / "Icon.png" + name = layers / "name.png" + text.call(name, "Tortoise Blocks", 104, "heavy", INK) + devices = layers / "devices.png" + text.call(devices, "iPad · iPhone · Mac · Apple Vision Pro", 44, "semibold", "3a3a3c") + title = layers / "title.png" + run("magick", background.to_s, + "(", icon.to_s, "-resize", "340x340", ")", "-gravity", "north", "-geometry", "+0+250", "-composite", + name.to_s, "-gravity", "north", "-geometry", "+0+620", "-composite", title.to_s) + ending = layers / "ending.png" + run("magick", background.to_s, + "(", icon.to_s, "-resize", "300x300", ")", "-gravity", "north", "-geometry", "+0+215", "-composite", + name.to_s, "-gravity", "north", "-geometry", "+0+545", "-composite", + devices.to_s, "-gravity", "north", "-geometry", "+0+705", "-composite", ending.to_s) + + film = WORK / "teaser.mp4" + graph = [ + "[0:v]fps=#{FPS},format=yuv420p,settb=AVTB[a]", + "[1:v]fps=#{FPS},format=yuv420p,settb=AVTB[b]", + "[2:v]fps=#{FPS},format=yuv420p,settb=AVTB[c]", + "[a][b]xfade=transition=fade:duration=#{FADE}:offset=#{(TITLE - FADE).round(3)}[ab]", + "[ab][c]xfade=transition=fade:duration=#{FADE}:offset=#{(TITLE + footage - 2 * FADE).round(3)}[v]" + ].join(";") + run("ffmpeg", "-v", "error", "-y", + "-loop", "1", "-t", TITLE.to_s, "-i", title.to_s, + "-i", footage_file.to_s, + "-loop", "1", "-t", ENDING.to_s, "-i", ending.to_s, + "-filter_complex", graph, "-map", "[v]", + "-c:v", "libx264", "-preset", "slow", "-crf", "17", "-pix_fmt", "yuv420p", + "-movflags", "+faststart", film.to_s) + length = `ffprobe -v error -show_entries format=duration -of csv=p=0 '#{film}'`.to_f + puts format("%s — %.1fs, %d taps, %d drags, %d captions", film, length, taps.size, drags.size, captions.size) +end + +FileUtils.mkdir_p(WORK) +record unless ARGV.include?("--compose") +compose diff --git a/Tools/teaser/text.swift b/Tools/teaser/text.swift new file mode 100644 index 0000000..dce9cd1 --- /dev/null +++ b/Tools/teaser/text.swift @@ -0,0 +1,46 @@ +// Renders one line of text to a transparent PNG, in SF Pro Rounded — the +// face the app's own titles use. `teaser.rb` compiles and runs it. +// +// text +// +// A Swift script rather than ImageMagick's `-annotate`, because the system +// font is a variable font and only AppKit can ask it for a weight and the +// rounded design; ImageMagick draws whatever instance FreeType hands it. +import AppKit + +let arguments = CommandLine.arguments +guard arguments.count == 6, let size = Double(arguments[3]), let hex = Int(arguments[5], radix: 16) +else { + FileHandle.standardError.write( + Data("usage: text \n".utf8)) + exit(64) +} +let weight: NSFont.Weight = + switch arguments[4] { + case "heavy": .heavy + case "semibold": .semibold + default: .bold + } +let colour = NSColor( + srgbRed: CGFloat((hex >> 16) & 0xFF) / 255, green: CGFloat((hex >> 8) & 0xFF) / 255, + blue: CGFloat(hex & 0xFF) / 255, alpha: 1) + +let system = NSFont.systemFont(ofSize: size, weight: weight) +let font = + system.fontDescriptor.withDesign(.rounded).flatMap { NSFont(descriptor: $0, size: size) } + ?? system +let string = NSAttributedString( + string: arguments[2], attributes: [.font: font, .foregroundColor: colour]) +let bounds = string.boundingRect(with: .zero, options: [.usesLineFragmentOrigin]) +let pad = size * 0.4 + +let bitmap = NSBitmapImageRep( + bitmapDataPlanes: nil, pixelsWide: Int(ceil(bounds.width + pad * 2)), + pixelsHigh: Int(ceil(bounds.height + pad * 2)), bitsPerSample: 8, samplesPerPixel: 4, + hasAlpha: true, isPlanar: false, colorSpaceName: .deviceRGB, bytesPerRow: 0, bitsPerPixel: 0)! +NSGraphicsContext.saveGraphicsState() +NSGraphicsContext.current = NSGraphicsContext(bitmapImageRep: bitmap) +string.draw(at: NSPoint(x: pad, y: pad)) +NSGraphicsContext.restoreGraphicsState() +try bitmap.representation(using: .png, properties: [:])!.write( + to: URL(fileURLWithPath: arguments[1])) diff --git a/TortoiseBlocksUITests/TeaserTests.swift b/TortoiseBlocksUITests/TeaserTests.swift new file mode 100644 index 0000000..203ab9f --- /dev/null +++ b/TortoiseBlocksUITests/TeaserTests.swift @@ -0,0 +1,529 @@ +// iPad only: the teaser is shot on the iPad simulator, and orientation, +// `XCUIDevice.system` and the keyboard keys this leans on do not exist on a +// Mac, which builds this target too (for `ScreenshotTests`). +#if os(iOS) + + import XCTest + + /// Performs the teaser video's script on an iPad, slowly enough to watch. + /// + /// The driver records the simulator's screen while this runs and cuts the + /// recording into the video; this does the pressing, and writes down when and + /// where each press happened, when each caption belongs and where the camera + /// should look, so the driver can draw the touches, lay the captions over the + /// right moments and zoom in on the part of the screen that matters. + /// + /// Everything is addressed in points on the landscape screen, which is what + /// both `XCUIElement.frame` and the log use; the driver maps them onto the + /// recording. + final class TeaserTests: XCTestCase { + private var events: FileHandle? + private var app: XCUIApplication! + + override func setUp() { + continueAfterFailure = false + } + + @MainActor + func testTeaser() throws { + let documents = try environment("TB_DOCUMENTS") + let log = try environment("TB_EVENTS") + FileManager.default.createFile(atPath: log, contents: nil) + events = FileHandle(forWritingAtPath: log) + defer { try? events?.close() } + + app = XCUIApplication() + app.launchArguments = ["-AppleLanguages", "(en)", "-AppleLocale", "en_US"] + app.launch() + XCUIDevice.shared.orientation = .landscapeLeft + + let document = URL(fileURLWithPath: documents) + .appendingPathComponent("My Drawing.tortoise") + XCUIDevice.shared.system.open(document) + + let play = app.buttons["play.fill"] + XCTAssertTrue(play.waitForExistence(timeout: 30), "the document never opened") + record(["type": "screen", "w": app.frame.width, "h": app.frame.height]) + pause(2) + record(["type": "start"]) + + // A run that stops half-way leaves the screen as it last saw it + // beside the log, so it says what *was* there and not only what was + // missing. + do { + try firstSteps() + try repeating() + try findingTheStar() + try addingColour() + try growingTheLine() + try showingTheCode() + } + catch { + try? app.debugDescription.write( + toFile: log + ".failure.txt", atomically: true, encoding: .utf8) + throw error + } + + record(["type": "end"]) + pause(1) + } + + // MARK: - The script + + /// A line, then a corner: tap, play, change, play again. + @MainActor + private func firstSteps() throws { + camera(.all) + pause(1) + caption("Tap a block.") + camera(.blocks) + pause(1.4) + try tap(palette("Forward")) + pause(1.3) + + caption("Press play.") + camera(.all) + pause(1.2) + try playDrawing() + pause(0.8) + + caption("Try, look, change.") + camera(.blocks) + pause(1.3) + try tap(palette("Turn Right")) + pause(0.7) + try tap(palette("Forward")) + pause(0.9) + camera(.all) + pause(1) + try playDrawing() + pause(1.2) + } + + /// The square the long way, then the same square with a repeat. + @MainActor + private func repeating() throws { + caption("Again and again?") + camera(.blocks) + pause(1.3) + for title in ["Turn Right", "Forward", "Turn Right", "Forward"] { + try tap(palette(title)) + pause(0.55) + } + pause(0.4) + camera(.all) + pause(1) + try playDrawing() + pause(1.2) + + caption("Repeat it.") + pause(1) + try tap(button("Delete All")) + pause(0.9) + // The alert's button, not the can: the can is labelled the same. + try tap(frame(of: app.alerts.buttons["Delete All"])) + pause(0.9) + camera(.blocks) + pause(1) + // A new container becomes the palette's target, so the next two land + // inside it. + try tap(palette("Repeat")) + pause(0.8) + try tap(palette("Forward")) + pause(0.6) + try tap(palette("Turn Right")) + pause(0.9) + camera(.all) + pause(1) + try playDrawing() + pause(1.2) + } + + /// 90° makes a square; the star is found by trying other angles. + @MainActor + private func findingTheStar() throws { + caption("What if we turn 120°?") + camera(.program) + pause(1.3) + try setNumber("Number 90", to: "120") + camera(.all) + pause(1) + try playDrawing() + pause(1.5) + + caption("Not quite… 144!") + camera(.program) + pause(1.3) + try setNumber("Number 120", to: "144") + try setNumber("Number 4", to: "5") + camera(.all) + pause(1) + try playDrawing() + pause(1.5) + } + + /// A pen colour and a thicker line, dragged in above the repeat. + @MainActor + private func addingColour() throws { + caption("Add some color.") + camera(.blocks) + pause(1.3) + try drag("Pen Color", above: "Number 5", bringing: "Color blue") + pause(0.8) + try tap(button("Color blue")) + pause(0.8) + try tap(button("orange")) + pause(0.8) + try drag("Pen Width", above: "Number 5", bringing: "Number 2") + pause(0.8) + try setNumber("Number 2", to: "6", camera: nil) + camera(.all) + pause(1) + try playDrawing() + pause(1.5) + } + + /// A box that grows every time round turns the star into a spiral. + @MainActor + private func growingTheLine() throws { + caption("Let the line grow.") + camera(.blocks) + pause(1.3) + try drag("Put in Box", above: "Number 5", bringing: "Number 10") + pause(0.8) + // The forward inside the repeat: from a number to the box. + try tap(button("Number 100")) + pause(0.8) + try tap(button("Box")) + pause(0.9) + dismissPopover() + pause(0.6) + // Still the palette's target, so this goes inside the repeat too. + try tap(palette("Add to Box")) + pause(0.8) + try setNumber("Number 5", to: "30", camera: nil) + camera(.all) + pause(1) + try playDrawing() + pause(1.2) + camera(.canvas) + linger(2.5) + pause(2.5) + } + + /// The same program as Swift. + @MainActor + private func showingTheCode() throws { + caption("It's real Swift.") + camera(.all) + pause(1) + try tap(frame(of: app.segmentedControls.firstMatch.buttons.element(boundBy: 1))) + pause(0.8) + camera(.code) + linger(4) + pause(5) + } + + // MARK: - Where the camera looks + + /// Regions of the landscape screen, in points. The iPad this is shot on + /// is fixed by the driver, so these are measured rather than derived. + private enum Shot { + case all + /// The palette and the program. + case blocks + /// The program, closer. + case program + /// The drawing. + case canvas + /// The code pane's text. + case code + + func rect(on screen: CGRect) -> CGRect { + switch self { + case .all: screen + case .blocks: CGRect(x: 0, y: 40, width: 730, height: 503) + case .program: CGRect(x: 200, y: 110, width: 620, height: 427) + case .canvas: CGRect(x: 690, y: 140, width: 520, height: 358) + case .code: CGRect(x: 586, y: 90, width: 624, height: 430) + } + } + } + + @MainActor + private func camera(_ shot: Shot) { + let rect = shot.rect(on: app.frame) + record([ + "type": "camera", + "rect": [rect.minX, rect.minY, rect.width, rect.height], + ]) + } + + // MARK: - The hands + + /// A palette entry by its title, scrolled on screen first if it is not. + /// + /// The title is not unique: the transport's step button is "Forward" too. + /// Of every button with that label this is the one furthest left, since + /// the palette is the first column — read from one snapshot, because + /// walking a query's matches one by one raced the UI and lost an element + /// between counting and fetching it. + /// + /// **Taps here are coordinates, so nothing scrolls for them.** An element's + /// own `tap()` scrolls it into view; a coordinate below the fold is simply + /// pressed, and on the 11-inch iPad that is where Repeat is — the first run + /// put a Start Fill where the repeat should have been. + @MainActor + private func palette(_ title: String) throws -> CGRect { + let top: CGFloat = 150 + let bottom = app.frame.height - 40 + var frame = try find(title, type: .button) { $0.minX < $1.minX } + for _ in 0..<4 where frame.minY < top || frame.maxY > bottom { + let distance = min(max(frame.midY - (top + bottom) / 2, -420), 420) + let from = CGPoint(x: frame.midX, y: distance > 0 ? bottom - 60 : top + 60) + swipe(from: from, to: CGPoint(x: from.x, y: from.y - distance)) + pause(0.5) + frame = try find(title, type: .button) { $0.minX < $1.minX } + } + return frame + } + + /// A button anywhere, by its accessibility label; the topmost if there + /// are several. + @MainActor + private func button(_ label: String) throws -> CGRect { + try find(label, type: .button) { $0.minY < $1.minY } + } + + @MainActor + private func find( + _ label: String, type: XCUIElement.ElementType, within timeout: TimeInterval = 10, + preferring order: (CGRect, CGRect) -> Bool + ) throws -> CGRect { + let deadline = Date().addingTimeInterval(timeout) + repeat { + var found: [CGRect] = [] + func walk(_ node: XCUIElementSnapshot) { + if node.elementType == type, node.label == label, !node.frame.isEmpty { + found.append(node.frame) + } + node.children.forEach(walk) + } + walk(try app.snapshot()) + if let best = found.min(by: order) { return best } + Thread.sleep(forTimeInterval: 0.3) + } while Date() < deadline + throw TeaserError.missing(label) + } + + @MainActor + private func frame(of element: XCUIElement) throws -> CGRect { + guard element.waitForExistence(timeout: 10) else { + throw TeaserError.missing(element.description) + } + return element.frame + } + + @MainActor + private func coordinate(_ point: CGPoint) -> XCUICoordinate { + app.coordinate(withNormalizedOffset: .zero) + .withOffset(CGVector(dx: point.x, dy: point.y)) + } + + @MainActor + private func tap(_ frame: CGRect) throws { + let point = CGPoint(x: frame.midX, y: frame.midY) + coordinate(point).tap() + record(["type": "tap", "x": point.x, "y": point.y]) + } + + /// Drags a palette entry into the program, dropping it in the gap above + /// the row the button `row` sits on, and makes sure it arrived: a block + /// wearing `bringing` has to turn up. + /// + /// Aimed a little *above* the gap's line: a gap is hit-tested over a whole + /// row's pitch, but the row below it paints later and wins right at its + /// own top edge — and a container's header is a drop target of its own + /// that puts the block inside. + /// + /// **A drop in the simulator does not always land.** The rows part, the + /// finger lets go, and nothing is inserted — seen once in five runs, with + /// the same gesture that works every other time. So a drag that brought + /// nothing is tried again, and the failed one is logged as a `cut`: the + /// driver takes it out of the film, so what is seen is the take that + /// worked. (The retry has not been needed since it was written, so the + /// cut has only been checked by reading.) + @MainActor + private func drag(_ title: String, above row: String, bringing label: String) throws { + for _ in 0..<3 { + let from = try palette(title) + let target = try button(row) + let start = CGPoint(x: from.midX, y: from.midY) + let end = CGPoint(x: target.minX + 60, y: target.minY - 22) + let hold = 0.7 + let velocity: CGFloat = 500 + let linger = 0.5 + let started = Date() + coordinate(start).press( + forDuration: hold, thenDragTo: coordinate(end), + withVelocity: XCUIGestureVelocity(velocity), thenHoldForDuration: linger) + if (try? find(label, type: .button, within: 3) { $0.minY < $1.minY }) != nil { + record([ + "type": "drag", "x": start.x, "y": start.y, "x2": end.x, "y2": end.y, + "t0": started.timeIntervalSince1970, "hold": hold, "linger": linger, + "velocity": velocity, + ]) + return + } + record(["type": "cut", "from": started.timeIntervalSince1970]) + pause(0.5) + } + throw TeaserError.missing("\(label) after dragging \(title) in") + } + + /// Scrolls with a finger: pressed only briefly, so it is a scroll and not + /// the long press that lifts a block, and held at the end so it does not + /// fling on past where it was aimed. + @MainActor + private func swipe(from start: CGPoint, to end: CGPoint) { + let velocity: CGFloat = 900 + let started = Date() + coordinate(start).press( + forDuration: 0.05, thenDragTo: coordinate(end), + withVelocity: XCUIGestureVelocity(velocity), thenHoldForDuration: 0.15) + record([ + "type": "drag", "x": start.x, "y": start.y, "x2": end.x, "y2": end.y, + "t0": started.timeIntervalSince1970, "hold": 0.05, "linger": 0.15, + "velocity": velocity, + ]) + } + + /// Opens a number chip, types the new value and closes it again. + /// + /// Typed on the hardware keyboard the driver attaches, so no on-screen + /// keyboard comes up — the way the maintainer's own recording was made, + /// with a pointer and a keyboard. The field is tapped in the middle, which + /// puts the caret after the digits, so the old value goes by backspace; + /// `⌘A` is not used, because selecting brought iPadOS 27's number keypad + /// up over the field and, now and then, the full keyboard under it. + @MainActor + private func setNumber(_ chip: String, to value: String, camera shot: Shot? = .program) + throws + { + if let shot { camera(shot) } + try tap(button(chip)) + pause(0.8) + try tap(frame(of: app.popovers.textFields.firstMatch)) + let editing = Date() + pause(0.6) + let digits = chip.split(separator: " ").last.map(\.count) ?? 0 + app.typeText(String(repeating: XCUIKeyboardKey.delete.rawValue, count: digits)) + record(["type": "key"]) + pause(0.3) + for character in value { + app.typeText(String(character)) + record(["type": "key"]) + pause(0.2) + } + pause(0.5) + // While the field has focus its caret blinks, and a blink is a new + // frame: without this the driver would keep every one of them. + record(["type": "typing", "from": editing.timeIntervalSince1970]) + dismissPopover() + pause(0.7) + } + + /// Closes a popover by tapping outside it — not drawn as a touch: it is + /// the tap nobody watching needs to see. + @MainActor + private func dismissPopover() { + let region = app.otherElements["PopoverDismissRegion"] + if region.exists { + region.tap() + } + else { + coordinate(CGPoint(x: 400, y: 740)).tap() + } + } + + /// Presses play and waits for the drawing to finish: the scrubber's value + /// has settled, and it either moved or had time to. + /// + /// Moving is not required, because a short drawing can be over before the + /// press returns — XCUITest waits for the app to go idle, which took nine + /// seconds once, and an eight-step triangle takes 0.8. Nor is ending + /// somewhere new: changing an angle leaves the step count alone, so the + /// run ends on exactly the value the last one did. + @MainActor + private func playDrawing() throws { + let scrubber = app.sliders.firstMatch + var last = scrubber.value.map { String(describing: $0) } + let pressed = Date() + try tap(frame(of: app.buttons["play.fill"])) + var moved = false + var settled = 0 + let deadline = Date().addingTimeInterval(60) + while Date() < deadline { + Thread.sleep(forTimeInterval: 0.2) + let now = scrubber.value.map { String(describing: $0) } + if now == last { + settled += 1 + } + else { + settled = 0 + moved = true + } + last = now + if settled >= 4, moved || Date().timeIntervalSince(pressed) > 3 { + linger(1.1) + return + } + } + throw TeaserError.stalled + } + + // MARK: - The log + + private func caption(_ text: String) { + record(["type": "caption", "text": text]) + } + + /// Asks the driver to keep this moment on screen for `seconds` even though + /// nothing moves: the driver cuts every still stretch short otherwise, and + /// a finished drawing is the one thing worth looking at for a while. + private func linger(_ seconds: TimeInterval) { + record(["type": "linger", "seconds": seconds]) + } + + private func pause(_ seconds: TimeInterval) { + Thread.sleep(forTimeInterval: seconds) + } + + private func record(_ event: [String: Any]) { + var event = event + event["t"] = Date().timeIntervalSince1970 + guard let data = try? JSONSerialization.data(withJSONObject: event) else { return } + events?.write(data + Data("\n".utf8)) + } + + private func environment(_ name: String) throws -> String { + guard let value = ProcessInfo.processInfo.environment[name], !value.isEmpty else { + throw XCTSkip("\(name) is not set — run this through Tools/teaser/teaser.rb") + } + return value + } + + private enum TeaserError: Error, CustomStringConvertible { + case missing(String) + case stalled + + var description: String { + switch self { + case .missing(let what): "nothing on screen matches \(what)" + case .stalled: "the drawing never finished" + } + } + } + } + +#endif