Repository navigation
fix: 发布 Feed 前同步并重新生成,修复旧任务重跑冲突 #1
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| #!/usr/bin/env python3 | ||
| """Publish from a disposable CI checkout, regenerating when main advances.""" | ||
|
|
||
| import json | ||
| from pathlib import Path | ||
| import subprocess | ||
| import sys | ||
|
|
||
|
|
||
| def run(*args, check=True): | ||
| return subprocess.run(args, check=check, timeout=90) | ||
|
|
||
|
|
||
| def revision(ref): | ||
| return subprocess.check_output(["git", "rev-parse", ref], text=True, timeout=90).strip() | ||
|
|
||
|
|
||
| def main(): | ||
| run("git", "config", "user.name", "github-actions[bot]") | ||
| run("git", "config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com") | ||
| for attempt in range(1, 4): | ||
| print(f"Publication attempt {attempt}/3.", flush=True) | ||
| run("git", "fetch", "origin", "main") | ||
| # releases.atom is generated: rebuild it on the latest base instead of rebasing it. | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Publish only generator output that the run tested. The test step runs on the triggering checkout ( Do not run the unit tests inside Suggested remedy: pass the tested commit (
|
||
| run("git", "reset", "--hard", "origin/main") | ||
| base = revision("HEAD") | ||
| config = json.loads(Path("feed.json").read_text(encoding="utf-8")) | ||
| for source in config["sources"]: | ||
| repo = str(Path(".sources") / source["directory"]) | ||
| run("git", "-C", repo, "fetch", "origin", "main") | ||
| run("git", "-C", repo, "reset", "--hard", "FETCH_HEAD") | ||
| run(sys.executable, "scripts/generate_feed.py") | ||
| run("git", "add", "releases.atom") | ||
| changes = run("git", "diff", "--cached", "--quiet", check=False) | ||
| if changes.returncode == 0: | ||
| print("No document changes.", flush=True) | ||
| return | ||
| if changes.returncode != 1: | ||
| changes.check_returncode() | ||
| run("git", "commit", "-m", "chore: update document feed") | ||
| push = run("git", "push", "origin", "HEAD:main", check=False) | ||
| if push.returncode == 0: | ||
| return | ||
| run("git", "fetch", "origin", "main") | ||
| if revision("origin/main") == base: | ||
|
Comment on lines
+41
to
+45
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
| print("Push failed without main advancing; not retrying.", file=sys.stderr) | ||
| raise SystemExit(push.returncode) | ||
| print("Main advanced during publication; regenerating on the new base.", flush=True) | ||
| raise SystemExit("Could not publish after 3 attempts: main kept advancing.") | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| main() | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,202 @@ | ||
| import json | ||
| import os | ||
| from pathlib import Path | ||
| import shutil | ||
| import signal | ||
| import subprocess | ||
| import sys | ||
| import tempfile | ||
| import textwrap | ||
| import unittest | ||
|
|
||
| sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) | ||
| from generate_feed import generate | ||
|
|
||
| ROOT = Path(__file__).resolve().parents[1] | ||
|
|
||
|
|
||
| class PublishFeedTests(unittest.TestCase): | ||
| def setUp(self): | ||
| self.temp = tempfile.TemporaryDirectory() | ||
| self.addCleanup(self.temp.cleanup) | ||
| self.root = Path(self.temp.name) | ||
| self.env = dict(os.environ, GIT_CONFIG_NOSYSTEM="1", | ||
| GIT_CONFIG_GLOBAL=os.devnull, GIT_TERMINAL_PROMPT="0") | ||
| self.config = json.loads((ROOT / "feed.json").read_text()) | ||
| self.config["sources"] = [{"repository": "example/source", "directory": "source", | ||
| "label": "Test", "prefix": ""}] | ||
| self.source = self.root / "sources/source" | ||
| self.source.mkdir(parents=True) | ||
| self.git(self.source, "init", "-b", "main") | ||
| self.identity(self.source) | ||
| self.update_source("Original article") | ||
|
|
||
| self.remote = self.root / "remote.git" | ||
| self.git(self.root, "init", "--bare", "--initial-branch=main", str(self.remote)) | ||
| self.writer = self.root / "writer" | ||
| self.writer.mkdir() | ||
| self.git(self.writer, "init", "-b", "main") | ||
| self.identity(self.writer) | ||
| (self.writer / "scripts").mkdir() | ||
| shutil.copy(ROOT / "scripts/generate_feed.py", self.writer / "scripts") | ||
| (self.writer / "feed.json").write_text(json.dumps(self.config)) | ||
| (self.writer / ".gitignore").write_text(".sources/\n") | ||
| self.write_feed(self.writer) | ||
| self.git(self.writer, "add", ".") | ||
| self.git(self.writer, "commit", "-m", "Initial feed") | ||
| self.git(self.writer, "remote", "add", "origin", str(self.remote)) | ||
| self.git(self.writer, "push", "origin", "main") | ||
| self.runner = self.root / "runner" | ||
| self.git(self.root, "clone", str(self.remote), str(self.runner)) | ||
| self.identity(self.runner) | ||
| (self.runner / ".sources").mkdir() | ||
| self.git(self.root, "clone", str(self.source), str(self.runner / ".sources/source")) | ||
|
|
||
| def git(self, repo, *args): | ||
| return subprocess.check_output(["git", "-C", str(repo), *args], | ||
| env=self.env, stderr=subprocess.PIPE, | ||
| text=True, timeout=10).strip() | ||
|
|
||
| def identity(self, repo): | ||
| self.git(repo, "config", "user.name", "Publication test") | ||
| self.git(repo, "config", "user.email", "test@example.com") | ||
|
|
||
| def update_source(self, title): | ||
| (self.source / "article.md").write_text(f"# {title}\n\nDocument body.\n") | ||
| self.git(self.source, "add", ".") | ||
| self.git(self.source, "commit", "-m", title) | ||
|
|
||
| def write_feed(self, repo): | ||
| content, _ = generate(self.config, self.root / "sources") | ||
| (repo / "releases.atom").write_bytes(content) | ||
|
|
||
| def publish(self): | ||
| command = [sys.executable, str(ROOT / "scripts/publish_feed.py")] | ||
| with subprocess.Popen(command, cwd=self.runner, env=self.env, | ||
| stdout=subprocess.PIPE, stderr=subprocess.PIPE, | ||
| text=True, start_new_session=True) as process: | ||
| try: | ||
| stdout, stderr = process.communicate(timeout=30) | ||
| except subprocess.TimeoutExpired: | ||
| os.killpg(process.pid, signal.SIGKILL) | ||
| process.communicate() | ||
| raise | ||
| return subprocess.CompletedProcess(command, process.returncode, stdout, stderr) | ||
|
|
||
| def install_race_hook(self, *, every_push=False, update_source=False): | ||
| hook = self.runner / ".git/hooks/pre-push" | ||
| hook.write_text(f"#!{sys.executable}\n" + textwrap.dedent(f"""\ | ||
| import os | ||
| from pathlib import Path | ||
| import subprocess | ||
| import sys | ||
|
|
||
| marker = Path({str(self.root / 'push-attempts')!r}) | ||
| attempt = int(marker.read_text()) + 1 if marker.exists() else 1 | ||
| marker.write_text(str(attempt)) | ||
| if attempt > 1 and not {every_push!r}: | ||
| sys.exit(0) | ||
| env = dict(os.environ) | ||
| for key in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_PREFIX"): | ||
| env.pop(key, None) | ||
|
|
||
| def git(repo, *args): | ||
| subprocess.run(["git", "-C", str(repo), *args], env=env, | ||
| check=True, timeout=10) | ||
|
|
||
| source = Path({str(self.source)!r}) | ||
| writer = Path({str(self.writer)!r}) | ||
| if {update_source!r}: | ||
| (source / "article.md").write_text("# Latest during push\\n") | ||
| git(source, "add", ".") | ||
| git(source, "commit", "-m", "New source article during publication") | ||
| (writer / "README.md").write_text(f"Concurrent change {{attempt}}\\n") | ||
| git(writer, "add", ".") | ||
| git(writer, "commit", "-m", f"Concurrent change {{attempt}}") | ||
| git(writer, "push", "origin", "main") | ||
| """)) | ||
| hook.chmod(0o755) | ||
|
|
||
| def test_stale_run_preserves_newer_feed_and_other_main_changes(self): | ||
| self.update_source("Intermediate article") | ||
| self.git(self.runner / ".sources/source", "pull", "--ff-only", "origin", "main") | ||
| self.write_feed(self.runner) | ||
| self.update_source("Latest article") | ||
| self.write_feed(self.writer) | ||
| (self.writer / "README.md").write_text("Concurrent main change\n") | ||
| self.git(self.writer, "add", ".") | ||
| self.git(self.writer, "commit", "-m", "Publish latest feed and documentation") | ||
| self.git(self.writer, "push", "origin", "main") | ||
| latest = self.git(self.remote, "rev-parse", "main") | ||
|
|
||
| result = self.publish() | ||
|
|
||
| self.assertEqual(result.returncode, 0, result.stdout + result.stderr) | ||
| self.assertEqual(self.git(self.remote, "rev-parse", "main"), latest) | ||
| self.assertEqual((self.runner / "releases.atom").read_bytes(), | ||
| (self.writer / "releases.atom").read_bytes()) | ||
| self.assertEqual((self.runner / "README.md").read_text(), "Concurrent main change\n") | ||
| self.assertEqual(self.git(self.runner / ".sources/source", "rev-parse", "HEAD"), | ||
| self.git(self.source, "rev-parse", "HEAD")) | ||
|
|
||
| def test_unchanged_feed_does_not_create_a_commit(self): | ||
| before = self.git(self.remote, "rev-parse", "main") | ||
| result = self.publish() | ||
| self.assertEqual(result.returncode, 0, result.stdout + result.stderr) | ||
| self.assertIn("No document changes.", result.stdout) | ||
| self.assertEqual(self.git(self.remote, "rev-parse", "main"), before) | ||
|
|
||
| def test_push_race_refreshes_sources_and_preserves_concurrent_commit(self): | ||
| self.update_source("Intermediate article") | ||
| self.install_race_hook(update_source=True) | ||
|
|
||
| result = self.publish() | ||
|
|
||
| self.assertEqual(result.returncode, 0, result.stdout + result.stderr) | ||
| self.assertEqual((self.root / "push-attempts").read_text(), "2") | ||
| self.git(self.remote, "merge-base", "--is-ancestor", | ||
| self.git(self.writer, "rev-parse", "HEAD"), "main") | ||
| self.assertEqual(self.git(self.remote, "show", "main:README.md"), "Concurrent change 1") | ||
| self.assertIn("Latest during push", self.git(self.remote, "show", "main:releases.atom")) | ||
| self.assertNotIn("Intermediate article", self.git(self.remote, "show", "main:releases.atom")) | ||
| self.assertEqual(self.git(self.remote, "show", "--format=", "--name-only", "main"), | ||
| "releases.atom") | ||
|
|
||
| def test_unrelated_push_failure_is_not_retried(self): | ||
| self.update_source("Unpublished article") | ||
| before = self.git(self.remote, "rev-parse", "main") | ||
| hook = self.remote / "hooks/pre-receive" | ||
| hook.write_text(f"#!{sys.executable}\n" + textwrap.dedent(f"""\ | ||
| from pathlib import Path | ||
| import sys | ||
| marker = Path({str(self.root / 'rejected-pushes')!r}) | ||
| with marker.open("a") as output: | ||
| output.write("rejected\\n") | ||
| print("Publication denied by remote policy", file=sys.stderr) | ||
| sys.exit(1) | ||
| """)) | ||
| hook.chmod(0o755) | ||
|
|
||
| result = self.publish() | ||
|
|
||
| self.assertNotEqual(result.returncode, 0) | ||
| self.assertIn("Publication denied by remote policy", result.stderr) | ||
| self.assertEqual((self.root / "rejected-pushes").read_text(), "rejected\n") | ||
| self.assertEqual(self.git(self.remote, "rev-parse", "main"), before) | ||
|
|
||
| def test_repeated_push_races_stop_after_three_attempts(self): | ||
| self.update_source("Unpublished article") | ||
| original = self.git(self.remote, "show", "main:releases.atom") | ||
| self.install_race_hook(every_push=True) | ||
|
|
||
| result = self.publish() | ||
|
|
||
| self.assertNotEqual(result.returncode, 0) | ||
| self.assertEqual((self.root / "push-attempts").read_text(), "3") | ||
| self.assertEqual(self.git(self.remote, "show", "main:releases.atom"), original) | ||
| self.assertEqual(self.git(self.remote, "show", "main:README.md"), "Concurrent change 3") | ||
| self.assertIn("3 attempts", result.stderr) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| unittest.main() |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Script executed:
Repository: kejun/docs-feed
Length of output: 4411
Script executed:
Repository: kejun/docs-feed
Length of output: 9912
Install requirements after the publish reset.
The publish step resets the checkout to
origin/mainand then runsgenerate_feed.py. The workflow installsrequirements.txtbefore that reset. If main adds a generator import and its requirement after the install, generation fails withImportErrorand this run pushes nothing. The fix is to install requirements after the reset, before generation.🐛 Suggested fix
run("git", "reset", "--hard", "origin/main") + run(sys.executable, "-m", "pip", "install", "-r", "requirements.txt") base = revision("HEAD")