Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 32 additions & 11 deletions argus/core/venue_review.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,17 @@ def selected_venue(state_root: Path | str) -> str:
return str(state.get("target_venue") or state.get("venue") or "").strip()


# Shown where a venue name would be when none was selected. A project can reach
# its final review with nothing having set ``target_venue``; the Reviewer then
# judges against the stated standard and the operator's bar, and its verdict
# counts. The recorded ``venue`` stays empty rather than holding this label.
UNSELECTED_VENUE = "the stated standard (no venue selected)"


def _venue_phrase(venue: str) -> str:
return "the selected venue" if venue else "the stated standard"


def selected_acceptance_minimum(state_root: Path | str) -> str:
"""Read the operator's bar, independently of the Reviewer's own rating."""
from .pipeline_state import read_pipeline_state
Expand Down Expand Up @@ -62,14 +73,14 @@ def _venue_key(value: str) -> str:
def normalize_venue_review(value: Any) -> dict[str, Any] | None:
if not isinstance(value, Mapping):
return None
venue = value.get("venue")
venue = value.get("venue", "")
rationale = value.get("rationale")
recommendation = re.sub(r"[\s-]+", "_", str(value.get("recommendation") or "").strip().lower())
if recommendation == "best_paper_level":
recommendation = "best_paper"
issues = value.get("blocking_issues")
if (
not isinstance(venue, str) or not venue.strip()
not isinstance(venue, str)
or not isinstance(rationale, str) or not rationale.strip()
or recommendation not in RECOMMENDATIONS
or not isinstance(value.get("acceptance_clear"), bool)
Expand All @@ -92,18 +103,18 @@ def normalize_venue_review(value: Any) -> dict[str, Any] | None:

def venue_review_issue(value: Any, *, venue: str, minimum: str = "weak_accept") -> str:
assessment = normalize_venue_review(value)
if not venue:
return "no selected venue for the final paper review"
if assessment is None:
return "missing or invalid explicit venue recommendation"
if _venue_key(assessment["venue"]) != _venue_key(venue):
# With no selected venue there is nothing to bind the verdict to; with one,
# the verdict must be for that venue.
if venue and _venue_key(assessment["venue"]) != _venue_key(venue):
return f"reviewed venue {assessment['venue']!r} differs from selected venue {venue!r}"
if minimum not in ACCEPTED_RECOMMENDATIONS:
return f"invalid operator venue acceptance minimum: {minimum!r}"
if RECOMMENDATIONS.index(assessment["recommendation"]) < RECOMMENDATIONS.index(minimum):
return f"venue recommendation is {assessment['recommendation']}; {minimum} or better is required"
if assessment["acceptance_clear"] is not True:
return "Reviewer did not clearly support acceptance at the selected venue"
return f"Reviewer did not clearly support acceptance at {_venue_phrase(venue)}"
if assessment["blocking_issues"]:
return "reject-level issues remain: " + "; ".join(assessment["blocking_issues"])
if assessment.get("revision_required") is True:
Expand Down Expand Up @@ -191,8 +202,11 @@ def enforce_venue_acceptance(
decision.status = "blocked"
decision.backend_unavailable = True
decision.backend_stop_kind = "backend_unavailable"
decision.reason = "Final Reviewer omitted a valid assessment for the selected venue."
decision.next_action = "Retry the independent Reviewer on the same paper and submit a review action with its actual recommendation for the selected venue."
decision.reason = f"Final Reviewer omitted a valid assessment for {_venue_phrase(venue)}."
decision.next_action = (
"Retry the independent Reviewer on the same paper and submit a review action "
f"with its actual recommendation for {_venue_phrase(venue)}."
)
return
current = paper_review_snapshot(artifact_root)
issue = venue_review_issue(assessment, venue=venue, minimum=minimum)
Expand Down Expand Up @@ -223,7 +237,8 @@ def enforce_venue_acceptance(
decision.reason = assessment["rationale"]
if not decision.next_action.strip():
repairs = "; ".join(assessment["blocking_issues"]) or assessment["rationale"]
decision.next_action = "Revise the current paper against the selected venue's standard: " + repairs
standard = "the selected venue's standard" if venue else "the stated standard"
decision.next_action = f"Revise the current paper against {standard}: " + repairs
if (
minimum in ACCEPTED_RECOMMENDATIONS
and assessment["recommendation"] in ACCEPTED_RECOMMENDATIONS
Expand Down Expand Up @@ -255,8 +270,14 @@ def venue_review_instruction(venue: str, *, minimum: str = "weak_accept") -> str
minimum_label = minimum.replace("_", " ")
return (
"## Final paper acceptance — mandatory operator standard\n"
f"Act as an independent reviewer for the currently selected venue: {venue or '(not selected)'}. "
"Read its researched criteria and the actual current manuscript, rendered pages, "
+ (
f"Act as an independent reviewer for the currently selected venue: {venue}. "
"Read its researched criteria and the actual current manuscript, rendered pages, "
if venue else
"No venue has been selected for this paper. Act as an independent reviewer for a "
"strong venue in its field, say which standard you applied, and read the actual "
"current manuscript, rendered pages, "
) +
"and claim-critical evidence. Judge novelty, significance, soundness, evidence, "
"reproducibility, presentation, and fit at that venue. Finishing edits, compiling, "
"an old certificate, or the Engineer's confidence is not an acceptance decision.\n"
Expand Down
4 changes: 3 additions & 1 deletion argus/reviewer/_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -581,12 +581,14 @@ def _persist_research_review(
or ""
).strip()
if decision.venue_review is not None:
from ..core.venue_review import UNSELECTED_VENUE

challenge = "\n".join(f"- {issue}" for issue in decision.venue_review["blocking_issues"])
text = (
"# Authoritative review\n\n"
+ (
"## Selected-venue assessment\n"
f"Venue: {decision.venue_review['venue']}\n\n"
f"Venue: {decision.venue_review['venue'] or UNSELECTED_VENUE}\n\n"
f"Recommendation: {decision.venue_review['recommendation']}\n\n"
if decision.venue_review is not None else f"**Judgment:** {decision.status}\n\n"
)
Expand Down
43 changes: 43 additions & 0 deletions tests/test_venue_acceptance.py
Original file line number Diff line number Diff line change
Expand Up @@ -319,3 +319,46 @@ def test_stage_certificate_keeps_venue_rating_and_expires_on_figure_changes(pape
record = latest_stage_review(paper, "review")
assert record["certified"] is False
assert "changed" in record["stale_reason"]


@pytest.mark.parametrize("recommendation", ["weak_accept", "weak_reject"])
def test_reviewer_verdict_counts_when_no_venue_was_selected(paper, recommendation):
from types import SimpleNamespace

from argus.core.stage_certificate import latest_stage_review, record_stage_review

state_root = paper
state = read_pipeline_state(state_root)
state.pop("target_venue", None)
state.pop("venue", None)
write_pipeline_state(state_root, state)
runner = _Runner(assessment(recommendation))
review = Reviewer(runner).evaluate(
objective="Judge the current paper", round_index=1, session_id=None,
main_summary="Ready for review", main_error=None, scope="final_submission",
config=ReviewerConfig(model="gpt-5.6-sol", active_vertical="research", working_dir=str(paper), artifact_root=str(paper), vertical_state_root=str(state_root)),
)
assert "No venue has been selected for this paper" in runner.prompt
assert "(not selected)" not in runner.prompt
# A verdict against the stated standard is a verdict: never a Reviewer backend failure.
assert not review.backend_unavailable
assert "omitted a valid assessment" not in (review.reason or "")
# No venue name is invented for the record.
assert review.venue_review["venue"] == ""
assert review.venue_review["recommendation"] == recommendation
report = (paper / "paper/REVIEW.md").read_text()
assert f"Recommendation: {recommendation}" in report
accepted = recommendation == "weak_accept"
assert review.final_submission_certified is accepted
# The finalizers apply the same check: an accepting verdict lets the project finish.
issue = current_venue_acceptance_issue(review, state_root=state_root, artifact_root=paper)
assert (issue == "") is accepted
record_stage_review(
state_root=state_root, project_root=paper, stage="review", item=SimpleNamespace(id="final-review"),
manager_action="complete", venue_review=review.venue_review,
venue_review_snapshot=review.venue_review_snapshot,
)
assert latest_stage_review(state_root, "review")["certified"] is accepted
if not accepted:
assert "selected venue" not in issue
assert "selected venue" not in review.next_action
Loading