From 6fcf82edf2d8e2d595010f52364bbdea46a33b73 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Mon, 1 Jun 2026 16:14:24 -0700 Subject: [PATCH 01/62] Ruff formatting and checking --- mpcontribs-api/src/mpcontribs_api/old/core.py | 647 ++++++++++++++++++ .../mpcontribs/ingester/cli.py | 4 +- .../mpcontribs/ingester/webui.py | 11 +- mpcontribs-kernel-gateway/make_seed.py | 1 - mpcontribs-lux/mpcontribs/lux/autogen.py | 2 +- .../esoteric_ephemera/test_schemas.py | 1 - mpcontribs-portal/mpcontribs/portal/views.py | 2 - .../users/als_beamline/scripts/__main__.py | 3 +- .../als_beamline/scripts/translate_PyPt.py | 3 - .../dilute_solute_diffusion/pre_submission.py | 6 +- .../mpcontribs/users/qmcdb/main/views.py | 5 +- .../mpcontribs/users/qmcdb/records/views.py | 7 +- .../users/redox_thermo_csp/pre_submission.py | 9 +- .../redox_thermo_csp/update_energy_data.py | 3 +- .../screening_inorganic_pv/pre_submission.py | 7 +- .../mpcontribs/users/swf/pre_submission.py | 2 - mpcontribs-portal/mpcontribs/users/utils.py | 3 +- .../2dmatpedia.ipynb | 5 +- .../ExpXAS.ipynb | 3 +- .../HFP2023.ipynb | 3 - .../MnO2_phase_selection.ipynb | 3 +- .../carrier_transport.ipynb | 6 +- .../dilute_solute_diffusion.ipynb | 6 +- .../experimental_thermo.ipynb | 2 - .../experimental_thermoelectrics.ipynb | 2 +- .../ferroelectrics.ipynb | 7 +- .../ion_ref_data.ipynb | 2 +- .../jarvis_dft.ipynb | 4 +- .../matscholar.ipynb | 3 +- .../mofexplorer.ipynb | 2 +- .../ocp/ocp-upload.ipynb | 1 - .../open_catalyst_project.ipynb | 4 +- .../perovskites_diffusion.ipynb | 4 +- .../pydatarecognition.ipynb | 4 +- .../screening_inorganic_pv.ipynb | 2 +- .../silicon_defects.ipynb | 4 +- .../springer_materials.ipynb | 2 +- .../transparent_conductors.ipynb | 1 - .../get_started.ipynb | 5 +- .../ml.materialsproject.org/get_started.ipynb | 4 +- mpcontribs-portal/wsgi.py | 1 - mpcontribs-serverless/make_download/app.py | 1 - 42 files changed, 715 insertions(+), 82 deletions(-) create mode 100644 mpcontribs-api/src/mpcontribs_api/old/core.py diff --git a/mpcontribs-api/src/mpcontribs_api/old/core.py b/mpcontribs-api/src/mpcontribs_api/old/core.py new file mode 100644 index 0000000000..a4a600db19 --- /dev/null +++ b/mpcontribs-api/src/mpcontribs_api/old/core.py @@ -0,0 +1,647 @@ +import os +import yaml + +from copy import deepcopy +from re import Pattern +from importlib import import_module +from flasgger.marshmallow_apispec import SwaggerView as OriginalSwaggerView +from flasgger.marshmallow_apispec import schema2jsonschema +from marshmallow_mongoengine import ModelSchema +from flask_mongorest.views import ResourceView +from mongoengine.queryset import DoesNotExist +from mongoengine.queryset.visitor import Q +from werkzeug.exceptions import Unauthorized +from mpcontribs.api.config import DOC_DIR +from mpcontribs.api import is_gunicorn, get_logger + +logger = get_logger(__name__) + + +def get_limit_params(resource, method): + default = resource.default_limit + bulk = {"BulkUpdate", "BulkDelete"} + maximum = resource.bulk_update_limit if method in bulk else resource.max_limit + return [ + { + "name": "_skip", + "in": "query", + "type": "integer", + "description": "number of items to skip", + }, + { + "name": "_limit", + "in": "query", + "type": "integer", + "default": default, + "maximum": maximum, + "description": "maximum number of items to return", + }, + { + "name": "page", + "in": "query", + "type": "integer", + "description": "page number to return (in batches of `per_page/_limit`; alternative to `_skip`)", + }, + { + "name": "per_page", + "in": "query", + "type": "integer", + "default": default, + "maximum": maximum, + "description": "maximum number of items to return per page (same as `_limit`)", + }, + ] + + +def get_filter_params(name, filters): + filter_params = [] + is_pattern = isinstance(name, Pattern) + label = name.pattern if is_pattern else name + for op in filters: + if op.op == "exact" and not is_pattern: + name = label + description = f"filter {label}" + else: + suffix = op.suf if hasattr(op, "suf") else op.op + name = f"{label}__{suffix}" + description = f"filter {label} via ${op.op}" + + filter_params.append( + { + "name": name, + "in": "query", + "type": op.typ, + "description": description, + } + ) + if op.typ == "array": + filter_params[-1]["items"] = {"type": "string"} + if hasattr(op, "fmt"): + filter_params[-1]["format"] = op.fmt + + if op.allow_negation: + suffix = "not__" + suffix += op.suf if hasattr(op, "suf") else op.op + name = f"{label}__{suffix}" + description = f"filter {label} via ${op.op}" + param = deepcopy(filter_params[-1]) + param["name"] = name + param["description"] = description + filter_params.append(param) + + return filter_params + + +def get_specs(klass, method, collection): + method_name = method.__name__ if hasattr(method, "__name__") else method + default_response = { + "description": "Error", + "schema": {"type": "object", "properties": {"error": {"type": "string"}}}, + } + id_field = klass.resource.document._meta["id_field"].capitalize() + doc_name = collection[:-1].capitalize() + fields_param = None + if klass.resource.fields is not None: + fields_avail = ( + klass.resource.fields + klass.resource.get_optional_fields() + ["_all"] + ) + description = f"List of fields to include in response ({fields_avail})." + description += " Use dot-notation for nested subfields." + fields_param = { + "name": "_fields", + "in": "query", + "default": klass.resource.fields, + "type": "array", + "items": {"type": "string"}, + "description": description, + } + + field_pagination_params = [] + for field, limits in klass.resource.fields_to_paginate.items(): + field_pagination_params.append( + { + "name": f"{field}_page", + "in": "query", + "default": 1, + "type": "integer", + "description": f"page to retrieve for {field} field", + } + ) + field_pagination_params.append( + { + "name": f"{field}_per_page", + "in": "query", + "default": limits[0], + "maximum": limits[1], + "type": "integer", + "description": f"number of items to retrieve per page for {field} field", + } + ) + + filter_params = [] + if hasattr(klass.resource, "filters"): + for k, v in klass.resource.filters.items(): + filter_params += get_filter_params(k, v) + + order_params = [] + if klass.resource.allowed_ordering: + allowed_ordering = [ + o.pattern if isinstance(o, Pattern) else o + for o in klass.resource.allowed_ordering + ] + order_params = [ + { + "name": "_sort", + "in": "query", + "type": "string", + "description": f"sort {collection} via {allowed_ordering}. Prepend +/- for asc/desc.", + } + ] + + spec = None + if method_name == "Fetch": + params = [ + { + "name": "pk", + "in": "path", + "type": "string", + "required": True, + "description": f"{collection[:-1]} (primary key)", + } + ] + if fields_param is not None: + params.append(fields_param) + params += field_pagination_params + spec = { + "summary": f"Retrieve a {collection[:-1]}.", + "operationId": f"get{doc_name}By{id_field}", + "parameters": params, + "responses": { + 200: { + "description": f"single {collection} entry", + "schema": {"$ref": f"#/definitions/{klass.schema_name}"}, + }, + "default": default_response, + }, + } + + elif method_name == "BulkFetch": + params = [fields_param] if fields_param is not None else [] + params += field_pagination_params + params += order_params + params += filter_params + schema_props = { + "data": { + "type": "array", + "items": {"$ref": f"#/definitions/{klass.schema_name}"}, + } + } + if klass.resource.paginate: + schema_props["has_more"] = {"type": "boolean"} + schema_props["total_count"] = {"type": "integer"} + schema_props["total_pages"] = {"type": "integer"} + params += get_limit_params(klass.resource, method_name) + spec = { + "summary": f"Filter and retrieve {collection}.", + "operationId": f"query{doc_name}s", + "parameters": params, + "responses": { + 200: { + "description": f"list of {collection}", + "schema": {"type": "object", "properties": schema_props}, + }, + "default": default_response, + }, + } + + elif method_name == "Download": + params = [ + { + "name": "short_mime", + "in": "path", + "type": "string", + "required": True, + "description": "MIME Download Type: gz", + "default": "gz", + }, + { + "name": "format", + "in": "query", + "type": "string", + "required": True, + "description": f"download {collection} in different formats: {klass.resource.download_formats}", + }, + ] + params += [fields_param] if fields_param is not None else [] + params += order_params + params += filter_params + if klass.resource.paginate: + params += get_limit_params(klass.resource, method_name) + spec = { + "summary": f"Filter and download {collection}.", + "operationId": f"download{doc_name}s", + "parameters": params, + "produces": ["application/gzip"], + "responses": { + 200: { + "description": f"{collection} download", + "schema": {"type": "file"}, + }, + "default": default_response, + }, + } + + elif method_name == "Create": + spec = { + "summary": f"Create a new {collection[:-1]}.", + "operationId": f"create{doc_name}", + "parameters": [ + { + "name": f"{collection[:-1]}", + "in": "body", + "description": f"The object to use for {collection[:-1]} creation", + "schema": {"$ref": f"#/definitions/{klass.schema_name}"}, + } + ], + "responses": { + 200: { + "description": f"{collection[:-1]} created", + "schema": {"$ref": f"#/definitions/{klass.schema_name}"}, + }, + "default": default_response, + }, + } + + elif method_name == "BulkCreate": + spec = { + "summary": f"Create new {collection[:-1]}(s).", + "operationId": f"create{doc_name}s", + "parameters": [ + { + "name": f"{collection}", + "in": "body", + "description": f"The objects to use for {collection[:-1]} creation", + "schema": { + "type": "array", + "items": {"$ref": f"#/definitions/{klass.schema_name}"}, + }, + } + ], + "responses": { + 200: { + "description": f"{collection} created", + "schema": { + "type": "object", + "properties": { + "count": {"type": "integer"}, + "data": { + "type": "array", + "items": {"$ref": f"#/definitions/{klass.schema_name}"}, + }, + }, + }, + }, + "default": default_response, + }, + } + + elif method_name == "Update": + spec = { + "summary": f"Update a {collection[:-1]}.", + "operationId": f"update{doc_name}By{id_field}", + "parameters": [ + { + "name": "pk", + "in": "path", + "type": "string", + "required": True, + "description": f"The {collection[:-1]} (primary key) to update", + }, + { + "name": f"{collection[:-1]}", + "in": "body", + "description": f"The object to use for {collection[:-1]} update", + "schema": {"type": "object"}, + }, + ], + "responses": { + 200: { + "description": f"{collection[:-1]} updated", + "schema": {"$ref": f"#/definitions/{klass.schema_name}"}, + }, + "default": default_response, + }, + } + elif method_name == "BulkUpdate": + params = filter_params + params.append( + { + "name": f"{collection}", + "in": "body", + "description": f"The object to use for {collection} bulk update", + "schema": {"type": "object"}, + } + ) + schema_props = {"count": {"type": "integer"}} + if klass.resource.paginate: + schema_props["has_more"] = {"type": "boolean"} + schema_props["total_count"] = {"type": "integer"} + schema_props["total_pages"] = {"type": "integer"} + params += get_limit_params(klass.resource, method_name) + spec = { + "summary": f"Filter and update {collection}.", + "operationId": f"update{doc_name}s", + "parameters": params, + "responses": { + 200: { + "description": f"Number of {collection} updated", + "schema": {"type": "object", "properties": schema_props}, + }, + "default": default_response, + }, + } + + elif method_name == "BulkDelete": + params = filter_params + schema_props = {"count": {"type": "integer"}} + if klass.resource.paginate: + schema_props["has_more"] = {"type": "boolean"} + schema_props["total_count"] = {"type": "integer"} + schema_props["total_pages"] = {"type": "integer"} + params += get_limit_params(klass.resource, method_name) + spec = { + "summary": f"Filter and delete {collection}.", + "operationId": f"delete{doc_name}s", + "parameters": params, + "responses": { + 200: { + "description": f"Number of {collection} deleted", + "schema": {"type": "object", "properties": schema_props}, + }, + "default": default_response, + }, + } + + elif method_name == "Delete": + spec = { + "summary": f"Delete a {collection[:-1]}.", + "operationId": f"delete{doc_name}By{id_field}", + "parameters": [ + { + "name": "pk", + "in": "path", + "type": "string", + "required": True, + "description": f"The {collection[:-1]} (primary key) to delete", + } + ], + "responses": { + 200: {"description": f"{collection[:-1]} deleted"}, + "default": default_response, + }, + } + + return spec + + +class SwaggerView(OriginalSwaggerView, ResourceView): + """A class-based view defining additional methods""" + + def __init_subclass__(cls, **kwargs): + """initialize Schema, decorators, definitions, and tags""" + super().__init_subclass__(**kwargs) + + if not __name__ == cls.__module__: + # e.g.: cls.__module__ = mpcontribs.api.projects.views + views_path = cls.__module__.split(".") + doc_path = ".".join(views_path[:-1] + ["document"]) + cls.tags = [views_path[-2]] + doc_filepath = doc_path.replace(".", os.sep) + ".py" + if os.path.exists(doc_filepath): + cls.doc_name = cls.tags[0].capitalize() + Model = getattr(import_module(doc_path), cls.doc_name) + cls.schema_name = cls.doc_name + "Schema" + cls.Schema = type( + cls.schema_name, + (ModelSchema, object), + { + "Meta": type( + "Meta", + (object,), + dict(model=Model, ordered=True, model_build_obj=False), + ) + }, + ) + cls.definitions = {cls.schema_name: schema2jsonschema(cls.Schema)} + cls.resource.schema = cls.Schema + + # write flask-mongorest swagger specs + for method in cls.methods: + spec = get_specs(cls, method, cls.tags[0]) + if spec: + dir_path = os.path.join(DOC_DIR, cls.tags[0]) + file_path = os.path.join(dir_path, method.__name__ + ".yml") + if not os.path.exists(file_path): + os.makedirs(dir_path, exist_ok=True) + + if is_gunicorn: + with open(file_path, "w") as f: + yaml.dump(spec, f) + logger.debug( + f"{cls.tags[0]}.{method.__name__} written to {file_path}" + ) + + def get_groups(self, request): + groups = request.headers.get("X-Authenticated-Groups", "").split(",") + groups += request.headers.get("X-Consumer-Groups", "").split(",") + return set(grp.strip() for grp in groups if grp) + + def is_anonymous(self, request): + if not request.headers.get("X-Consumer-Username", ""): + return True + + is_anonymous = request.headers.get("X-Anonymous-Consumer", False) + if isinstance(is_anonymous, str): + is_anonymous = False if is_anonymous == "false" else True + + return is_anonymous + + def is_external(self, request): + return request.headers.get( + "X-Forwarded-Host" + ) is not None and not request.headers.get("Origin") + + def is_admin(self, request): + groups = self.get_groups(request) + admin_group = os.environ.get("ADMIN_GROUP", "admin") + return admin_group in groups + + def is_project_user(self, request, obj): + if hasattr(obj, "owner"): + owner = obj.owner + project = obj.name + elif hasattr(obj, "project"): + owner = obj.project.owner + project = obj.project.name + else: + raise Unauthorized(f"Unable to authorize {obj}") + + groups = self.get_groups(request) + username = request.headers.get("X-Consumer-Username") + return project in groups or owner == username + + def is_admin_or_project_user(self, request, obj): + if self.is_anonymous(request): + return False + + if self.is_admin(request): + return True + + return self.is_project_user(request, obj) + + def get_projects(self): + # project is LazyReferenceFields (multiple queries) + module = import_module("mpcontribs.api.projects.document") + Projects = getattr(module, "Projects") + exclude = list(Projects._fields.keys()) + only = ["name", "owner", "is_public", "is_approved"] + return Projects.objects.exclude(*exclude).only(*only) + + def get_projects_filter(self, username, groups, filter_names=None): + projects = self.get_projects() + if filter_names: + projects = projects.filter(name__in=filter_names) + + q = {"private": [], "public": []} + + for project in projects: + if project.owner == username or project.name in groups: + q["private"].append(project.name) + elif project.is_public and project.is_approved: + q["public"].append(project.name) + + # reduced query + qfilter = Q() + if q["private"]: + qfilter |= Q(project__in=q["private"]) + if q["public"]: + qfilter |= Q(project__in=q["public"], is_public=True) + + return qfilter + + def has_read_permission(self, request, qs): + if self.is_admin(request): + return qs # admins can read all entries + + groups = self.get_groups(request) + is_anonymous = self.is_anonymous(request) + is_external = self.is_external(request) + username = request.headers.get("X-Consumer-Username") + approved_public_filter = Q(is_public=True, is_approved=True) + + if request.path.startswith("/projects/"): + # external or internal requests can both read full project info + # anonymous requests can only read public approved projects + if is_anonymous: + return qs.filter(approved_public_filter) + + # authenticated requests can read approved public or accessible non-public projects + qfilter = approved_public_filter | Q(owner=username) + if groups: + qfilter |= Q(name__in=list(groups)) + + return qs.filter(qfilter) + else: + # contributions are set private/public independent from projects + # anonymous requests: + # - external: only meta-data of public contributions in approved public projects + # - internal: full public contributions in approved public projects + # authenticated requests: + # - private contributions in a public project are only accessible to owner/group + # - any contributions in a private project are only accessible to owner/group + component = request.path.split("/")[1] + + if component == "contributions": + q = qs._query + if is_anonymous and is_external: + qs = qs.exclude("data") + + if q and "project" in q and isinstance(q["project"], str): + projects = self.get_projects() + try: + project = projects.get(name=q["project"]) + except DoesNotExist: + return qs.none() + + if project.owner == username or project.name in groups: + return qs + elif project.is_public and project.is_approved: + return qs.filter(is_public=True) + else: + return qs.none() + else: + names = None + if q and "project" in q and "$in" in q["project"]: + names = q.pop("project").pop("$in") + + qfilter = self.get_projects_filter( + username, groups, filter_names=names + ) + return qs.filter(qfilter) + else: + # get component Object IDs for queryset + pk = request.view_args.get("pk") + from mpcontribs.api.contributions.document import get_resource + + resource = get_resource(component) + + def qfilter(qs): + return qs.clone() + + if pk: + ids = [resource.get_object(pk, qfilter=qfilter).id] + else: + ids = [o.id for o in resource.get_objects(qfilter=qfilter)[0]] + + if not ids: + return qs.none() + + # get list of readable contributions and their component Object IDs + module = import_module("mpcontribs.api.contributions.document") + Contributions = getattr(module, "Contributions") + qfilter = self.get_projects_filter(username, groups) + component = component[:-1] if component == "notebooks" else component + qfilter &= Q(**{f"{component}__in": ids}) + contribs = ( + Contributions.objects(qfilter).only(component).limit(len(ids)) + ) + # return new queryset using "ids__in" + readable_ids = ( + [getattr(contrib, component).id for contrib in contribs] + if component == "notebook" + else [ + dbref.id + for contrib in contribs + for dbref in getattr(contrib, component) + if dbref.id in ids + ] + ) + if not readable_ids: + return qs.none() + + qs._query_obj = Q(id__in=readable_ids) + # exclude optional fields if anonymous external request + if is_anonymous and is_external: + exclude = resource.get_optional_fields() + qs = qs.exclude(*exclude) + + return qs + + def has_add_permission(self, request, obj): + return self.is_admin_or_project_user(request, obj) + + def has_change_permission(self, request, obj): + return self.is_admin_or_project_user(request, obj) + + def has_delete_permission(self, request, obj): + return self.is_admin_or_project_user(request, obj) diff --git a/mpcontribs-ingester/mpcontribs/ingester/cli.py b/mpcontribs-ingester/mpcontribs/ingester/cli.py index 53965cc507..53267ef457 100644 --- a/mpcontribs-ingester/mpcontribs/ingester/cli.py +++ b/mpcontribs-ingester/mpcontribs/ingester/cli.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- # http://flask.pocoo.org/docs/0.10/patterns/appdispatch/ -import os, argparse, string +import os +import argparse +import string from werkzeug.serving import run_simple from werkzeug.wsgi import DispatcherMiddleware, SharedDataMiddleware from flask import Flask diff --git a/mpcontribs-ingester/mpcontribs/ingester/webui.py b/mpcontribs-ingester/mpcontribs/ingester/webui.py index 5d4f711563..c0b4debe56 100644 --- a/mpcontribs-ingester/mpcontribs/ingester/webui.py +++ b/mpcontribs-ingester/mpcontribs/ingester/webui.py @@ -1,7 +1,14 @@ from __future__ import unicode_literals, print_function, absolute_import -import json, os, socket, codecs, time, psutil -import sys, warnings, multiprocessing +import json +import os +import socket +import codecs +import time +import psutil +import sys +import warnings +import multiprocessing from tempfile import gettempdir from flask import render_template, request, Response, Blueprint, current_app from flask import url_for, redirect, make_response, stream_with_context, jsonify diff --git a/mpcontribs-kernel-gateway/make_seed.py b/mpcontribs-kernel-gateway/make_seed.py index f92a5df7c9..28aa21e0d1 100644 --- a/mpcontribs-kernel-gateway/make_seed.py +++ b/mpcontribs-kernel-gateway/make_seed.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import ddtrace.auto import nbformat as nbf nb = nbf.v4.new_notebook() diff --git a/mpcontribs-lux/mpcontribs/lux/autogen.py b/mpcontribs-lux/mpcontribs/lux/autogen.py index aeee406c21..95d6255663 100644 --- a/mpcontribs-lux/mpcontribs/lux/autogen.py +++ b/mpcontribs-lux/mpcontribs/lux/autogen.py @@ -100,7 +100,7 @@ def pydantic_model(self) -> Type[BaseModel]: self.file_name, orient=orient, lines=self.fmt == "jsonl" ) break - except Exception as exc: + except Exception: continue else: raise ValueError( diff --git a/mpcontribs-lux/tests/projects/esoteric_ephemera/test_schemas.py b/mpcontribs-lux/tests/projects/esoteric_ephemera/test_schemas.py index 70b6ec8019..0d74c5213c 100644 --- a/mpcontribs-lux/tests/projects/esoteric_ephemera/test_schemas.py +++ b/mpcontribs-lux/tests/projects/esoteric_ephemera/test_schemas.py @@ -2,7 +2,6 @@ import gzip import json -from pathlib import Path import numpy as np import pytest diff --git a/mpcontribs-portal/mpcontribs/portal/views.py b/mpcontribs-portal/mpcontribs/portal/views.py index 566a6b6542..ad2c12c1e0 100644 --- a/mpcontribs-portal/mpcontribs/portal/views.py +++ b/mpcontribs-portal/mpcontribs/portal/views.py @@ -12,8 +12,6 @@ from redis import Redis from io import BytesIO from copy import deepcopy -from pathlib import Path -from shutil import make_archive, rmtree from nbconvert import HTMLExporter from bravado.exception import HTTPNotFound from json2html import Json2Html diff --git a/mpcontribs-portal/mpcontribs/users/als_beamline/scripts/__main__.py b/mpcontribs-portal/mpcontribs/users/als_beamline/scripts/__main__.py index bcd0122bda..4fc72ada4a 100644 --- a/mpcontribs-portal/mpcontribs/users/als_beamline/scripts/__main__.py +++ b/mpcontribs-portal/mpcontribs/users/als_beamline/scripts/__main__.py @@ -1,4 +1,5 @@ -import argparse, os +import argparse +import os from mpcontribs.io.archieml.mpfile import MPFile from pre_submission import * diff --git a/mpcontribs-portal/mpcontribs/users/als_beamline/scripts/translate_PyPt.py b/mpcontribs-portal/mpcontribs/users/als_beamline/scripts/translate_PyPt.py index 995f5d045d..232914ab86 100644 --- a/mpcontribs-portal/mpcontribs/users/als_beamline/scripts/translate_PyPt.py +++ b/mpcontribs-portal/mpcontribs/users/als_beamline/scripts/translate_PyPt.py @@ -1,6 +1,3 @@ -import pandas as pd -import os -from scipy.interpolate import interp2d def get_translate(workdir=None): diff --git a/mpcontribs-portal/mpcontribs/users/dilute_solute_diffusion/pre_submission.py b/mpcontribs-portal/mpcontribs/users/dilute_solute_diffusion/pre_submission.py index 045f5bd71f..de41e67fa2 100644 --- a/mpcontribs-portal/mpcontribs/users/dilute_solute_diffusion/pre_submission.py +++ b/mpcontribs-portal/mpcontribs/users/dilute_solute_diffusion/pre_submission.py @@ -1,4 +1,6 @@ -import os, json, requests, sys +import os +import json +import requests from pandas import read_excel, isnull, ExcelWriter, Series from mpcontribs.io.core.recdict import RecursiveDict from mpcontribs.io.core.utils import clean_value, nest_dict @@ -60,7 +62,7 @@ def run(mpfile, hosts=None, download=False): if hosts is not None: if isinstance(hosts, int) and idx + 1 > hosts: break - elif isinstance(hosts, list) and not host in hosts: + elif isinstance(hosts, list) and host not in hosts: continue print("get mp-id for {}".format(host)) diff --git a/mpcontribs-portal/mpcontribs/users/qmcdb/main/views.py b/mpcontribs-portal/mpcontribs/users/qmcdb/main/views.py index 4e4c20ac48..cd80073db9 100644 --- a/mpcontribs-portal/mpcontribs/users/qmcdb/main/views.py +++ b/mpcontribs-portal/mpcontribs/users/qmcdb/main/views.py @@ -1,8 +1,5 @@ from django.shortcuts import render -from django.http import HttpResponseRedirect -from django.contrib.auth.decorators import login_required -from records.forms import MaterialQueryForm, MaterialSubmissionForm -from records.tables import QMCDBSetTable +from records.forms import MaterialQueryForm from records.models import QMCDBSet from django.utils.safestring import mark_safe from django.utils.html import escape diff --git a/mpcontribs-portal/mpcontribs/users/qmcdb/records/views.py b/mpcontribs-portal/mpcontribs/users/qmcdb/records/views.py index c0f0cc1372..589ef8e3e7 100644 --- a/mpcontribs-portal/mpcontribs/users/qmcdb/records/views.py +++ b/mpcontribs-portal/mpcontribs/users/qmcdb/records/views.py @@ -1,18 +1,13 @@ from __future__ import division from django.shortcuts import render from django.http import HttpResponseRedirect, HttpResponse -from django.contrib.auth.decorators import login_required from rest_framework import status from rest_framework.decorators import api_view from rest_framework.response import Response -from records.forms import MaterialQueryForm, MaterialSubmissionForm +from records.forms import MaterialSubmissionForm from records.models import QMCDBSet from records.serializers import QMCDBSetSerializer -from rest_framework.renderers import JSONRenderer -from rest_framework.parsers import JSONParser -from django.utils.six import BytesIO from django.utils.safestring import mark_safe -import numpy as np def manual_qmc_record_submission(request): diff --git a/mpcontribs-portal/mpcontribs/users/redox_thermo_csp/pre_submission.py b/mpcontribs-portal/mpcontribs/users/redox_thermo_csp/pre_submission.py index 15cfaf4de7..e95fd1f18c 100644 --- a/mpcontribs-portal/mpcontribs/users/redox_thermo_csp/pre_submission.py +++ b/mpcontribs-portal/mpcontribs/users/redox_thermo_csp/pre_submission.py @@ -1,16 +1,17 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -import os, json, re, sys +import os +import json +import re +import sys from glob import glob from datetime import datetime from itertools import groupby import pandas as pd -from mpcontribs.io.core.utils import get_composition_from_string from mpcontribs.io.core.recdict import RecursiveDict -from mpcontribs.io.core.utils import clean_value, read_csv, nest_dict +from mpcontribs.io.core.utils import clean_value, read_csv from mpcontribs.io.core.components import Table from mpcontribs.users.utils import duplicate_check -from mpcontribs.users.redox_thermo_csp.utils import redenth_act, get_debye_temp def get_fit_pars(sample_number): diff --git a/mpcontribs-portal/mpcontribs/users/redox_thermo_csp/update_energy_data.py b/mpcontribs-portal/mpcontribs/users/redox_thermo_csp/update_energy_data.py index b2eed60d99..3701993937 100644 --- a/mpcontribs-portal/mpcontribs/users/redox_thermo_csp/update_energy_data.py +++ b/mpcontribs-portal/mpcontribs/users/redox_thermo_csp/update_energy_data.py @@ -2,7 +2,6 @@ import datetime import os import shutil -import numpy as np from energy_analysis import EnergyAnalysis as enera from views import unstable_phases as unst @@ -21,7 +20,7 @@ new_energy_data = old_energy_data for db_id in paramlist: - if not "Exp" in db_id: + if "Exp" not in db_id: print(db_id) data_source = "Theo" # updates only theoretical data celsius = "True" # always True, parameter input in K currently disabled diff --git a/mpcontribs-portal/mpcontribs/users/screening_inorganic_pv/pre_submission.py b/mpcontribs-portal/mpcontribs/users/screening_inorganic_pv/pre_submission.py index 454a5597e3..aef7b1c136 100644 --- a/mpcontribs-portal/mpcontribs/users/screening_inorganic_pv/pre_submission.py +++ b/mpcontribs-portal/mpcontribs/users/screening_inorganic_pv/pre_submission.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- -import os, json +import os +import json from pandas import DataFrame from mpcontribs.io.core.recdict import RecursiveDict from mpcontribs.io.core.utils import clean_value @@ -46,11 +47,11 @@ def run(mpfile, **kwargs): rd = RecursiveDict({"formula": formula}) for k, v in config.items(): value = clean_value(d[k], v[1], max_dgts=4) - if not "." in v[0]: + if "." not in v[0]: rd[v[0]] = value else: keys = v[0].split(".") - if not keys[0] in rd: + if keys[0] not in rd: rd[keys[0]] = RecursiveDict({keys[1]: value}) else: rd[keys[0]][keys[1]] = value diff --git a/mpcontribs-portal/mpcontribs/users/swf/pre_submission.py b/mpcontribs-portal/mpcontribs/users/swf/pre_submission.py index 6c46bc0c1a..8c48375d97 100644 --- a/mpcontribs-portal/mpcontribs/users/swf/pre_submission.py +++ b/mpcontribs-portal/mpcontribs/users/swf/pre_submission.py @@ -1,5 +1,4 @@ from mpcontribs.config import mp_level01_titles -from mpcontribs.io.core.recdict import RecursiveDict from mpcontribs.io.core.utils import clean_value, get_composition_from_string from mpcontribs.users.utils import duplicate_check @@ -26,7 +25,6 @@ def round_to_100_percent(number_set, digit_after_decimal=1): def run(mpfile, **kwargs): import pymatgen import pandas as pd - from mpcontribs.users.swf.rest.rester import SwfRester # load data from google sheet google_sheet = mpfile.document[mp_level01_titles[0]].pop("google_sheet") diff --git a/mpcontribs-portal/mpcontribs/users/utils.py b/mpcontribs-portal/mpcontribs/users/utils.py index 4cf02217e9..e49f04401a 100644 --- a/mpcontribs-portal/mpcontribs/users/utils.py +++ b/mpcontribs-portal/mpcontribs/users/utils.py @@ -1,4 +1,5 @@ -import inspect, os +import inspect +import os from typing import Any, Dict diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/2dmatpedia.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/2dmatpedia.ipynb index daf3c45748..73641ce9f3 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/2dmatpedia.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/2dmatpedia.ipynb @@ -6,9 +6,10 @@ "metadata": {}, "outputs": [], "source": [ - "import os, gzip, json\n", + "import os\n", + "import gzip\n", + "import json\n", "from mpcontribs.client import Client\n", - "from pymatgen.core import Structure\n", "from pymatgen.ext.matproj import MPRester\n", "from urllib.request import urlretrieve\n", "from monty.json import MontyDecoder" diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/ExpXAS.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/ExpXAS.ipynb index d2fdf9ba09..89ad71cfd9 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/ExpXAS.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/ExpXAS.ipynb @@ -10,8 +10,7 @@ "from mpcontribs.client import Client\n", "from pathlib import Path\n", "from pandas import read_csv\n", - "import pandas as pd\n", - "import numpy as np" + "import pandas as pd" ] }, { diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/HFP2023.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/HFP2023.ipynb index 9b0f05ceb1..61acc0ccc0 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/HFP2023.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/HFP2023.ipynb @@ -9,15 +9,12 @@ "source": [ "%env MPRESTER_MUTE_PROGRESS_BARS 1\n", "# pip install mpcontribs-client mp_api pandas flatten_dict\n", - "import os\n", "import gzip\n", "import json\n", "\n", "from pathlib import Path\n", "from mpcontribs.client import Client\n", - "from mp_api.client import MPRester\n", "from pymatgen.core import Structure\n", - "from pandas import read_csv\n", "from flatten_dict import flatten, unflatten" ] }, diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/MnO2_phase_selection.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/MnO2_phase_selection.ipynb index d6c63cb805..328c0f2c2f 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/MnO2_phase_selection.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/MnO2_phase_selection.ipynb @@ -6,7 +6,8 @@ "metadata": {}, "outputs": [], "source": [ - "import json, os\n", + "import json\n", + "import os\n", "from mpcontribs.client import Client\n", "from pymatgen.core import Composition, Structure\n", "from pymatgen.ext.matproj import MPRester\n", diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/carrier_transport.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/carrier_transport.ipynb index 68e1c4f64d..e574ababd3 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/carrier_transport.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/carrier_transport.ipynb @@ -7,13 +7,13 @@ "outputs": [], "source": [ "from mpcontribs.client import Client\n", - "import gzip, json, os\n", + "import gzip\n", + "import json\n", + "import os\n", "import numpy as np\n", "from pandas import DataFrame\n", - "from collections import defaultdict\n", "from tqdm.notebook import tqdm\n", "from unflatten import unflatten\n", - "from pathlib import Path\n", "\n", "name = 'carrier_transport'" ] diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/dilute_solute_diffusion.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/dilute_solute_diffusion.ipynb index 3ce892c236..a26d81d3bb 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/dilute_solute_diffusion.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/dilute_solute_diffusion.ipynb @@ -28,7 +28,9 @@ "metadata": {}, "outputs": [], "source": [ - "import os, json, requests, sys\n", + "import os\n", + "import json\n", + "import requests\n", "from pandas import read_excel, isnull, ExcelWriter, Series\n", "from mp_api.client import MPRester\n", "from pathlib import Path\n", @@ -108,7 +110,7 @@ " if hosts is not None:\n", " if isinstance(hosts, int) and idx + 1 > hosts:\n", " break\n", - " elif isinstance(hosts, list) and not host in hosts:\n", + " elif isinstance(hosts, list) and host not in hosts:\n", " continue\n", "\n", " print(\"get mp-id for {}\".format(host))\n", diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/experimental_thermo.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/experimental_thermo.ipynb index a02e07d84a..4ddf73bb08 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/experimental_thermo.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/experimental_thermo.ipynb @@ -48,8 +48,6 @@ "from pathlib import Path\n", "import re\n", "from tqdm import tqdm\n", - "import numpy as np\n", - "import xlrd\n", "from monty.serialization import loadfn, dumpfn" ] }, diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/experimental_thermoelectrics.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/experimental_thermoelectrics.ipynb index fc639fd76a..b3164a38e3 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/experimental_thermoelectrics.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/experimental_thermoelectrics.ipynb @@ -11,7 +11,7 @@ "from mp_api.client import MPRester\n", "import pandas as pd\n", "import os\n", - "from flatten_dict import unflatten, flatten\n", + "from flatten_dict import unflatten\n", "from math import isnan" ] }, diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/ferroelectrics.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/ferroelectrics.ipynb index 458b629c00..5a2a1a8876 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/ferroelectrics.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/ferroelectrics.ipynb @@ -11,10 +11,9 @@ "source": [ "import json\n", "import numpy as np\n", - "from mpcontribs.client import Client, Attachment\n", + "from mpcontribs.client import Client\n", "from pathlib import Path\n", - "from flatten_dict import flatten, unflatten\n", - "from pymatgen.core import Structure" + "from flatten_dict import flatten, unflatten" ] }, { @@ -224,7 +223,7 @@ " if conf and k.startswith('polarization') and isinstance(v, list):\n", " name, fields = conf[\"name\"], conf[\"fields\"]\n", " contrib[\"data\"].setdefault(name, {})\n", - " if not \"unit\" in conf:\n", + " if \"unit\" not in conf:\n", " vmax, unit = max(v), fields[\"max\"]\n", " contrib[\"data\"][name]['max'] = f\"{round(vmax, 3)} {unit}\" if unit else v\n", " contrib[\"data\"][name]['index'] = v.index(vmax)\n", diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/ion_ref_data.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/ion_ref_data.ipynb index 4543e68cdb..318359a7c4 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/ion_ref_data.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/ion_ref_data.ipynb @@ -29,7 +29,7 @@ "outputs": [], "source": [ "from pprint import pprint\n", - "from monty.serialization import loadfn, dumpfn\n", + "from monty.serialization import loadfn\n", "from pymatgen.core.ion import Ion" ] }, diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/jarvis_dft.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/jarvis_dft.ipynb index bb08b09503..7f08f74b59 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/jarvis_dft.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/jarvis_dft.ipynb @@ -6,7 +6,9 @@ "metadata": {}, "outputs": [], "source": [ - "import os, json, tarfile\n", + "import os\n", + "import json\n", + "import tarfile\n", "from mpcontribs.client import Client\n", "from urllib.request import urlretrieve\n", "from monty.json import MontyDecoder\n", diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/matscholar.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/matscholar.ipynb index 31e5a2c243..bc5766f6de 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/matscholar.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/matscholar.ipynb @@ -7,8 +7,7 @@ "metadata": {}, "outputs": [], "source": [ - "from pathlib import Path\n", - "from mpcontribs.client import Client, Attachment" + "from mpcontribs.client import Client" ] }, { diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/mofexplorer.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/mofexplorer.ipynb index a9dc872a85..c399fc6f96 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/mofexplorer.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/mofexplorer.ipynb @@ -76,7 +76,7 @@ " raw = vs[-1].replace(\"^3\", \"³\")\n", " if raw in ureg:\n", " value, unit = vs[0], raw\n", - " except Exception as e:\n", + " except Exception:\n", " value, unit = v, None\n", " else:\n", " value, unit = vs[0], None\n", diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/ocp/ocp-upload.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/ocp/ocp-upload.ipynb index 7f82031902..15a7de18c9 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/ocp/ocp-upload.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/ocp/ocp-upload.ipynb @@ -12,7 +12,6 @@ "from ujson import load\n", "from pymatgen.core.structure import Molecule, Structure\n", "from pathlib import Path\n", - "from time import time\n", "from mpcontribs.client import Client\n", "from tqdm.auto import tqdm" ] diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/open_catalyst_project.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/open_catalyst_project.ipynb index 25f34580fc..934ac1ba4d 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/open_catalyst_project.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/open_catalyst_project.ipynb @@ -9,10 +9,8 @@ "source": [ "from mpcontribs.client import Client\n", "from monty.serialization import loadfn\n", - "from json import loads\n", "from pymatgen.core.structure import Molecule, Structure\n", - "from pathlib import Path\n", - "from time import time" + "from pathlib import Path" ] }, { diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/perovskites_diffusion.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/perovskites_diffusion.ipynb index 026d09bec2..634cb3c77c 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/perovskites_diffusion.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/perovskites_diffusion.ipynb @@ -53,7 +53,7 @@ "metadata": {}, "outputs": [], "source": [ - "import tarfile, os\n", + "import tarfile\n", "from pandas import read_excel\n", "\n", "units = {\n", @@ -97,7 +97,7 @@ " key = keys[col]\n", " if isinstance(key, str):\n", " key = key.strip()\n", - " if not key in abbreviations:\n", + " if key not in abbreviations:\n", " abbreviations[key] = col\n", " else:\n", " key = col.strip().lower()\n", diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/pydatarecognition.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/pydatarecognition.ipynb index 4f1d69e8b4..23c100fe54 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/pydatarecognition.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/pydatarecognition.ipynb @@ -8,11 +8,9 @@ "outputs": [], "source": [ "%env MPRESTER_MUTE_PROGRESS_BARS 1\n", - "import os\n", "from pathlib import Path\n", "from mpcontribs.client import Client\n", - "from mp_api.client import MPRester\n", - "from flatten_dict import unflatten, flatten\n", + "from flatten_dict import unflatten\n", "from pymatgen.io.cif import CifParser\n", "from pandas import DataFrame\n", "import numpy as np" diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/screening_inorganic_pv.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/screening_inorganic_pv.ipynb index cb9e56014f..270bc924d1 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/screening_inorganic_pv.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/screening_inorganic_pv.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "outputs": [], "source": [ - "import os, json\n", + "import json\n", "from pathlib import Path\n", "from pandas import DataFrame\n", "from mpcontribs.client import Client\n", diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/silicon_defects.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/silicon_defects.ipynb index b03018e553..6c100a231c 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/silicon_defects.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/silicon_defects.ipynb @@ -11,7 +11,7 @@ "from mpcontribs.client import Client, Attachment\n", "from pymatgen.core import Structure\n", "from pathlib import Path\n", - "from flatten_dict import flatten, unflatten" + "from flatten_dict import flatten" ] }, { @@ -95,7 +95,7 @@ "}\n", "\n", "for k, v in list(reorg.items()):\n", - " if not \"unit\" in v:\n", + " if \"unit\" not in v:\n", " root_field = reorg.pop(k).get(\"field\")\n", " \n", " for kk, vv in excitation_reorg.items():\n", diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/springer_materials.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/springer_materials.ipynb index ce4e3b1dfd..e7d0958f16 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/springer_materials.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/springer_materials.ipynb @@ -11,7 +11,7 @@ "import re\n", "from glob import glob\n", "from mpcontribs.client import Client\n", - "from flatten_dict import unflatten, flatten" + "from flatten_dict import unflatten" ] }, { diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/transparent_conductors.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/transparent_conductors.ipynb index bc8a8feef7..f123264613 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/transparent_conductors.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/transparent_conductors.ipynb @@ -7,7 +7,6 @@ "metadata": {}, "outputs": [], "source": [ - "import tarfile, os\n", "import numpy as np\n", "from pandas import read_excel\n", "from mpcontribs.client import Client" diff --git a/mpcontribs-portal/notebooks/lightsources.materialsproject.org/get_started.ipynb b/mpcontribs-portal/notebooks/lightsources.materialsproject.org/get_started.ipynb index c3162b1883..3b53d08f6d 100644 --- a/mpcontribs-portal/notebooks/lightsources.materialsproject.org/get_started.ipynb +++ b/mpcontribs-portal/notebooks/lightsources.materialsproject.org/get_started.ipynb @@ -7,8 +7,6 @@ "outputs": [], "source": [ "import os\n", - "import json\n", - "import gzip\n", "from zipfile import ZipFile\n", "from io import StringIO, BytesIO\n", "from numpy import where\n", @@ -16,8 +14,7 @@ "from pandas import to_numeric, read_csv\n", "from mpcontribs.client import Client, Attachment\n", "from tqdm.notebook import tqdm\n", - "from decimal import Decimal\n", - "from pathlib import Path" + "from decimal import Decimal" ] }, { diff --git a/mpcontribs-portal/notebooks/ml.materialsproject.org/get_started.ipynb b/mpcontribs-portal/notebooks/ml.materialsproject.org/get_started.ipynb index a13eefdc69..651a268de2 100644 --- a/mpcontribs-portal/notebooks/ml.materialsproject.org/get_started.ipynb +++ b/mpcontribs-portal/notebooks/ml.materialsproject.org/get_started.ipynb @@ -6,7 +6,9 @@ "metadata": {}, "outputs": [], "source": [ - "import wget, json, os, math\n", + "import wget\n", + "import json\n", + "import math\n", "from pathlib import Path\n", "from string import capwords\n", "from pybtex.database import parse_string\n", diff --git a/mpcontribs-portal/wsgi.py b/mpcontribs-portal/wsgi.py index e314b76514..96ebc59a28 100644 --- a/mpcontribs-portal/wsgi.py +++ b/mpcontribs-portal/wsgi.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import re import os -import ddtrace.auto import django_settings_file from django.core.wsgi import get_wsgi_application from whitenoise import WhiteNoise diff --git a/mpcontribs-serverless/make_download/app.py b/mpcontribs-serverless/make_download/app.py index 96f98401a7..9783183249 100644 --- a/mpcontribs-serverless/make_download/app.py +++ b/mpcontribs-serverless/make_download/app.py @@ -1,6 +1,5 @@ # TODO ddtrace import os -import json import logging import boto3 From 767bc9c500e9320caafa8fe55d704c6b8db36b47 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Thu, 4 Jun 2026 14:54:30 -0700 Subject: [PATCH 02/62] Infra changes for rewrite --- .claude/worktrees/unify-component-service | 1 + mpcontribs-api/scripts/start.sh | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 160000 .claude/worktrees/unify-component-service diff --git a/.claude/worktrees/unify-component-service b/.claude/worktrees/unify-component-service new file mode 160000 index 0000000000..7d84eafaec --- /dev/null +++ b/.claude/worktrees/unify-component-service @@ -0,0 +1 @@ +Subproject commit 7d84eafaec7223ee57b2b626f02815075a8dfc47 diff --git a/mpcontribs-api/scripts/start.sh b/mpcontribs-api/scripts/start.sh index 1452743286..729e75932d 100755 --- a/mpcontribs-api/scripts/start.sh +++ b/mpcontribs-api/scripts/start.sh @@ -10,4 +10,19 @@ PMGRC=$HOME/.pmgrc.yaml set -x +<<<<<< HEAD exec uvicorn mpcontribs_api.app:app --host 0.0.0.0 --port "$API_PORT" --workers "${NWORKERS:-2}" +||||||| parent of ba089932 (Infra changes for rewrite) +if [[ -n "$DD_TRACE_HOST" ]]; then + wait-for-it.sh "$DD_TRACE_HOST" -q -s -t 10 && STATS_ARG="--statsd-host $DD_AGENT_HOST:8125" || echo "WARNING: datadog agent unreachable" +fi + +[[ -n "$STATS_ARG" ]] && CMD="ddtrace-run gunicorn $STATS_ARG" || CMD="gunicorn" +exec $WAIT_FOR -- $CMD $SERVER_APP +======= +if [[ -n "$DD_TRACE_HOST" ]]; then + wait-for-it.sh "$DD_TRACE_HOST" -q -s -t 10 || echo "WARNING: datadog agent unreachable" +fi + +exec uvicorn mpcontribs_api.app:app --host 0.0.0.0 --port "$API_PORT" +>>>>>>> ba089932 (Infra changes for rewrite) From 9cd35cba72d53951adabe2b55f823e447ee6ed1e Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 2 Jun 2026 02:27:05 +0000 Subject: [PATCH 03/62] upgrade dependencies for deployment --- mpcontribs-api/requirements/deployment.txt | 551 ++++++++++++++++++ mpcontribs-client/requirements/deployment.txt | 18 + .../requirements/deployment.txt | 24 + 3 files changed, 593 insertions(+) create mode 100644 mpcontribs-api/requirements/deployment.txt diff --git a/mpcontribs-api/requirements/deployment.txt b/mpcontribs-api/requirements/deployment.txt new file mode 100644 index 0000000000..7056ece33a --- /dev/null +++ b/mpcontribs-api/requirements/deployment.txt @@ -0,0 +1,551 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --output-file=MPContribs/mpcontribs-api/requirements/deployment.txt MPContribs/mpcontribs-api/pyproject.toml python/requirements.txt +# +anyio==4.13.0 + # via jupyter-server +apispec==5.2.2 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +argon2-cffi==25.1.0 + # via + # jupyter-server + # notebook +argon2-cffi-bindings==25.1.0 + # via argon2-cffi +arrow==1.4.0 + # via isoduration +asn1crypto==1.5.1 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +asttokens==3.0.1 + # via stack-data +atlasq-tschaume==0.11.1.dev2 + # via flask-mongorest-mpcontribs +attrs==26.1.0 + # via + # jsonschema + # referencing +backports-zstd==1.5.0 + # via flask-compress +beautifulsoup4==4.14.3 + # via nbconvert +bibtexparser==1.4.4 + # via pymatgen-core +bleach[css]==6.3.0 + # via nbconvert +blinker==1.9.0 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +boltons==25.0.0 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +boto3==1.43.19 + # via flask-mongorest-mpcontribs +botocore==1.43.19 + # via + # boto3 + # s3transfer +brotli==1.2.0 + # via flask-compress +bytecode==0.17.0 + # via ddtrace +certifi==2026.5.20 + # via requests +cffi==2.0.0 + # via + # argon2-cffi-bindings + # cryptography +charset-normalizer==3.4.7 + # via requests +click==8.4.1 + # via + # flask + # rq +comm==0.2.3 + # via ipykernel +contourpy==1.3.3 + # via matplotlib +cramjam==2.11.0 + # via python-snappy +crontab==1.0.5 + # via rq-scheduler +cryptography==48.0.0 + # via pyopenssl +css-html-js-minify==2.5.5 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +cycler==0.12.1 + # via matplotlib +dateparser==1.4.0 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +ddtrace==4.3.0 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +debugpy==1.8.21 + # via ipykernel +decorator==5.3.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert +dnspython==2.8.0 + # via + # mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) + # pymongo +entrypoints==0.4 + # via jupyter-client +envier==0.6.1 + # via ddtrace +executing==2.2.1 + # via stack-data +fastjsonschema==2.21.2 + # via nbformat +fastnumbers==5.1.1 + # via flask-mongorest-mpcontribs +filetype==1.2.0 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +flasgger-tschaume==0.9.7 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +flask==2.2.5 + # via + # flasgger-tschaume + # flask-compress + # flask-marshmallow + # flask-mongoengine-tschaume + # flask-rq2 + # flask-sse +flask-compress==1.24 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +flask-marshmallow==1.4.0 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +flask-mongoengine-tschaume==1.1.0 + # via flask-mongorest-mpcontribs +flask-mongorest-mpcontribs==3.3.0 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +flask-rq2==18.3 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +flask-sse==1.0.0 + # via flask-mongorest-mpcontribs +flatten-dict==0.5.0 + # via flask-mongorest-mpcontribs +flexcache==0.3 + # via pint +flexparser==0.4 + # via pint +fonttools==4.63.0 + # via matplotlib +fqdn==1.5.1 + # via jsonschema +freezegun==1.5.5 + # via rq-scheduler +gevent==26.5.0 + # via gunicorn +greenlet==3.5.1 + # via gevent +gunicorn[gevent]==24.1.1 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +idna==3.17 + # via + # anyio + # jsonschema + # requests +ipykernel==6.29.5 + # via + # nbclassic + # notebook +ipython==9.14.0 + # via ipykernel +ipython-genutils==0.2.0 + # via + # nbclassic + # notebook +ipython-pygments-lexers==1.1.1 + # via ipython +isoduration==20.11.0 + # via jsonschema +itsdangerous==2.2.0 + # via flask +jedi==0.20.0 + # via ipython +jinja2==3.1.6 + # via + # flask + # jupyter-server + # mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) + # nbconvert + # notebook +jmespath==1.1.0 + # via + # boto3 + # botocore +joblib==1.5.3 + # via pymatgen-core +json2html==1.3.0 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +jsonpointer==3.1.1 + # via jsonschema +jsonschema[format-nongpl]==4.26.0 + # via + # flasgger-tschaume + # jupyter-events + # nbformat +jsonschema-specifications==2025.9.1 + # via jsonschema +jupyter-client==7.4.9 + # via + # ipykernel + # jupyter-server + # nbclient + # notebook +jupyter-core==5.9.1 + # via + # ipykernel + # jupyter-client + # jupyter-server + # nbclient + # nbconvert + # nbformat + # notebook +jupyter-events==0.12.1 + # via jupyter-server +jupyter-server==2.19.0 + # via notebook-shim +jupyter-server-terminals==0.5.4 + # via jupyter-server +jupyterlab-pygments==0.3.0 + # via nbconvert +kiwisolver==1.5.0 + # via matplotlib +lark==1.3.1 + # via rfc3987-syntax +lxml==6.1.1 + # via pymatgen-core +markupsafe==3.0.3 + # via + # jinja2 + # nbconvert + # werkzeug +marshmallow==3.26.2 + # via + # flask-marshmallow + # marshmallow-mongoengine + # mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +marshmallow-mongoengine==0.31.2 + # via flask-mongorest-mpcontribs +matplotlib==3.10.9 + # via + # -r python/requirements.txt + # pymatgen-core +matplotlib-inline==0.2.2 + # via + # ipykernel + # ipython +mimerender-pr36==0.0.2 + # via flask-mongorest-mpcontribs +mistune==3.2.1 + # via + # flasgger-tschaume + # nbconvert +mongoengine==0.29.3 + # via + # atlasq-tschaume + # flask-mongoengine-tschaume + # marshmallow-mongoengine +monty==2026.5.18 + # via pymatgen-core +more-itertools==11.1.0 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +mpmath==1.3.0 + # via sympy +narwhals==2.22.0 + # via plotly +nbclassic==1.3.3 + # via notebook +nbclient==0.10.4 + # via nbconvert +nbconvert==7.17.1 + # via + # jupyter-server + # notebook +nbformat==5.10.4 + # via + # jupyter-server + # mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) + # nbclient + # nbconvert + # notebook +nest-asyncio==1.6.0 + # via + # ipykernel + # jupyter-client + # nbclassic + # notebook +networkx==3.6.1 + # via pymatgen-core +notebook==6.5.7 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +notebook-shim==0.2.4 + # via nbclassic +numpy==2.4.6 + # via + # -r python/requirements.txt + # contourpy + # matplotlib + # monty + # mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) + # pandas + # pymatgen-core + # scipy + # spglib +opentelemetry-api==1.42.1 + # via ddtrace +orjson==3.11.9 + # via + # flask-mongorest-mpcontribs + # pymatgen-core +overrides==7.7.0 + # via jupyter-server +packaging==26.2 + # via + # gunicorn + # ipykernel + # jupyter-events + # jupyter-server + # marshmallow + # matplotlib + # nbconvert + # plotly +palettable==3.3.3 + # via pymatgen-core +pandas==3.0.3 + # via + # -r python/requirements.txt + # pymatgen-core +pandocfilters==1.5.1 + # via nbconvert +parso==0.8.7 + # via jedi +pexpect==4.9.0 + # via ipython +pillow==12.2.0 + # via matplotlib +pint==0.25.3 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +platformdirs==4.10.0 + # via + # jupyter-core + # pint +plotly==6.7.0 + # via pymatgen-core +prometheus-client==0.25.0 + # via + # jupyter-server + # notebook +prompt-toolkit==3.0.52 + # via ipython +psutil==7.2.2 + # via + # ipykernel + # ipython +psycopg2-binary==2.9.12 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +ptyprocess==0.7.0 + # via + # pexpect + # terminado +pure-eval==0.2.3 + # via stack-data +pycparser==3.0 + # via cffi +pygments==2.20.0 + # via + # ipython + # ipython-pygments-lexers + # nbconvert +pymatgen==2026.5.4 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +pymatgen-core==2026.5.18 + # via pymatgen +pymongo==4.17.0 + # via + # flask-mongorest-mpcontribs + # mongoengine +pyopenssl==26.2.0 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +pyparsing==3.3.2 + # via + # bibtexparser + # matplotlib +python-dateutil==2.9.0.post0 + # via + # arrow + # botocore + # dateparser + # flask-mongorest-mpcontribs + # freezegun + # jupyter-client + # matplotlib + # pandas + # rq-scheduler +python-json-logger==4.1.0 + # via jupyter-events +python-mimeparse==2.0.0 + # via mimerender-pr36 +python-snappy==0.7.3 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +pytz==2026.2 + # via dateparser +pyyaml==6.0.3 + # via + # flasgger-tschaume + # jupyter-events +pyzmq==27.1.0 + # via + # ipykernel + # jupyter-client + # jupyter-server + # notebook +redis==8.0.0 + # via + # flask-rq2 + # flask-sse + # rq +referencing==0.37.0 + # via + # jsonschema + # jsonschema-specifications + # jupyter-events +regex==2026.5.9 + # via dateparser +requests==2.34.2 + # via + # atlasq-tschaume + # pymatgen-core +rfc3339-validator==0.1.4 + # via + # jsonschema + # jupyter-events +rfc3986-validator==0.1.1 + # via + # jsonschema + # jupyter-events +rfc3987-syntax==1.1.0 + # via jsonschema +rpds-py==2026.5.1 + # via + # jsonschema + # referencing +rq==2.3.2 + # via + # flask-rq2 + # mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) + # rq-scheduler +rq-scheduler==0.14.0 + # via flask-rq2 +ruamel-yaml==0.19.1 + # via monty +s3transfer==0.18.0 + # via boto3 +scipy==1.17.1 + # via + # -r python/requirements.txt + # pymatgen-core +send2trash==2.1.0 + # via + # jupyter-server + # notebook +setproctitle==1.3.7 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +six==1.17.0 + # via + # flasgger-tschaume + # flask-sse + # python-dateutil + # rfc3339-validator +soupsieve==2.8.4 + # via beautifulsoup4 +spglib==2.7.0 + # via pymatgen-core +stack-data==0.6.3 + # via ipython +supervisor==4.3.0 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +sympy==1.14.0 + # via pymatgen-core +tabulate==0.10.0 + # via pymatgen-core +terminado==0.18.1 + # via + # jupyter-server + # jupyter-server-terminals + # notebook +tinycss2==1.4.0 + # via bleach +tornado==6.5.6 + # via + # ipykernel + # jupyter-client + # jupyter-server + # notebook + # terminado +tqdm==4.67.3 + # via pymatgen-core +traitlets==5.15.0 + # via + # ipykernel + # ipython + # jupyter-client + # jupyter-core + # jupyter-events + # jupyter-server + # matplotlib-inline + # nbclient + # nbconvert + # nbformat + # notebook +typing-extensions==4.15.0 + # via + # anyio + # beautifulsoup4 + # flexcache + # flexparser + # ipython + # opentelemetry-api + # pint + # pyopenssl + # referencing + # spglib +tzdata==2026.2 + # via arrow +tzlocal==5.3.1 + # via dateparser +uncertainties==3.2.3 + # via + # mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) + # pymatgen-core +uri-template==1.3.0 + # via jsonschema +urllib3==2.7.0 + # via + # botocore + # requests +wcwidth==0.7.0 + # via prompt-toolkit +webcolors==25.10.0 + # via jsonschema +webencodings==0.5.1 + # via + # bleach + # tinycss2 +websocket-client==1.9.0 + # via + # jupyter-server + # mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) +werkzeug==3.1.8 + # via + # flasgger-tschaume + # flask +wrapt==2.2.1 + # via ddtrace +zope-event==6.2 + # via gevent +zope-interface==8.5 + # via gevent +zstandard==0.25.0 + # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) diff --git a/mpcontribs-client/requirements/deployment.txt b/mpcontribs-client/requirements/deployment.txt index f19c9e1489..231613f00b 100644 --- a/mpcontribs-client/requirements/deployment.txt +++ b/mpcontribs-client/requirements/deployment.txt @@ -50,13 +50,25 @@ fonttools==4.63.0 # via matplotlib fqdn==1.5.1 # via jsonschema +<<<<<<< HEAD idna==3.18 +||||||| parent of 5ef6d1d6 (upgrade dependencies for deployment) +idna==3.16 +======= +idna==3.17 +>>>>>>> 5ef6d1d6 (upgrade dependencies for deployment) # via # jsonschema # requests importlib-resources==7.1.0 # via swagger-spec-validator +<<<<<<< HEAD ipython==9.14.1 +||||||| parent of 5ef6d1d6 (upgrade dependencies for deployment) +ipython==9.13.0 +======= +ipython==9.14.0 +>>>>>>> 5ef6d1d6 (upgrade dependencies for deployment) # via mpcontribs-client (MPContribs/mpcontribs-client/pyproject.toml) ipython-pygments-lexers==1.1.1 # via ipython @@ -100,7 +112,13 @@ msgpack==1.2.1 # via # bravado # bravado-core +<<<<<<< HEAD narwhals==2.22.1 +||||||| parent of 5ef6d1d6 (upgrade dependencies for deployment) +narwhals==2.21.2 +======= +narwhals==2.22.0 +>>>>>>> 5ef6d1d6 (upgrade dependencies for deployment) # via plotly networkx==3.6.1 # via pymatgen-core diff --git a/mpcontribs-kernel-gateway/requirements/deployment.txt b/mpcontribs-kernel-gateway/requirements/deployment.txt index 827d405953..e3a21ebecc 100644 --- a/mpcontribs-kernel-gateway/requirements/deployment.txt +++ b/mpcontribs-kernel-gateway/requirements/deployment.txt @@ -82,7 +82,13 @@ fonttools==4.63.0 # via matplotlib fqdn==1.5.1 # via jsonschema +<<<<<<< HEAD idna==3.18 +||||||| parent of 5ef6d1d6 (upgrade dependencies for deployment) +idna==3.16 +======= +idna==3.17 +>>>>>>> 5ef6d1d6 (upgrade dependencies for deployment) # via # anyio # jsonschema @@ -91,7 +97,13 @@ importlib-resources==7.1.0 # via swagger-spec-validator ipykernel==7.3.0 # via -r MPContribs/mpcontribs-kernel-gateway/requirements.in +<<<<<<< HEAD ipython==9.14.1 +||||||| parent of 5ef6d1d6 (upgrade dependencies for deployment) +ipython==9.13.0 +======= +ipython==9.14.0 +>>>>>>> 5ef6d1d6 (upgrade dependencies for deployment) # via # ipykernel # ipywidgets @@ -190,7 +202,13 @@ msgpack==1.2.1 # via # bravado # bravado-core +<<<<<<< HEAD narwhals==2.22.1 +||||||| parent of 5ef6d1d6 (upgrade dependencies for deployment) +narwhals==2.21.2 +======= +narwhals==2.22.0 +>>>>>>> 5ef6d1d6 (upgrade dependencies for deployment) # via plotly nbclient==0.11.0 # via nbconvert @@ -383,7 +401,13 @@ terminado==0.18.1 # jupyter-server-terminals tinycss2==1.5.1 # via bleach +<<<<<<< HEAD tornado==6.5.7 +||||||| parent of 5ef6d1d6 (upgrade dependencies for deployment) +tornado==6.5.5 +======= +tornado==6.5.6 +>>>>>>> 5ef6d1d6 (upgrade dependencies for deployment) # via # ipykernel # jupyter-client From 70d795054b6ed5707b006b5e43f2c15715256879 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 17 Jun 2026 15:05:22 -0700 Subject: [PATCH 04/62] Revert changes to deployment of other submodules --- mpcontribs-client/requirements/deployment.txt | 18 -------------- .../requirements/deployment.txt | 24 ------------------- 2 files changed, 42 deletions(-) diff --git a/mpcontribs-client/requirements/deployment.txt b/mpcontribs-client/requirements/deployment.txt index 231613f00b..f19c9e1489 100644 --- a/mpcontribs-client/requirements/deployment.txt +++ b/mpcontribs-client/requirements/deployment.txt @@ -50,25 +50,13 @@ fonttools==4.63.0 # via matplotlib fqdn==1.5.1 # via jsonschema -<<<<<<< HEAD idna==3.18 -||||||| parent of 5ef6d1d6 (upgrade dependencies for deployment) -idna==3.16 -======= -idna==3.17 ->>>>>>> 5ef6d1d6 (upgrade dependencies for deployment) # via # jsonschema # requests importlib-resources==7.1.0 # via swagger-spec-validator -<<<<<<< HEAD ipython==9.14.1 -||||||| parent of 5ef6d1d6 (upgrade dependencies for deployment) -ipython==9.13.0 -======= -ipython==9.14.0 ->>>>>>> 5ef6d1d6 (upgrade dependencies for deployment) # via mpcontribs-client (MPContribs/mpcontribs-client/pyproject.toml) ipython-pygments-lexers==1.1.1 # via ipython @@ -112,13 +100,7 @@ msgpack==1.2.1 # via # bravado # bravado-core -<<<<<<< HEAD narwhals==2.22.1 -||||||| parent of 5ef6d1d6 (upgrade dependencies for deployment) -narwhals==2.21.2 -======= -narwhals==2.22.0 ->>>>>>> 5ef6d1d6 (upgrade dependencies for deployment) # via plotly networkx==3.6.1 # via pymatgen-core diff --git a/mpcontribs-kernel-gateway/requirements/deployment.txt b/mpcontribs-kernel-gateway/requirements/deployment.txt index e3a21ebecc..827d405953 100644 --- a/mpcontribs-kernel-gateway/requirements/deployment.txt +++ b/mpcontribs-kernel-gateway/requirements/deployment.txt @@ -82,13 +82,7 @@ fonttools==4.63.0 # via matplotlib fqdn==1.5.1 # via jsonschema -<<<<<<< HEAD idna==3.18 -||||||| parent of 5ef6d1d6 (upgrade dependencies for deployment) -idna==3.16 -======= -idna==3.17 ->>>>>>> 5ef6d1d6 (upgrade dependencies for deployment) # via # anyio # jsonschema @@ -97,13 +91,7 @@ importlib-resources==7.1.0 # via swagger-spec-validator ipykernel==7.3.0 # via -r MPContribs/mpcontribs-kernel-gateway/requirements.in -<<<<<<< HEAD ipython==9.14.1 -||||||| parent of 5ef6d1d6 (upgrade dependencies for deployment) -ipython==9.13.0 -======= -ipython==9.14.0 ->>>>>>> 5ef6d1d6 (upgrade dependencies for deployment) # via # ipykernel # ipywidgets @@ -202,13 +190,7 @@ msgpack==1.2.1 # via # bravado # bravado-core -<<<<<<< HEAD narwhals==2.22.1 -||||||| parent of 5ef6d1d6 (upgrade dependencies for deployment) -narwhals==2.21.2 -======= -narwhals==2.22.0 ->>>>>>> 5ef6d1d6 (upgrade dependencies for deployment) # via plotly nbclient==0.11.0 # via nbconvert @@ -401,13 +383,7 @@ terminado==0.18.1 # jupyter-server-terminals tinycss2==1.5.1 # via bleach -<<<<<<< HEAD tornado==6.5.7 -||||||| parent of 5ef6d1d6 (upgrade dependencies for deployment) -tornado==6.5.5 -======= -tornado==6.5.6 ->>>>>>> 5ef6d1d6 (upgrade dependencies for deployment) # via # ipykernel # jupyter-client From 5860ae0d99eb5badc81419b3c95fa43e373d11d8 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 24 Jun 2026 15:42:14 -0700 Subject: [PATCH 05/62] Added Project.category to group together logically connected projects --- .../src/mpcontribs_api/domains/projects/models.py | 12 ++++++++++-- .../mpcontribs_api/domains/projects/repository.py | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py index 7216c82acc..728c1275b4 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py @@ -51,6 +51,7 @@ class Project(BaseDocumentWithInput[ShortStr]): stats: Stats # Optional + category: ShortStr | None = None references: list[Reference] = Field(default_factory=list) long_title: str | None = None other: dict[str, Any] = Field(default_factory=dict) @@ -84,6 +85,7 @@ class ProjectOut(DocumentOut[ShortStr]): authors: str | None = None description: str | None = None title: ShortStr | None = None + category: ShortStr | None = None owner: PrefixedEmail | None = None other: dict[str, Any] | None = None is_public: bool | None = None @@ -117,6 +119,11 @@ class ProjectFilter(BaseFilter): owner__neq: PrefixedEmail | None = None owner__ilike: str | None = None + category: ShortStr | None = None + category__in: list[ShortStr] | None = None + category__neq: ShortStr | None = None + category__ilike: str | None = None + # fuzzy only long_title__ilike: str | None = None @@ -147,12 +154,13 @@ class ProjectPatch(BaseModel): title: ShortStr | None = None authors: str | None = None description: str | None = None + category: ShortStr | None = None owner: PrefixedEmail | None = None unique_identifiers: bool | None = None references: list[Reference] = Field(default_factory=list) long_title: str | None = None other: dict[str, Any] = Field(default_factory=dict) columns: list[Column] = Field(default_factory=list) - is_public: bool = False - is_approved: bool = False + is_public: bool | None = None + is_approved: bool | None = None license: Literal["CCA4", "CCPD"] | None = None diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py index 0e6f96a96c..000be07dca 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py @@ -54,7 +54,7 @@ async def get_projects( """Query the Project collection, scoped to the current user. See ``get_many``.""" return await self.get_many(pagination=pagination, filter=filter, fields=fields) - async def get_project_by_id(self, id: str, fields: frozenset[str] | None): + async def get_project_by_id(self, id: str, fields: frozenset[str] | None) -> Project | ProjectOut | None: """Find a single project by id, scoped to the current user. See ``get_by_id``.""" return await self.get_by_id(id, fields) From e64f2f2683f9da2209657b6de51596e11419620b Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 15 Jul 2026 09:03:37 -0700 Subject: [PATCH 06/62] Removed stale 'old' directory --- mpcontribs-api/src/mpcontribs_api/old/core.py | 647 ------------------ 1 file changed, 647 deletions(-) delete mode 100644 mpcontribs-api/src/mpcontribs_api/old/core.py diff --git a/mpcontribs-api/src/mpcontribs_api/old/core.py b/mpcontribs-api/src/mpcontribs_api/old/core.py deleted file mode 100644 index a4a600db19..0000000000 --- a/mpcontribs-api/src/mpcontribs_api/old/core.py +++ /dev/null @@ -1,647 +0,0 @@ -import os -import yaml - -from copy import deepcopy -from re import Pattern -from importlib import import_module -from flasgger.marshmallow_apispec import SwaggerView as OriginalSwaggerView -from flasgger.marshmallow_apispec import schema2jsonschema -from marshmallow_mongoengine import ModelSchema -from flask_mongorest.views import ResourceView -from mongoengine.queryset import DoesNotExist -from mongoengine.queryset.visitor import Q -from werkzeug.exceptions import Unauthorized -from mpcontribs.api.config import DOC_DIR -from mpcontribs.api import is_gunicorn, get_logger - -logger = get_logger(__name__) - - -def get_limit_params(resource, method): - default = resource.default_limit - bulk = {"BulkUpdate", "BulkDelete"} - maximum = resource.bulk_update_limit if method in bulk else resource.max_limit - return [ - { - "name": "_skip", - "in": "query", - "type": "integer", - "description": "number of items to skip", - }, - { - "name": "_limit", - "in": "query", - "type": "integer", - "default": default, - "maximum": maximum, - "description": "maximum number of items to return", - }, - { - "name": "page", - "in": "query", - "type": "integer", - "description": "page number to return (in batches of `per_page/_limit`; alternative to `_skip`)", - }, - { - "name": "per_page", - "in": "query", - "type": "integer", - "default": default, - "maximum": maximum, - "description": "maximum number of items to return per page (same as `_limit`)", - }, - ] - - -def get_filter_params(name, filters): - filter_params = [] - is_pattern = isinstance(name, Pattern) - label = name.pattern if is_pattern else name - for op in filters: - if op.op == "exact" and not is_pattern: - name = label - description = f"filter {label}" - else: - suffix = op.suf if hasattr(op, "suf") else op.op - name = f"{label}__{suffix}" - description = f"filter {label} via ${op.op}" - - filter_params.append( - { - "name": name, - "in": "query", - "type": op.typ, - "description": description, - } - ) - if op.typ == "array": - filter_params[-1]["items"] = {"type": "string"} - if hasattr(op, "fmt"): - filter_params[-1]["format"] = op.fmt - - if op.allow_negation: - suffix = "not__" - suffix += op.suf if hasattr(op, "suf") else op.op - name = f"{label}__{suffix}" - description = f"filter {label} via ${op.op}" - param = deepcopy(filter_params[-1]) - param["name"] = name - param["description"] = description - filter_params.append(param) - - return filter_params - - -def get_specs(klass, method, collection): - method_name = method.__name__ if hasattr(method, "__name__") else method - default_response = { - "description": "Error", - "schema": {"type": "object", "properties": {"error": {"type": "string"}}}, - } - id_field = klass.resource.document._meta["id_field"].capitalize() - doc_name = collection[:-1].capitalize() - fields_param = None - if klass.resource.fields is not None: - fields_avail = ( - klass.resource.fields + klass.resource.get_optional_fields() + ["_all"] - ) - description = f"List of fields to include in response ({fields_avail})." - description += " Use dot-notation for nested subfields." - fields_param = { - "name": "_fields", - "in": "query", - "default": klass.resource.fields, - "type": "array", - "items": {"type": "string"}, - "description": description, - } - - field_pagination_params = [] - for field, limits in klass.resource.fields_to_paginate.items(): - field_pagination_params.append( - { - "name": f"{field}_page", - "in": "query", - "default": 1, - "type": "integer", - "description": f"page to retrieve for {field} field", - } - ) - field_pagination_params.append( - { - "name": f"{field}_per_page", - "in": "query", - "default": limits[0], - "maximum": limits[1], - "type": "integer", - "description": f"number of items to retrieve per page for {field} field", - } - ) - - filter_params = [] - if hasattr(klass.resource, "filters"): - for k, v in klass.resource.filters.items(): - filter_params += get_filter_params(k, v) - - order_params = [] - if klass.resource.allowed_ordering: - allowed_ordering = [ - o.pattern if isinstance(o, Pattern) else o - for o in klass.resource.allowed_ordering - ] - order_params = [ - { - "name": "_sort", - "in": "query", - "type": "string", - "description": f"sort {collection} via {allowed_ordering}. Prepend +/- for asc/desc.", - } - ] - - spec = None - if method_name == "Fetch": - params = [ - { - "name": "pk", - "in": "path", - "type": "string", - "required": True, - "description": f"{collection[:-1]} (primary key)", - } - ] - if fields_param is not None: - params.append(fields_param) - params += field_pagination_params - spec = { - "summary": f"Retrieve a {collection[:-1]}.", - "operationId": f"get{doc_name}By{id_field}", - "parameters": params, - "responses": { - 200: { - "description": f"single {collection} entry", - "schema": {"$ref": f"#/definitions/{klass.schema_name}"}, - }, - "default": default_response, - }, - } - - elif method_name == "BulkFetch": - params = [fields_param] if fields_param is not None else [] - params += field_pagination_params - params += order_params - params += filter_params - schema_props = { - "data": { - "type": "array", - "items": {"$ref": f"#/definitions/{klass.schema_name}"}, - } - } - if klass.resource.paginate: - schema_props["has_more"] = {"type": "boolean"} - schema_props["total_count"] = {"type": "integer"} - schema_props["total_pages"] = {"type": "integer"} - params += get_limit_params(klass.resource, method_name) - spec = { - "summary": f"Filter and retrieve {collection}.", - "operationId": f"query{doc_name}s", - "parameters": params, - "responses": { - 200: { - "description": f"list of {collection}", - "schema": {"type": "object", "properties": schema_props}, - }, - "default": default_response, - }, - } - - elif method_name == "Download": - params = [ - { - "name": "short_mime", - "in": "path", - "type": "string", - "required": True, - "description": "MIME Download Type: gz", - "default": "gz", - }, - { - "name": "format", - "in": "query", - "type": "string", - "required": True, - "description": f"download {collection} in different formats: {klass.resource.download_formats}", - }, - ] - params += [fields_param] if fields_param is not None else [] - params += order_params - params += filter_params - if klass.resource.paginate: - params += get_limit_params(klass.resource, method_name) - spec = { - "summary": f"Filter and download {collection}.", - "operationId": f"download{doc_name}s", - "parameters": params, - "produces": ["application/gzip"], - "responses": { - 200: { - "description": f"{collection} download", - "schema": {"type": "file"}, - }, - "default": default_response, - }, - } - - elif method_name == "Create": - spec = { - "summary": f"Create a new {collection[:-1]}.", - "operationId": f"create{doc_name}", - "parameters": [ - { - "name": f"{collection[:-1]}", - "in": "body", - "description": f"The object to use for {collection[:-1]} creation", - "schema": {"$ref": f"#/definitions/{klass.schema_name}"}, - } - ], - "responses": { - 200: { - "description": f"{collection[:-1]} created", - "schema": {"$ref": f"#/definitions/{klass.schema_name}"}, - }, - "default": default_response, - }, - } - - elif method_name == "BulkCreate": - spec = { - "summary": f"Create new {collection[:-1]}(s).", - "operationId": f"create{doc_name}s", - "parameters": [ - { - "name": f"{collection}", - "in": "body", - "description": f"The objects to use for {collection[:-1]} creation", - "schema": { - "type": "array", - "items": {"$ref": f"#/definitions/{klass.schema_name}"}, - }, - } - ], - "responses": { - 200: { - "description": f"{collection} created", - "schema": { - "type": "object", - "properties": { - "count": {"type": "integer"}, - "data": { - "type": "array", - "items": {"$ref": f"#/definitions/{klass.schema_name}"}, - }, - }, - }, - }, - "default": default_response, - }, - } - - elif method_name == "Update": - spec = { - "summary": f"Update a {collection[:-1]}.", - "operationId": f"update{doc_name}By{id_field}", - "parameters": [ - { - "name": "pk", - "in": "path", - "type": "string", - "required": True, - "description": f"The {collection[:-1]} (primary key) to update", - }, - { - "name": f"{collection[:-1]}", - "in": "body", - "description": f"The object to use for {collection[:-1]} update", - "schema": {"type": "object"}, - }, - ], - "responses": { - 200: { - "description": f"{collection[:-1]} updated", - "schema": {"$ref": f"#/definitions/{klass.schema_name}"}, - }, - "default": default_response, - }, - } - elif method_name == "BulkUpdate": - params = filter_params - params.append( - { - "name": f"{collection}", - "in": "body", - "description": f"The object to use for {collection} bulk update", - "schema": {"type": "object"}, - } - ) - schema_props = {"count": {"type": "integer"}} - if klass.resource.paginate: - schema_props["has_more"] = {"type": "boolean"} - schema_props["total_count"] = {"type": "integer"} - schema_props["total_pages"] = {"type": "integer"} - params += get_limit_params(klass.resource, method_name) - spec = { - "summary": f"Filter and update {collection}.", - "operationId": f"update{doc_name}s", - "parameters": params, - "responses": { - 200: { - "description": f"Number of {collection} updated", - "schema": {"type": "object", "properties": schema_props}, - }, - "default": default_response, - }, - } - - elif method_name == "BulkDelete": - params = filter_params - schema_props = {"count": {"type": "integer"}} - if klass.resource.paginate: - schema_props["has_more"] = {"type": "boolean"} - schema_props["total_count"] = {"type": "integer"} - schema_props["total_pages"] = {"type": "integer"} - params += get_limit_params(klass.resource, method_name) - spec = { - "summary": f"Filter and delete {collection}.", - "operationId": f"delete{doc_name}s", - "parameters": params, - "responses": { - 200: { - "description": f"Number of {collection} deleted", - "schema": {"type": "object", "properties": schema_props}, - }, - "default": default_response, - }, - } - - elif method_name == "Delete": - spec = { - "summary": f"Delete a {collection[:-1]}.", - "operationId": f"delete{doc_name}By{id_field}", - "parameters": [ - { - "name": "pk", - "in": "path", - "type": "string", - "required": True, - "description": f"The {collection[:-1]} (primary key) to delete", - } - ], - "responses": { - 200: {"description": f"{collection[:-1]} deleted"}, - "default": default_response, - }, - } - - return spec - - -class SwaggerView(OriginalSwaggerView, ResourceView): - """A class-based view defining additional methods""" - - def __init_subclass__(cls, **kwargs): - """initialize Schema, decorators, definitions, and tags""" - super().__init_subclass__(**kwargs) - - if not __name__ == cls.__module__: - # e.g.: cls.__module__ = mpcontribs.api.projects.views - views_path = cls.__module__.split(".") - doc_path = ".".join(views_path[:-1] + ["document"]) - cls.tags = [views_path[-2]] - doc_filepath = doc_path.replace(".", os.sep) + ".py" - if os.path.exists(doc_filepath): - cls.doc_name = cls.tags[0].capitalize() - Model = getattr(import_module(doc_path), cls.doc_name) - cls.schema_name = cls.doc_name + "Schema" - cls.Schema = type( - cls.schema_name, - (ModelSchema, object), - { - "Meta": type( - "Meta", - (object,), - dict(model=Model, ordered=True, model_build_obj=False), - ) - }, - ) - cls.definitions = {cls.schema_name: schema2jsonschema(cls.Schema)} - cls.resource.schema = cls.Schema - - # write flask-mongorest swagger specs - for method in cls.methods: - spec = get_specs(cls, method, cls.tags[0]) - if spec: - dir_path = os.path.join(DOC_DIR, cls.tags[0]) - file_path = os.path.join(dir_path, method.__name__ + ".yml") - if not os.path.exists(file_path): - os.makedirs(dir_path, exist_ok=True) - - if is_gunicorn: - with open(file_path, "w") as f: - yaml.dump(spec, f) - logger.debug( - f"{cls.tags[0]}.{method.__name__} written to {file_path}" - ) - - def get_groups(self, request): - groups = request.headers.get("X-Authenticated-Groups", "").split(",") - groups += request.headers.get("X-Consumer-Groups", "").split(",") - return set(grp.strip() for grp in groups if grp) - - def is_anonymous(self, request): - if not request.headers.get("X-Consumer-Username", ""): - return True - - is_anonymous = request.headers.get("X-Anonymous-Consumer", False) - if isinstance(is_anonymous, str): - is_anonymous = False if is_anonymous == "false" else True - - return is_anonymous - - def is_external(self, request): - return request.headers.get( - "X-Forwarded-Host" - ) is not None and not request.headers.get("Origin") - - def is_admin(self, request): - groups = self.get_groups(request) - admin_group = os.environ.get("ADMIN_GROUP", "admin") - return admin_group in groups - - def is_project_user(self, request, obj): - if hasattr(obj, "owner"): - owner = obj.owner - project = obj.name - elif hasattr(obj, "project"): - owner = obj.project.owner - project = obj.project.name - else: - raise Unauthorized(f"Unable to authorize {obj}") - - groups = self.get_groups(request) - username = request.headers.get("X-Consumer-Username") - return project in groups or owner == username - - def is_admin_or_project_user(self, request, obj): - if self.is_anonymous(request): - return False - - if self.is_admin(request): - return True - - return self.is_project_user(request, obj) - - def get_projects(self): - # project is LazyReferenceFields (multiple queries) - module = import_module("mpcontribs.api.projects.document") - Projects = getattr(module, "Projects") - exclude = list(Projects._fields.keys()) - only = ["name", "owner", "is_public", "is_approved"] - return Projects.objects.exclude(*exclude).only(*only) - - def get_projects_filter(self, username, groups, filter_names=None): - projects = self.get_projects() - if filter_names: - projects = projects.filter(name__in=filter_names) - - q = {"private": [], "public": []} - - for project in projects: - if project.owner == username or project.name in groups: - q["private"].append(project.name) - elif project.is_public and project.is_approved: - q["public"].append(project.name) - - # reduced query - qfilter = Q() - if q["private"]: - qfilter |= Q(project__in=q["private"]) - if q["public"]: - qfilter |= Q(project__in=q["public"], is_public=True) - - return qfilter - - def has_read_permission(self, request, qs): - if self.is_admin(request): - return qs # admins can read all entries - - groups = self.get_groups(request) - is_anonymous = self.is_anonymous(request) - is_external = self.is_external(request) - username = request.headers.get("X-Consumer-Username") - approved_public_filter = Q(is_public=True, is_approved=True) - - if request.path.startswith("/projects/"): - # external or internal requests can both read full project info - # anonymous requests can only read public approved projects - if is_anonymous: - return qs.filter(approved_public_filter) - - # authenticated requests can read approved public or accessible non-public projects - qfilter = approved_public_filter | Q(owner=username) - if groups: - qfilter |= Q(name__in=list(groups)) - - return qs.filter(qfilter) - else: - # contributions are set private/public independent from projects - # anonymous requests: - # - external: only meta-data of public contributions in approved public projects - # - internal: full public contributions in approved public projects - # authenticated requests: - # - private contributions in a public project are only accessible to owner/group - # - any contributions in a private project are only accessible to owner/group - component = request.path.split("/")[1] - - if component == "contributions": - q = qs._query - if is_anonymous and is_external: - qs = qs.exclude("data") - - if q and "project" in q and isinstance(q["project"], str): - projects = self.get_projects() - try: - project = projects.get(name=q["project"]) - except DoesNotExist: - return qs.none() - - if project.owner == username or project.name in groups: - return qs - elif project.is_public and project.is_approved: - return qs.filter(is_public=True) - else: - return qs.none() - else: - names = None - if q and "project" in q and "$in" in q["project"]: - names = q.pop("project").pop("$in") - - qfilter = self.get_projects_filter( - username, groups, filter_names=names - ) - return qs.filter(qfilter) - else: - # get component Object IDs for queryset - pk = request.view_args.get("pk") - from mpcontribs.api.contributions.document import get_resource - - resource = get_resource(component) - - def qfilter(qs): - return qs.clone() - - if pk: - ids = [resource.get_object(pk, qfilter=qfilter).id] - else: - ids = [o.id for o in resource.get_objects(qfilter=qfilter)[0]] - - if not ids: - return qs.none() - - # get list of readable contributions and their component Object IDs - module = import_module("mpcontribs.api.contributions.document") - Contributions = getattr(module, "Contributions") - qfilter = self.get_projects_filter(username, groups) - component = component[:-1] if component == "notebooks" else component - qfilter &= Q(**{f"{component}__in": ids}) - contribs = ( - Contributions.objects(qfilter).only(component).limit(len(ids)) - ) - # return new queryset using "ids__in" - readable_ids = ( - [getattr(contrib, component).id for contrib in contribs] - if component == "notebook" - else [ - dbref.id - for contrib in contribs - for dbref in getattr(contrib, component) - if dbref.id in ids - ] - ) - if not readable_ids: - return qs.none() - - qs._query_obj = Q(id__in=readable_ids) - # exclude optional fields if anonymous external request - if is_anonymous and is_external: - exclude = resource.get_optional_fields() - qs = qs.exclude(*exclude) - - return qs - - def has_add_permission(self, request, obj): - return self.is_admin_or_project_user(request, obj) - - def has_change_permission(self, request, obj): - return self.is_admin_or_project_user(request, obj) - - def has_delete_permission(self, request, obj): - return self.is_admin_or_project_user(request, obj) From da7af78cf676c7708b50efd9c98bdd74adbbe6ca Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 15 Jul 2026 09:18:21 -0700 Subject: [PATCH 07/62] Added custom str types to help with str validation (originally from contrib-data-handling branch --- .../mpcontribs_api/domains/_shared/types.py | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/_shared/types.py b/mpcontribs-api/src/mpcontribs_api/domains/_shared/types.py index bfaf0e4a1f..68061141a7 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/_shared/types.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/_shared/types.py @@ -1,4 +1,5 @@ import re +import unicodedata from enum import StrEnum from typing import Annotated @@ -108,3 +109,91 @@ def _serialize_frame(data: pl.DataFrame) -> dict: ), WithJsonSchema({"type": "object"}, mode="serialization"), ] + + +def _nfkc_casefold(value: str) -> str: + """NFKC + casefold: the case-insensitive, compatibility-folded form used for search/matching. + + Surrounding whitespace is stripped by :func:`nfkc_normalize` before casefolding. + """ + return nfkc_normalize(value).casefold() + + +def nfkc_normalize(value: str) -> str: + """Return ``value`` in Unicode NFKC (compatibility composition) form, preserving case. + + NFKC folds *compatibility* variants onto a canonical form — the MICRO SIGN U+00B5 becomes the + Greek mu, the ``fi`` ligature becomes ``fi``, full-width characters become half-width, and so on. + Unlike :func:`_nfkc_casefold` it does not casefold, so human-facing labels keep their original + case. It is a superset of :func:`nfc_normalize` (NFKC output is already NFC-stable). + + Leading/trailing whitespace is stripped (NFKC first, so compatibility whitespace such as the + NBSP U+00A0 folds to a plain space and is then trimmed) so ``" Foo "`` and ``"Foo"`` collapse to + the same stored form. + """ + return unicodedata.normalize("NFKC", value).strip() + + +def nfc_normalize(value: str) -> str: + """Return ``value`` in Unicode NFC (canonical composition) form. + + NFC folds canonically-equivalent codepoints onto one representative — e.g. the OHM SIGN + (U+2126) and Ångström sign (U+212B) collapse onto the Greek capital omega and ``Å``. This keeps + equivalent spellings of units, labels, and query terms comparable byte-for-byte. It is a no-op on + pure ASCII. NFC is deliberately *not* NFKC: it does not casefold or apply compatibility folding + (so the MICRO SIGN U+00B5 and Greek mu U+03BC stay distinct). + + Leading/trailing whitespace is stripped so equivalent spellings compare byte-for-byte. Note NFC + (unlike NFKC) does not fold compatibility whitespace, but :meth:`str.strip` trims all Unicode + whitespace regardless, so an NBSP-padded value is still trimmed. + """ + return unicodedata.normalize("NFC", value).strip() + + +# Acronym boundary: an uppercase letter followed by an uppercase-then-lowercase +# pair. The trailing capital begins a new word, so ``HTTPResponse`` splits as +# ``HTTP|Response`` +_ACRONYM_BOUNDARY = re.compile(r"(?<=[A-Z])(?=[A-Z][a-z])") + +# camelCase/PascalCase boundary: a lowercase letter or digit immediately followed +# by an uppercase letter (``bandGap`` -> ``band|Gap``). The ``0-9`` in the lookbehind +# also splits ``digit->UPPER`` (``Al2O3`` -> ``al2_o3``) +_CAMEL_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])") + +# Any run of characters that isn't an ASCII letter or digit collapses to a single +# underscore (spaces, hyphens, punctuation, etc.). +_NON_ALNUM_RUN = re.compile(r"[^a-zA-Z0-9]+") + +_SPECIAL_TERMS = { + "pH": "ph", +} +_SPECIAL_RE = re.compile("|".join(re.escape(k) for k in _SPECIAL_TERMS)) if _SPECIAL_TERMS else None + + +def to_snake_case(name: str) -> str: + """Coerce a single key token to canonical ``snake_case``. + + Rewrites known irregular terms, splits ``camelCase``/``PascalCase`` and + acronym boundaries, lowercases, and collapses every run of non-alphanumeric + characters to a single underscore, trimming leading/trailing underscores. + """ + s = name + if _SPECIAL_RE is not None: + s = _SPECIAL_RE.sub(lambda m: _SPECIAL_TERMS[m.group()], s) + s = _ACRONYM_BOUNDARY.sub("_", s) + s = _CAMEL_BOUNDARY.sub("_", s) + s = _NON_ALNUM_RUN.sub("_", s) + return s.strip("_").lower() + + +# Converts strs to snake case +SnakeCaseStr = Annotated[str, BeforeValidator(func=to_snake_case)] + +# Converts strs to searchable form (NFKC compatibility fold + casefold) +SearchStr = Annotated[str, BeforeValidator(func=_nfkc_casefold)] + +# NFKC-normalizes strs (compatibility fold, case preserved) — for human-facing labels/names +NFKCStr = Annotated[str, BeforeValidator(func=nfkc_normalize)] + +# Converts strs to pretty display form (keeps unicode and most formatting) +DisplayStr = Annotated[str, BeforeValidator(func=nfc_normalize)] From d47cb79133a2e10684bcef88d323cff37db945d4 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 15 Jul 2026 15:31:12 -0700 Subject: [PATCH 08/62] Generalized _shared/repo methods over model-defined identifiers --- .../mpcontribs_api/domains/_shared/models.py | 16 +++ .../domains/_shared/repository.py | 125 +++++++++++++++++- .../domains/contributions/models.py | 11 +- 3 files changed, 146 insertions(+), 6 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/_shared/models.py b/mpcontribs-api/src/mpcontribs_api/domains/_shared/models.py index 2dfb073c66..d055d27a3c 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/_shared/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/_shared/models.py @@ -27,6 +27,22 @@ class BaseDocumentWithInput[TId](Document): # Required, non-null, resource-specific id. Overrides Document's optional ``PydanticObjectId`` id. id: TId = Field(alias="_id") # pyright: ignore[reportGeneralTypeIssues, reportIncompatibleVariableOverride] + @classmethod + def identifier_fields(cls) -> frozenset[str]: + """Field names that uniquely identify a document in this collection. + + This is the natural/unique key a caller can supply without first knowing the Mongo ``_id`` + (e.g. ``{"name", "owner"}`` for a project group). The repository pairs these names with + caller-supplied values to locate a single resource, and rejects any value dict whose keys + don't match this set. Defaults to the primary key; subclasses with a meaningful compound key + override it. + """ + return frozenset({"id"}) + + def identifiers(self) -> dict[str, Any]: + """This document's identifier field values, keyed by :meth:`identifier_fields`.""" + return {field: getattr(self, field) for field in self.identifier_fields()} + @classmethod def from_input_model(cls, data: Any) -> Self: """Translate a validated input payload into a full stored document.""" diff --git a/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py index 66c940d3bc..76a7e8bf45 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py @@ -105,6 +105,62 @@ async def get_many( next_cursor = encode_cursor(str(items[-1].id)) if has_more and items else None return Page(items=items, next_cursor=next_cursor) + def _identifier_query(self, identifiers: dict[str, Any]) -> dict[str, Any]: + """Turn a ``{field: value}`` identifier dict into a scoped Mongo query fragment. + + The keys must be exactly the model's :meth:`identifier_fields` + ``id`` is remapped Mongo's ``_id`` (mirroring ``BaseFilter._get_filter_conditions``) + since a raw dict query does not go through Beanie's alias resolution. + + Args: + identifiers (dict[str, Any]): identifier field values keyed by ``identifier_fields`` + """ + expected = self.document_model.identifier_fields() + if identifiers.keys() != expected: + raise ValidationError( + "identifiers must match the model's identifier fields exactly", + expected=sorted(expected), + received=sorted(identifiers.keys()), + ) + return {("_id" if key == "id" else key): value for key, value in identifiers.items()} + + async def _resolve_one_id(self, identifiers: dict[str, Any], session: AsyncClientSession | None = None) -> Any: + """Resolve the single scoped ``_id`` matching ``identifiers``, or ``None`` if absent. + + Enforces uniqueness: the identifier fields are meant to key at most one document, so if two + are found (a duplicate under a supposedly-unique key) this raises ``ConflictError`` rather + than silently picking one. + """ + query = self._identifier_query(identifiers) + projection = self.out_model.projection(frozenset({"id"})) + docs = ( + await self.document_model.find(self._scope, query, session=session).limit(2).project(projection).to_list() + ) # pyright: ignore[reportArgumentType] + if len(docs) > 1: + raise ConflictError("identifiers matched more than one document", identifiers=identifiers) + return docs[0].id if docs else None + + async def get_one( + self, + identifiers: dict[str, Any], + fields: frozenset[str] | None = None, + ) -> TOut | None: + """Return the single scoped document matching ``identifiers``, projected to ``fields``. + + Returns ``None`` when nothing matches, but ``ConflictError`` if the identifiers match + more than one document. + + Args: + identifiers (dict[str, Any]): identifier field values keyed by ``identifier_fields`` + fields (frozenset[str] | None): fields to project; if None the full document is returned + """ + query = self._identifier_query(identifiers) + projection = self.out_model.projection(fields) + docs = await self.document_model.find(self._scope, query).limit(2).project(projection).to_list() # pyright: ignore[reportArgumentType] + if len(docs) > 1: + raise ConflictError("identifiers matched more than one document", identifiers=identifiers) + return docs[0] if docs else None + async def get_by_id(self, id: Any, fields: frozenset[str] | None = None) -> TDoc | TOut | None: """Return a single scoped document by id, projected to the requested fields. @@ -146,17 +202,55 @@ async def insert_one(self, in_resource: TIn) -> TDoc: await document.insert() return document + async def delete(self, filter: TFilter, session: AsyncClientSession | None = None) -> DeleteResponse: + """Delete every scoped document matching an arbitrary ``filter``. + + This is the bulk path (e.g. "delete every ProjectGroup with owner == X"). It does not raise + on an empty match — a zero count is a valid, unambiguous outcome for a filter delete. Scoping + ensures callers cannot delete documents they are not permitted to see. + + Args: + filter (TFilter): the fastapi-filter query to apply on top of the user scope + session (AsyncClientSession | None): optional client session for transactions + """ + query = filter.filter(self.document_model.find(self._scope, session=session)) + result = await query.delete_many(session=session) + if result is None: + raise ValidationError("DeleteResult not returned internally") + return DeleteResponse.from_delete_result(result) + + async def delete_one( + self, identifiers: dict[str, Any], session: AsyncClientSession | None = None + ) -> DeleteResponse: + """Delete the single scoped document matching ``identifiers``. + + Uniqueness is checked before anything is deleted (see :meth:`_resolve_one_id`), so a + duplicate raises ``ConflictError`` and an absent resource raises ``NotFoundError`` — this + never deletes more than the one intended document. + + Args: + identifiers (dict[str, Any]): identifier field values keyed by ``identifier_fields`` + session (AsyncClientSession | None): optional client session for transactions + """ + oid = await self._resolve_one_id(identifiers, session=session) + if oid is None: + raise NotFoundError(f"{self.document_model.__name__} not found", identifiers=identifiers) + return await self.delete_by_id(oid, session=session) + async def delete_by_id(self, id: Any, session: AsyncClientSession | None = None) -> DeleteResponse: - """Delete a single scoped document by id. + """Delete a single scoped document by its primary key (``_id``). - Scoping ensures callers cannot delete documents they are not permitted to see. + Scoping ensures callers cannot delete documents they are not permitted to see; an id that is + absent or out of scope raises ``NotFoundError``. Kept distinct from :meth:`delete_one`, whose + key is the semantic ``identifier_fields`` (which differs from ``_id`` for some resources). Args: - id (str): the id of the document to delete + id (Any): the primary key of the document to delete + session (AsyncClientSession | None): optional client session for transactions """ doc = await self.document_model.find_one(self._scope, self.document_model.id == id, session=session) - if not doc: - raise NotFoundError("Document with id not found", id=id) + if doc is None: + raise NotFoundError(self._not_found(id)) await doc.delete(session=session) return DeleteResponse(num_deleted=1) @@ -211,6 +305,27 @@ async def patch(self, id: Any, update: TPatch) -> TDoc: raise NotFoundError(self._not_found(id)) return updated + async def patch_one( + self, + identifiers: dict[str, Any], + update: TPatch, + session: AsyncClientSession | None = None, + ) -> TDoc: + """Partially update the single scoped document matching ``identifiers``. + + Resolves the target by its unique identifier fields (raising ``ConflictError`` on a + duplicate, ``NotFoundError`` when absent) and then applies the patch via :meth:`patch`. + + Args: + identifiers (dict[str, Any]): identifier field values keyed by ``identifier_fields`` + update (TPatch): the partial update to apply; unset fields are dropped + session (AsyncClientSession | None): optional client session for transactions + """ + oid = await self._resolve_one_id(identifiers, session=session) + if oid is None: + raise NotFoundError(f"{self.document_model.__name__} not found", identifiers=identifiers) + return await self.patch(oid, update) + def _hash_payload(self, payload: dict[str, Any], *, separators: tuple[str, str] = (",", ":")) -> str: canonical = json.dumps( payload, diff --git a/mpcontribs-api/src/mpcontribs_api/domains/contributions/models.py b/mpcontribs-api/src/mpcontribs_api/domains/contributions/models.py index 8c4ceadfe1..7b70d10a77 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/contributions/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/contributions/models.py @@ -106,6 +106,11 @@ class Settings: IndexModel(keys=[("attachments.$id", ASCENDING)], name="ref_attachments"), ] + @classmethod + def identifier_fields(cls) -> frozenset[str]: + """A contribution is uniquely identified (within a version) by ``project`` + ``identifier``.""" + return frozenset({"project", "identifier"}) + class Contribution(ContributionBase): """Models what is actually stored in the database.""" @@ -160,7 +165,11 @@ def component_count(self) -> int: return len(self.structures or []) + len(self.tables or []) + len(self.attachments or []) def identifiers(self) -> dict[str, str]: - """Returns a dict of unique identifiers for a contribution (outside of id).""" + """Returns this contribution's identifier values (see ``identifier_fields``). + + Overrides the base to narrow the value type to ``str`` for the callers (bulk error + reporting, ``upsert_contribution_by_identifiers``) that key on it. + """ return {"project": self.project, "identifier": self.identifier} From b136c412e58a5a25f378375d159702d985faccaf Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 15 Jul 2026 15:32:05 -0700 Subject: [PATCH 09/62] Created ProjectGroups to model collections of projects --- .../src/mpcontribs_api/api/v1/router.py | 2 + mpcontribs-api/src/mpcontribs_api/app.py | 2 + .../domains/project_groups/dependencies.py | 13 ++ .../domains/project_groups/models.py | 107 ++++++++++++ .../domains/project_groups/repository.py | 74 ++++++++ .../domains/project_groups/router.py | 121 +++++++++++++ .../tests/integration/db/conftest.py | 10 +- .../db/test_project_groups_repository.py | 159 ++++++++++++++++++ .../tests/unit/domains/test_shared_models.py | 42 +++++ 9 files changed, 529 insertions(+), 1 deletion(-) create mode 100644 mpcontribs-api/src/mpcontribs_api/domains/project_groups/dependencies.py create mode 100644 mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py create mode 100644 mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py create mode 100644 mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py create mode 100644 mpcontribs-api/tests/integration/db/test_project_groups_repository.py diff --git a/mpcontribs-api/src/mpcontribs_api/api/v1/router.py b/mpcontribs-api/src/mpcontribs_api/api/v1/router.py index 871ce25bdd..530751946d 100644 --- a/mpcontribs-api/src/mpcontribs_api/api/v1/router.py +++ b/mpcontribs-api/src/mpcontribs_api/api/v1/router.py @@ -3,6 +3,7 @@ from mpcontribs_api.domains.attachments.router import router as attachments_router from mpcontribs_api.domains.contributions.router import router as contributions_router from mpcontribs_api.domains.limits.router import router as limits_router +from mpcontribs_api.domains.project_groups.router import router as project_groups_router from mpcontribs_api.domains.projects.router import router as projects_router from mpcontribs_api.domains.structures.router import router as structures_router from mpcontribs_api.domains.tables.router import router as tables_router @@ -12,6 +13,7 @@ router.include_router(attachments_router, prefix="/attachments", tags=["attachments"]) router.include_router(contributions_router, prefix="/contributions", tags=["contributions"]) router.include_router(limits_router, prefix="/limits", tags=["limits"]) +router.include_router(project_groups_router, prefix="/project_groups", tags=["project_groups"]) router.include_router(projects_router, prefix="/projects", tags=["projects"]) router.include_router(structures_router, prefix="/structures", tags=["structures"]) router.include_router(tables_router, prefix="/tables", tags=["tables"]) diff --git a/mpcontribs-api/src/mpcontribs_api/app.py b/mpcontribs-api/src/mpcontribs_api/app.py index aa370d0e45..de13195b31 100644 --- a/mpcontribs-api/src/mpcontribs_api/app.py +++ b/mpcontribs-api/src/mpcontribs_api/app.py @@ -21,6 +21,7 @@ from mpcontribs_api.domains.attachments.models import Attachment from mpcontribs_api.domains.contributions.models import Contribution from mpcontribs_api.domains.healthcheck.router import router as healthcheck_router +from mpcontribs_api.domains.project_groups.models import ProjectGroup from mpcontribs_api.domains.projects.models import Project from mpcontribs_api.domains.structures.models import Structure from mpcontribs_api.domains.tables.models import Table @@ -63,6 +64,7 @@ async def _setup_mongo(app: FastAPI, settings: Settings, stack: AsyncExitStack) database=client[settings.mongo.db_name], document_models=[ Project, + ProjectGroup, Contribution, Attachment, Structure, diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/dependencies.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/dependencies.py new file mode 100644 index 0000000000..d1e0aefc54 --- /dev/null +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/dependencies.py @@ -0,0 +1,13 @@ +from typing import Annotated + +from fastapi import Depends + +from mpcontribs_api.dependencies import UserDep +from mpcontribs_api.domains.project_groups.repository import ProjectGroupRepository + + +def get_project_group_repository(user: UserDep) -> ProjectGroupRepository: + return ProjectGroupRepository(user) + + +ProjectGroupDep = Annotated[ProjectGroupRepository, Depends(get_project_group_repository)] diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py new file mode 100644 index 0000000000..671398d9ce --- /dev/null +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py @@ -0,0 +1,107 @@ +from beanie import Link, PydanticObjectId +from bson.errors import InvalidId +from pydantic import Field, field_validator +from pymongo import ASCENDING, IndexModel + +from mpcontribs_api.domains._shared.filters import BaseFilter +from mpcontribs_api.domains._shared.models import BaseDocumentWithInput, DocumentOut +from mpcontribs_api.domains._shared.types import PrefixedEmail, SearchStr, ShortStr +from mpcontribs_api.domains.projects.models import Project +from mpcontribs_api.exceptions import ValidationError +from mpcontribs_api.projection import SparseFieldsModel + + +class ProjectGroup(BaseDocumentWithInput[PydanticObjectId]): + name: SearchStr = Field(max_length=50) + owner: PrefixedEmail + projects: list[Link[Project]] + description: str = Field(max_length=100) + + class Settings: + name = "project_groups" + indexes = [ + IndexModel( + keys=[("name", ASCENDING), ("owner", ASCENDING)], + name="name_owner", + unique=True, + ) + ] + validate_on_save = True + + @classmethod + def identifier_fields(cls) -> frozenset[str]: + """A ``ProjectGroup`` is uniquely identified by its ``name`` + ``owner``.""" + return frozenset({"name", "owner"}) + + @field_validator("projects") + @classmethod + def _reject_duplicate_refs( + cls, + value: list[Link[Project] | Project], + ) -> list[Link[Project] | Project]: + seen: set[ShortStr] = set() + for item in value: + ref_id = item.ref.id if isinstance(item, Link) else item.id + if ref_id in seen: + raise ValidationError( + message="duplicate Project reference in ProjectGroup", + duplicate_id=ref_id, + ) + seen.add(ref_id) + return value + + +class ProjectGroupIn(ProjectGroup): + pass + + +class ProjectGroupOut(DocumentOut[PydanticObjectId]): + name: SearchStr | None = None + owner: PrefixedEmail | None = None + projects: list[Link[Project]] | None = None + description: str | None = None + + @staticmethod + def default_fields() -> list[str]: + return [ + "name", + "description", + "projects", + ] + + +class ProjectGroupPatch(SparseFieldsModel): + name: SearchStr | None = None + owner: PrefixedEmail | None = None + projects: list[Link[Project]] | None = None + description: str | None = None + + +class ProjectGroupFilter(BaseFilter): + id: PydanticObjectId | None = None + id__in: list[PydanticObjectId] | None = None + id__neq: PydanticObjectId | None = None + + name: SearchStr | None = None + name__in: list[SearchStr] | None = None + name__neq: ShortStr | None = None + + owner: PrefixedEmail | None = None + owner__in: list[PrefixedEmail] | None = None + owner__neq: PrefixedEmail | None = None + + order_by: list[str] | None = None + + class Constants(BaseFilter.Constants): + model: ProjectGroup + + @field_validator("id", mode="before") + @classmethod + def convert_str_to_oid(cls, v: str): + try: + return PydanticObjectId(v) + except InvalidId as err: + raise ValidationError( + "Invalid ObjectId format. Must be 12-byte input or a 24-character hex string", + oid=v, + ) from err diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py new file mode 100644 index 0000000000..ff360015e3 --- /dev/null +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py @@ -0,0 +1,74 @@ +from typing import Any + +from mpcontribs_api.authz import User +from mpcontribs_api.domains._shared.models import DeleteResponse +from mpcontribs_api.domains._shared.repository import MongoDbRepository +from mpcontribs_api.domains._shared.types import PrefixedEmail, SearchStr +from mpcontribs_api.domains.project_groups.models import ( + ProjectGroup, + ProjectGroupFilter, + ProjectGroupIn, + ProjectGroupOut, + ProjectGroupPatch, +) +from mpcontribs_api.pagination import CursorParams, Page + + +class ProjectGroupRepository( + MongoDbRepository[ProjectGroup, ProjectGroupIn, ProjectGroupOut, ProjectGroupFilter, ProjectGroupPatch] +): + document_model = ProjectGroup + out_model = ProjectGroupOut + + @staticmethod + def _build_scope(user: User) -> dict[str, Any]: + """Provides scope based on current user's permitted groups and publicly released data.""" + if user.is_admin: + return {} + ors: list[dict[str, Any]] = [{"is_public": True, "is_approved": True}] + if not user.is_anonymous: + ors.append({"owner": user.username}) + if user.groups: + ors.append({"_id": {"$in": sorted(user.groups)}}) + return {"$or": ors} + + async def get_project_groups( + self, + pagination: CursorParams, + filter: ProjectGroupFilter, + fields: frozenset[str] | None, + ) -> Page[ProjectGroupOut]: + """Return paginated project groups matching a filter. + + Args: + pagination (CursorParams): arguments for cursor-based pagination + filter (ProjectGroupFilter): optional filters to select ProjectGroups + fields (frozenset[str] | None): the fields to return to a user + """ + return await self.get_many(pagination=pagination, filter=filter, fields=fields) + + async def get_project_group( + self, + name: SearchStr, + owner: PrefixedEmail, + fields: frozenset[str] | None, + ) -> ProjectGroupOut | None: + """Return the single project group identified by ``name`` + ``owner``. See ``get_one``.""" + return await self.get_one({"name": name, "owner": owner}, fields) + + async def insert_project_group(self, project_group: ProjectGroupIn) -> ProjectGroup: + return await self.insert_one(in_resource=project_group) + + async def patch_project_group( + self, name: SearchStr, owner: PrefixedEmail, update: ProjectGroupPatch + ) -> ProjectGroup: + """Patch the single project group identified by ``name`` + ``owner``. See ``patch_one``.""" + return await self.patch_one({"name": name, "owner": owner}, update) + + async def delete_project_group(self, name: SearchStr, owner: PrefixedEmail) -> DeleteResponse: + """Delete the single project group identified by ``name`` + ``owner``. See ``delete_one``.""" + return await self.delete_one({"name": name, "owner": owner}) + + async def delete_project_groups(self, filter: ProjectGroupFilter) -> DeleteResponse: + """Bulk-delete every scoped project group matching ``filter``. See ``delete``.""" + return await self.delete(filter) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py new file mode 100644 index 0000000000..9069237271 --- /dev/null +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py @@ -0,0 +1,121 @@ +from typing import Annotated + +from fastapi import APIRouter, Depends, Response, status +from fastapi_filter import FilterDepends + +from mpcontribs_api.dependencies import require_user +from mpcontribs_api.domains._shared.models import DeleteResponse +from mpcontribs_api.domains._shared.types import FieldSelector, PrefixedEmail, SearchStr +from mpcontribs_api.domains.project_groups.dependencies import ProjectGroupDep +from mpcontribs_api.domains.project_groups.models import ( + ProjectGroupFilter, + ProjectGroupIn, + ProjectGroupOut, + ProjectGroupPatch, +) +from mpcontribs_api.pagination import CursorParams + +router = APIRouter() + + +@router.get("") +async def get_project_groups( + repo: ProjectGroupDep, + pagination: Annotated[CursorParams, Depends()], + filter: ProjectGroupFilter = FilterDepends(ProjectGroupFilter), + fields: FieldSelector = ProjectGroupOut.default_fields(), +): + """Return paginated project groups matching a filter. + + Args: + repo (ProjectGroupDep): the project group repo we depend on + pagination (CursorParams): arguments for cursor-based pagination + filter (ProjectGroupFilter): optional filters to select ProjectGroups + fields (FieldSelector): the fields to return to a user + """ + selected = ProjectGroupOut.parse_fields(fields) + return await repo.get_project_groups(pagination=pagination, filter=filter, fields=selected) + + +@router.get("/item") +async def get_project_group( + repo: ProjectGroupDep, + name: SearchStr, + owner: PrefixedEmail, + fields: FieldSelector = ProjectGroupOut.default_fields(), +): + """Return the single project group identified by ``name`` + ``owner``. + + Args: + repo (ProjectGroupDep): the project group repo we depend on + name (SearchStr): the project group's name + owner (PrefixedEmail): the project group's owner + fields (FieldSelector): the fields to return to a user + """ + selected = ProjectGroupOut.parse_fields(fields) + return await repo.get_project_group(name=name, owner=owner, fields=selected) + + +@router.post("", response_model=ProjectGroupOut, status_code=status.HTTP_201_CREATED, dependencies=[Depends(require_user)]) +async def insert_project_group( + repo: ProjectGroupDep, + project_group: ProjectGroupIn, +): + """Insert a new project group. + + Args: + repo (ProjectGroupDep): the project group repo we depend on + project_group (ProjectGroupIn): the project group to insert + """ + return await repo.insert_project_group(project_group=project_group) + + +@router.patch("/item", response_model=ProjectGroupOut, dependencies=[Depends(require_user)]) +async def patch_project_group( + repo: ProjectGroupDep, + name: SearchStr, + owner: PrefixedEmail, + update: ProjectGroupPatch, +): + """Partially update the project group identified by ``name`` + ``owner``. + + Args: + repo (ProjectGroupDep): the project group repo we depend on + name (SearchStr): the project group's name + owner (PrefixedEmail): the project group's owner + update (ProjectGroupPatch): the partial update to apply - unset fields are dropped + """ + return await repo.patch_project_group(name=name, owner=owner, update=update) + + +@router.delete("/item", status_code=status.HTTP_204_NO_CONTENT, dependencies=[Depends(require_user)]) +async def delete_project_group( + repo: ProjectGroupDep, + name: SearchStr, + owner: PrefixedEmail, +): + """Delete the single project group identified by ``name`` + ``owner``. + + Raises 404 if no such group is visible to the caller, 409 if the identifiers are ambiguous. + + Args: + repo (ProjectGroupDep): the project group repo we depend on + name (SearchStr): the project group's name + owner (PrefixedEmail): the project group's owner + """ + await repo.delete_project_group(name=name, owner=owner) + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +@router.delete("", response_model=DeleteResponse, dependencies=[Depends(require_user)]) +async def delete_project_groups( + repo: ProjectGroupDep, + filter: ProjectGroupFilter = FilterDepends(ProjectGroupFilter), +): + """Bulk-delete every project group matching ``filter`` (e.g. all with a given owner). + + Args: + repo (ProjectGroupDep): the project group repo we depend on + filter (ProjectGroupFilter): the query selecting which project groups to delete + """ + return await repo.delete_project_groups(filter=filter) diff --git a/mpcontribs-api/tests/integration/db/conftest.py b/mpcontribs-api/tests/integration/db/conftest.py index beab815b9f..f4f8caa08f 100644 --- a/mpcontribs-api/tests/integration/db/conftest.py +++ b/mpcontribs-api/tests/integration/db/conftest.py @@ -6,6 +6,7 @@ from mpcontribs_api.config import get_settings from mpcontribs_api.domains.attachments.models import Attachment from mpcontribs_api.domains.contributions.models import Contribution +from mpcontribs_api.domains.project_groups.models import ProjectGroup from mpcontribs_api.domains.projects.models import Project from mpcontribs_api.domains.structures.models import Structure from mpcontribs_api.domains.tables.models import Table @@ -66,7 +67,7 @@ async def db(mongo_client): database = mongo_client[settings.mongo.db_name] await init_beanie( database=database, - document_models=[Project, Contribution, Structure, Table, Attachment], + document_models=[Project, ProjectGroup, Contribution, Structure, Table, Attachment], ) yield database @@ -97,3 +98,10 @@ async def clean_components(db): yield for collection in ("structures", "tables", "attachments"): await db[collection].delete_many({}) + + +@pytest_asyncio.fixture(autouse=True) +async def clean_project_groups(db): + await db["project_groups"].delete_many({}) + yield + await db["project_groups"].delete_many({}) diff --git a/mpcontribs-api/tests/integration/db/test_project_groups_repository.py b/mpcontribs-api/tests/integration/db/test_project_groups_repository.py new file mode 100644 index 0000000000..b247c66043 --- /dev/null +++ b/mpcontribs-api/tests/integration/db/test_project_groups_repository.py @@ -0,0 +1,159 @@ +import pytest +from beanie import PydanticObjectId + +from mpcontribs_api.authz import User +from mpcontribs_api.domains.project_groups.models import ( + ProjectGroup, + ProjectGroupFilter, + ProjectGroupIn, + ProjectGroupPatch, +) +from mpcontribs_api.domains.project_groups.repository import ProjectGroupRepository +from mpcontribs_api.exceptions import ConflictError, NotFoundError, ValidationError + +# Share the session event loop (see the projects repo test for why). +pytestmark = [pytest.mark.db, pytest.mark.asyncio(loop_scope="session")] + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +ADMIN = User(username="google:admin@example.com", groups=frozenset({"admin"})) +ALICE = User(username="google:alice@example.com", groups=frozenset({"mp-team"})) +ANON = User() + +ALICE_EMAIL = "google:alice@example.com" + + +def _repo(user: User) -> ProjectGroupRepository: + return ProjectGroupRepository(user) + + +def _group_in(name: str, owner: str = ALICE_EMAIL, **overrides) -> ProjectGroupIn: + defaults = { + "_id": PydanticObjectId(), + "name": name, + "owner": owner, + "projects": [], + "description": "a group", + } + defaults.update(overrides) + return ProjectGroupIn(**defaults) + + +async def _insert(name: str, owner: str = ALICE_EMAIL, **overrides) -> ProjectGroup: + return await _repo(ADMIN).insert_project_group(_group_in(name, owner, **overrides)) + + +# --------------------------------------------------------------------------- +# get_one +# --------------------------------------------------------------------------- + + +class TestGetOne: + async def test_returns_group_by_identifiers(self, db): + await _insert("group-a") + found = await _repo(ADMIN).get_project_group(name="group-a", owner=ALICE_EMAIL, fields=None) + assert found is not None + assert found.name == "group-a" + assert found.owner == ALICE_EMAIL + + async def test_returns_none_when_absent(self, db): + found = await _repo(ADMIN).get_project_group(name="missing", owner=ALICE_EMAIL, fields=None) + assert found is None + + async def test_out_of_scope_returns_none(self, db): + # Alice's private group is invisible to an anonymous caller. + await _insert("group-priv") + found = await _repo(ANON).get_project_group(name="group-priv", owner=ALICE_EMAIL, fields=None) + assert found is None + + +# --------------------------------------------------------------------------- +# delete_one (identifier-keyed, single-resource, raises) +# --------------------------------------------------------------------------- + + +class TestDeleteOne: + async def test_deletes_matching_group(self, db): + await _insert("del-a") + result = await _repo(ADMIN).delete_project_group(name="del-a", owner=ALICE_EMAIL) + assert result.num_deleted == 1 + assert await ProjectGroup.find_one(ProjectGroup.name == "del-a") is None + + async def test_absent_raises_not_found(self, db): + with pytest.raises(NotFoundError): + await _repo(ADMIN).delete_project_group(name="nope", owner=ALICE_EMAIL) + + async def test_out_of_scope_raises_not_found(self, db): + # Alice's group is out of scope for anon, so it "does not exist" for them. + await _insert("del-scoped") + with pytest.raises(NotFoundError): + await _repo(ANON).delete_project_group(name="del-scoped", owner=ALICE_EMAIL) + # ...and it is untouched. + assert await ProjectGroup.find_one(ProjectGroup.name == "del-scoped") is not None + + async def test_wrong_identifier_keys_raise_validation(self, db): + with pytest.raises(ValidationError): + await _repo(ADMIN).delete_one({"name": "x"}) # missing 'owner' + + async def test_duplicate_identifiers_raise_conflict(self, db): + # The name_owner unique index normally makes this impossible; drop it so we can plant a + # duplicate and exercise the defensive uniqueness guard in _resolve_one_id. Tolerant of a + # prior drop within the same session. + try: + await db["project_groups"].drop_index("name_owner") + except Exception: + pass + await db["project_groups"].insert_many( + [ + {"_id": PydanticObjectId(), "name": "dup", "owner": ALICE_EMAIL, "projects": [], "description": "d"}, + {"_id": PydanticObjectId(), "name": "dup", "owner": ALICE_EMAIL, "projects": [], "description": "d"}, + ] + ) + with pytest.raises(ConflictError): + await _repo(ADMIN).delete_project_group(name="dup", owner=ALICE_EMAIL) + + +# --------------------------------------------------------------------------- +# patch_one +# --------------------------------------------------------------------------- + + +class TestPatchOne: + async def test_updates_field(self, db): + await _insert("patch-a", description="before") + updated = await _repo(ADMIN).patch_project_group( + name="patch-a", owner=ALICE_EMAIL, update=ProjectGroupPatch(description="after") + ) + assert updated.description == "after" + + async def test_absent_raises_not_found(self, db): + with pytest.raises(NotFoundError): + await _repo(ADMIN).patch_project_group( + name="ghost", owner=ALICE_EMAIL, update=ProjectGroupPatch(description="x") + ) + + +# --------------------------------------------------------------------------- +# delete (arbitrary-filter bulk) +# --------------------------------------------------------------------------- + + +class TestDeleteByFilter: + async def test_bulk_deletes_all_matching_owner(self, db): + await _insert("bulk-1") + await _insert("bulk-2") + await _insert("other", owner="google:bob@example.com") + result = await _repo(ADMIN).delete_project_groups( + filter=ProjectGroupFilter(owner=ALICE_EMAIL) + ) + assert result.num_deleted == 2 + assert await ProjectGroup.find_one(ProjectGroup.owner == "google:bob@example.com") is not None + + async def test_no_match_returns_zero(self, db): + result = await _repo(ADMIN).delete_project_groups( + filter=ProjectGroupFilter(owner="google:nobody@example.com") + ) + assert result.num_deleted == 0 diff --git a/mpcontribs-api/tests/unit/domains/test_shared_models.py b/mpcontribs-api/tests/unit/domains/test_shared_models.py index 2ba8c3c9d8..b7cb13ad7d 100644 --- a/mpcontribs-api/tests/unit/domains/test_shared_models.py +++ b/mpcontribs-api/tests/unit/domains/test_shared_models.py @@ -8,6 +8,9 @@ DocumentOut, ) from mpcontribs_api.domains.attachments.models import Attachment, AttachmentIn +from mpcontribs_api.domains.contributions.models import Contribution, ContributionIn +from mpcontribs_api.domains.project_groups.models import ProjectGroup +from mpcontribs_api.domains.projects.models import Project from mpcontribs_api.pagination import encode_cursor # --------------------------------------------------------------------------- @@ -98,6 +101,45 @@ def test_serializes_under_id_not_underscore_id(self): # --------------------------------------------------------------------------- +# --------------------------------------------------------------------------- +# identifier_fields() / identifiers() contract +# --------------------------------------------------------------------------- + + +class TestIdentifierContract: + def test_default_identifier_fields_is_primary_key(self): + # Content-addressed components fall back to the base default. + assert Attachment.identifier_fields() == frozenset({"id"}) + + def test_project_uses_id(self): + assert Project.identifier_fields() == frozenset({"id"}) + + def test_project_group_uses_name_and_owner(self): + assert ProjectGroup.identifier_fields() == frozenset({"name", "owner"}) + + def test_contribution_uses_project_and_identifier(self): + assert Contribution.identifier_fields() == frozenset({"project", "identifier"}) + assert ContributionIn.identifier_fields() == frozenset({"project", "identifier"}) + + def test_default_identifiers_reads_values_off_instance(self): + oid = PydanticObjectId() + doc = Attachment.from_input(_attachment_in()) + doc.id = oid + assert doc.identifiers() == {"id": oid} + + def test_contribution_identifiers_returns_natural_key_values(self): + contrib = ContributionIn( + **{ + "_id": PydanticObjectId(), + "project": "test-project", + "identifier": "mp-1234", + "formula": "Fe2O3", + "data": {}, + } + ) + assert contrib.identifiers() == {"project": "test-project", "identifier": "mp-1234"} + + class TestDeleteResponse: def test_from_delete_result(self): result = DeleteResult({"n": 3}, acknowledged=True) From cc5cc1652ea30f7cb8f3793457310a97de16ee67 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 15 Jul 2026 17:02:44 -0700 Subject: [PATCH 10/62] Added project_groups service to coordinate manipulation of projects within a project group --- .../domains/project_groups/service.py | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py new file mode 100644 index 0000000000..f059ea19c2 --- /dev/null +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py @@ -0,0 +1,111 @@ +from beanie import Link + +from mpcontribs_api.domains._shared.bulk import BulkFailure, BulkWriteSummary +from mpcontribs_api.domains._shared.types import PrefixedEmail, SearchStr, ShortStr +from mpcontribs_api.domains.project_groups.models import ProjectGroupOut +from mpcontribs_api.domains.project_groups.repository import ProjectGroupRepository +from mpcontribs_api.domains.projects.repository import MongoDbProjectRepository +from mpcontribs_api.exceptions import NotFoundError + +# Fields the membership operations need off a resolved group: its id (target of the update) and its +# current members (so deletion can tell members from non-members). +_GROUP_FIELDS = frozenset({"id", "projects"}) + + +class ProjectGroupService: + """Coordinates project-group membership changes across the groups and projects collections""" + + def __init__( + self, + groups: ProjectGroupRepository, + projects: MongoDbProjectRepository, + ) -> None: + self._groups = groups + self._projects = projects + + async def _resolve_by_id(self, group_id: str) -> ProjectGroupOut: + """Resolve a visible group by its ObjectId, or raise ``NotFoundError``.""" + oid = self._groups._convert_object_id(group_id) + group = await self._groups.get_by_id(oid, fields=_GROUP_FIELDS) + if group is None: + raise NotFoundError("ProjectGroup not found", id=group_id) + return group # pyright: ignore[reportReturnType] # projected reads return the out model + + async def _resolve_by_identifiers(self, name: SearchStr, owner: PrefixedEmail) -> ProjectGroupOut: + """Resolve a visible group by its ``(name, owner)`` identifiers, or raise ``NotFoundError``. + + Propagates ``ConflictError`` from the repository if the identifiers are ambiguous. + """ + group = await self._groups.get_one({"name": name, "owner": owner}, fields=_GROUP_FIELDS) + if group is None: + raise NotFoundError("ProjectGroup not found", name=name, owner=owner) + return group + + async def _add(self, group: ProjectGroupOut, project_ids: list[ShortStr]) -> BulkWriteSummary[str]: + """Validate each project against the projects collection, then add the valid ones. + + A project that does not exist or is not visible to the caller is reported as a failed item; + the rest are added in a single atomic ``$addToSet`` (idempotent for existing members). + """ + failed: list[BulkFailure] = [] + valid: list[ShortStr] = [] + for index, pid in enumerate(project_ids): + if await self._projects.get_by_id(pid, fields=frozenset({"id"})) is None: + failed.append( + BulkFailure( + index=index, + identifier={"id": pid}, + error_code="not_found", + message=f"Project {pid} not found or not visible", + ) + ) + elif pid not in valid: + valid.append(pid) + + if valid: + await self._groups.add_project_refs(group.id, valid) # pyright: ignore[reportArgumentType] # id is set on a resolved group + return BulkWriteSummary(total=len(project_ids), succeeded=valid, failed=failed) + + async def _delete(self, group: ProjectGroupOut, project_ids: list[ShortStr]) -> BulkWriteSummary[str]: + """Delete requested members from the group; non-members are reported as failed items.""" + current = { + (link.ref.id if isinstance(link, Link) else link.id) for link in (group.projects or []) + } + failed: list[BulkFailure] = [] + present: list[ShortStr] = [] + for index, pid in enumerate(project_ids): + if pid not in current: + failed.append( + BulkFailure( + index=index, + identifier={"id": pid}, + error_code="not_found", + message=f"Project {pid} is not a member of this group", + ) + ) + elif pid not in present: + present.append(pid) + + if present: + await self._groups.delete_project_refs(group.id, present) # pyright: ignore[reportArgumentType] # id is set on a resolved group + return BulkWriteSummary(total=len(project_ids), succeeded=present, failed=failed) + + async def add_projects_by_id(self, group_id: str, project_ids: list[ShortStr]) -> BulkWriteSummary[str]: + """Add projects to the group identified by ``group_id``.""" + return await self._add(await self._resolve_by_id(group_id), project_ids) + + async def add_projects_by_identifiers( + self, name: SearchStr, owner: PrefixedEmail, project_ids: list[ShortStr] + ) -> BulkWriteSummary[str]: + """Add projects to the group identified by ``(name, owner)``.""" + return await self._add(await self._resolve_by_identifiers(name, owner), project_ids) + + async def delete_projects_by_id(self, group_id: str, project_ids: list[ShortStr]) -> BulkWriteSummary[str]: + """Delete projects from the group identified by ``group_id``.""" + return await self._delete(await self._resolve_by_id(group_id), project_ids) + + async def delete_projects_by_identifiers( + self, name: SearchStr, owner: PrefixedEmail, project_ids: list[ShortStr] + ) -> BulkWriteSummary[str]: + """Delete projects from the group identified by ``(name, owner)``.""" + return await self._delete(await self._resolve_by_identifiers(name, owner), project_ids) From 91055fa2b7b6bc2799bf78c8ad5f9f65de376aa2 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 15 Jul 2026 17:03:09 -0700 Subject: [PATCH 11/62] Created ProjectGroupServiceDep --- .../domains/project_groups/dependencies.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/dependencies.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/dependencies.py index d1e0aefc54..7220513aca 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/dependencies.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/dependencies.py @@ -4,6 +4,8 @@ from mpcontribs_api.dependencies import UserDep from mpcontribs_api.domains.project_groups.repository import ProjectGroupRepository +from mpcontribs_api.domains.project_groups.service import ProjectGroupService +from mpcontribs_api.domains.projects.repository import MongoDbProjectRepository def get_project_group_repository(user: UserDep) -> ProjectGroupRepository: @@ -11,3 +13,13 @@ def get_project_group_repository(user: UserDep) -> ProjectGroupRepository: ProjectGroupDep = Annotated[ProjectGroupRepository, Depends(get_project_group_repository)] + + +def get_project_group_service(user: UserDep) -> ProjectGroupService: + return ProjectGroupService( + groups=ProjectGroupRepository(user), + projects=MongoDbProjectRepository(user), + ) + + +ProjectGroupServiceDep = Annotated[ProjectGroupService, Depends(get_project_group_service)] From a4a41886794e20fc495972b212b543341c2b8ad0 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 15 Jul 2026 17:03:45 -0700 Subject: [PATCH 12/62] Added logic for manipulation of group membership of projects --- .../domains/project_groups/models.py | 8 +++- .../domains/project_groups/repository.py | 47 ++++++++++++++++++- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py index 671398d9ce..4c2546d81f 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py @@ -1,6 +1,6 @@ from beanie import Link, PydanticObjectId from bson.errors import InvalidId -from pydantic import Field, field_validator +from pydantic import BaseModel, Field, field_validator from pymongo import ASCENDING, IndexModel from mpcontribs_api.domains._shared.filters import BaseFilter @@ -77,6 +77,12 @@ class ProjectGroupPatch(SparseFieldsModel): description: str | None = None +class ProjectRefs(BaseModel): + """Request body for adding/removing projects from a group: the project ids to (un)link.""" + + project_ids: list[ShortStr] = Field(default_factory=list) + + class ProjectGroupFilter(BaseFilter): id: PydanticObjectId | None = None id__in: list[PydanticObjectId] | None = None diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py index ff360015e3..eee5c19ced 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py @@ -1,9 +1,14 @@ from typing import Any +from beanie import PydanticObjectId, UpdateResponse +from beanie.operators import AddToSet, Pull +from bson import DBRef +from pymongo.asynchronous.client_session import AsyncClientSession + from mpcontribs_api.authz import User from mpcontribs_api.domains._shared.models import DeleteResponse from mpcontribs_api.domains._shared.repository import MongoDbRepository -from mpcontribs_api.domains._shared.types import PrefixedEmail, SearchStr +from mpcontribs_api.domains._shared.types import PrefixedEmail, SearchStr, ShortStr from mpcontribs_api.domains.project_groups.models import ( ProjectGroup, ProjectGroupFilter, @@ -72,3 +77,43 @@ async def delete_project_group(self, name: SearchStr, owner: PrefixedEmail) -> D async def delete_project_groups(self, filter: ProjectGroupFilter) -> DeleteResponse: """Bulk-delete every scoped project group matching ``filter``. See ``delete``.""" return await self.delete(filter) + + async def add_project_refs( + self, + group_id: PydanticObjectId, + project_ids: list[ShortStr], + session: AsyncClientSession | None = None, + ) -> ProjectGroup | None: + """Atomically add project references to a scoped group, deduplicating existing members. + + Args: + group_id (PydanticObjectId): the id of the group to modify + project_ids (list[ShortStr]): project ids to add (already validated by the service) + session (AsyncClientSession | None): optional client session for transactions + """ + refs = [DBRef("projects", pid) for pid in project_ids] + query = self.document_model.find_one(self._scope, self.document_model.id == group_id, session=session).update( + AddToSet({"projects": {"$each": refs}}), + response_type=UpdateResponse.NEW_DOCUMENT, + ) + return await query # pyright: ignore[reportGeneralTypeIssues] # beanie UpdateQuery is awaitable + + async def delete_project_refs( + self, + group_id: PydanticObjectId, + project_ids: list[ShortStr], + session: AsyncClientSession | None = None, + ) -> ProjectGroup | None: + """Atomically delete project references from a scoped group. + + Args: + group_id (PydanticObjectId): the id of the group to modify + project_ids (list[ShortStr]): project ids to delete + session (AsyncClientSession | None): optional client session for transactions + """ + refs = [DBRef("projects", pid) for pid in project_ids] + query = self.document_model.find_one(self._scope, self.document_model.id == group_id, session=session).update( + Pull({"projects": {"$in": refs}}), + response_type=UpdateResponse.NEW_DOCUMENT, + ) + return await query # pyright: ignore[reportGeneralTypeIssues] # beanie UpdateQuery is awaitable From e5d715d43dc18cca4c81da61c148231e7c1647b5 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 15 Jul 2026 17:04:29 -0700 Subject: [PATCH 13/62] Added routes for individual project group operations --- .../domains/project_groups/router.py | 53 ++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py index 9069237271..a27b485b34 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py @@ -4,14 +4,16 @@ from fastapi_filter import FilterDepends from mpcontribs_api.dependencies import require_user +from mpcontribs_api.domains._shared.bulk import BulkWriteSummary from mpcontribs_api.domains._shared.models import DeleteResponse from mpcontribs_api.domains._shared.types import FieldSelector, PrefixedEmail, SearchStr -from mpcontribs_api.domains.project_groups.dependencies import ProjectGroupDep +from mpcontribs_api.domains.project_groups.dependencies import ProjectGroupDep, ProjectGroupServiceDep from mpcontribs_api.domains.project_groups.models import ( ProjectGroupFilter, ProjectGroupIn, ProjectGroupOut, ProjectGroupPatch, + ProjectRefs, ) from mpcontribs_api.pagination import CursorParams @@ -119,3 +121,52 @@ async def delete_project_groups( filter (ProjectGroupFilter): the query selecting which project groups to delete """ return await repo.delete_project_groups(filter=filter) + + +@router.post("/item/projects", response_model=BulkWriteSummary[str], dependencies=[Depends(require_user)]) +async def add_projects_by_identifiers( + service: ProjectGroupServiceDep, + name: SearchStr, + owner: PrefixedEmail, + body: ProjectRefs, +): + """Add projects to the group identified by ``name`` + ``owner``. + + Each project is verified against the projects collection (scoped to the caller); unknown or + invisible projects are reported per-item in the response rather than failing the whole request. + """ + return await service.add_projects_by_identifiers(name=name, owner=owner, project_ids=body.project_ids) + + +@router.delete("/item/projects", response_model=BulkWriteSummary[str], dependencies=[Depends(require_user)]) +async def delete_projects_by_identifiers( + service: ProjectGroupServiceDep, + name: SearchStr, + owner: PrefixedEmail, + body: ProjectRefs, +): + """Delete projects from the group identified by ``name`` + ``owner``. + + Ids that are not members of the group are reported per-item in the response. + """ + return await service.delete_projects_by_identifiers(name=name, owner=owner, project_ids=body.project_ids) + + +@router.post("/{id}/projects", response_model=BulkWriteSummary[str], dependencies=[Depends(require_user)]) +async def add_projects_by_id( + service: ProjectGroupServiceDep, + id: str, + body: ProjectRefs, +): + """Add projects to the group identified by ``id``. See ``add_projects_by_identifiers``.""" + return await service.add_projects_by_id(group_id=id, project_ids=body.project_ids) + + +@router.delete("/{id}/projects", response_model=BulkWriteSummary[str], dependencies=[Depends(require_user)]) +async def delete_projects_by_id( + service: ProjectGroupServiceDep, + id: str, + body: ProjectRefs, +): + """Delete projects from the group identified by ``id``. See ``delete_projects_by_identifiers``.""" + return await service.delete_projects_by_id(group_id=id, project_ids=body.project_ids) From 7b6400d675971c3f633accecae617a8ae64206c0 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 15 Jul 2026 17:04:42 -0700 Subject: [PATCH 14/62] Added tests --- .../db/test_project_groups_service.py | 154 ++++++++++++++++++ .../domains/test_project_group_service.py | 144 ++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100644 mpcontribs-api/tests/integration/db/test_project_groups_service.py create mode 100644 mpcontribs-api/tests/unit/domains/test_project_group_service.py diff --git a/mpcontribs-api/tests/integration/db/test_project_groups_service.py b/mpcontribs-api/tests/integration/db/test_project_groups_service.py new file mode 100644 index 0000000000..d9c034f420 --- /dev/null +++ b/mpcontribs-api/tests/integration/db/test_project_groups_service.py @@ -0,0 +1,154 @@ +import pytest +from beanie import PydanticObjectId + +from mpcontribs_api.authz import User +from mpcontribs_api.domains.project_groups.models import ProjectGroup, ProjectGroupIn +from mpcontribs_api.domains.project_groups.repository import ProjectGroupRepository +from mpcontribs_api.domains.project_groups.service import ProjectGroupService +from mpcontribs_api.domains.projects.models import ProjectIn, Stats +from mpcontribs_api.domains.projects.repository import MongoDbProjectRepository +from mpcontribs_api.exceptions import ConflictError, NotFoundError + +pytestmark = [pytest.mark.db, pytest.mark.asyncio(loop_scope="session")] + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +ADMIN = User(username="google:admin@example.com", groups=frozenset({"admin"})) +ALICE = User(username="google:alice@example.com", groups=frozenset({"mp-team"})) +ANON = User() + +ALICE_EMAIL = "google:alice@example.com" +BOB_EMAIL = "google:bob@example.com" +STATS = Stats(columns=0, contributions=0, tables=0, structures=0, attachments=0, size=0.0) + + +def _service(user: User = ADMIN) -> ProjectGroupService: + return ProjectGroupService(groups=ProjectGroupRepository(user), projects=MongoDbProjectRepository(user)) + + +async def _insert_project(pid: str, owner: str = ALICE_EMAIL, **overrides): + payload = { + "_id": pid, + "title": pid[:30], + "authors": "Author", + "description": "desc", + "owner": owner, + "unique_identifiers": True, + "stats": STATS, + } + payload.update(overrides) + return await MongoDbProjectRepository(ADMIN).insert_project(ProjectIn(**payload)) + + +async def _insert_group(name: str, owner: str = ALICE_EMAIL) -> ProjectGroup: + return await ProjectGroupRepository(ADMIN).insert_project_group( + ProjectGroupIn(_id=PydanticObjectId(), name=name, owner=owner, projects=[], description="d") + ) + + +async def _members(group_id: PydanticObjectId) -> list[str]: + doc = await ProjectGroup.find_one(ProjectGroup.id == group_id) + assert doc is not None + return sorted(link.ref.id for link in (doc.projects or [])) + + +# --------------------------------------------------------------------------- +# add +# --------------------------------------------------------------------------- + + +class TestAdd: + async def test_add_by_id_links_projects(self, db): + group = await _insert_group("add-id") + await _insert_project("mp-1") + await _insert_project("mp-2") + summary = await _service().add_projects_by_id(str(group.id), ["mp-1", "mp-2"]) + assert summary.succeeded == ["mp-1", "mp-2"] + assert summary.failed == [] + assert await _members(group.id) == ["mp-1", "mp-2"] + + async def test_add_by_identifiers_links_projects(self, db): + group = await _insert_group("add-ident") + await _insert_project("mp-x") + summary = await _service().add_projects_by_identifiers("add-ident", ALICE_EMAIL, ["mp-x"]) + assert summary.succeeded == ["mp-x"] + assert await _members(group.id) == ["mp-x"] + + async def test_add_is_idempotent(self, db): + group = await _insert_group("add-idem") + await _insert_project("mp-1") + await _service().add_projects_by_id(str(group.id), ["mp-1"]) + await _service().add_projects_by_id(str(group.id), ["mp-1"]) + assert await _members(group.id) == ["mp-1"] + + async def test_missing_project_fails_and_leaves_group_unchanged(self, db): + group = await _insert_group("add-missing") + summary = await _service().add_projects_by_id(str(group.id), ["ghost"]) + assert summary.succeeded == [] + assert summary.failed[0].error_code == "not_found" + assert await _members(group.id) == [] + + async def test_out_of_scope_project_fails(self, db): + # Bob's private project is invisible to Alice, so she cannot link it. + group = await _insert_group("add-scope") + await _insert_project("mp-bob", owner=BOB_EMAIL) + summary = await _service(ALICE).add_projects_by_id(str(group.id), ["mp-bob"]) + assert summary.succeeded == [] + assert summary.failed[0].error_code == "not_found" + assert await _members(group.id) == [] + + async def test_group_not_visible_raises_not_found(self, db): + group = await _insert_group("add-priv") # owned by Alice, invisible to anon + with pytest.raises(NotFoundError): + await _service(ANON).add_projects_by_id(str(group.id), []) + + async def test_ambiguous_identifiers_raise_conflict(self, db): + # Drop the unique index so we can plant a duplicate and exercise the uniqueness guard. + try: + await db["project_groups"].drop_index("name_owner") + except Exception: + pass + await db["project_groups"].insert_many( + [ + {"_id": PydanticObjectId(), "name": "dup", "owner": ALICE_EMAIL, "projects": [], "description": "d"}, + {"_id": PydanticObjectId(), "name": "dup", "owner": ALICE_EMAIL, "projects": [], "description": "d"}, + ] + ) + with pytest.raises(ConflictError): + await _service().add_projects_by_identifiers("dup", ALICE_EMAIL, []) + + +# --------------------------------------------------------------------------- +# delete +# --------------------------------------------------------------------------- + + +class TestDelete: + async def test_delete_by_id_unlinks_project(self, db): + group = await _insert_group("rm-id") + await _insert_project("mp-1") + await _insert_project("mp-2") + await _service().add_projects_by_id(str(group.id), ["mp-1", "mp-2"]) + summary = await _service().delete_projects_by_id(str(group.id), ["mp-1"]) + assert summary.succeeded == ["mp-1"] + assert await _members(group.id) == ["mp-2"] + + async def test_delete_by_identifiers_unlinks_project(self, db): + group = await _insert_group("rm-ident") + await _insert_project("mp-1") + await _service().add_projects_by_id(str(group.id), ["mp-1"]) + summary = await _service().delete_projects_by_identifiers("rm-ident", ALICE_EMAIL, ["mp-1"]) + assert summary.succeeded == ["mp-1"] + assert await _members(group.id) == [] + + async def test_delete_non_member_reported_as_failure(self, db): + group = await _insert_group("rm-nonmember") + await _insert_project("mp-1") + await _service().add_projects_by_id(str(group.id), ["mp-1"]) + summary = await _service().delete_projects_by_id(str(group.id), ["ghost"]) + assert summary.succeeded == [] + assert summary.failed[0].error_code == "not_found" + assert await _members(group.id) == ["mp-1"] diff --git a/mpcontribs-api/tests/unit/domains/test_project_group_service.py b/mpcontribs-api/tests/unit/domains/test_project_group_service.py new file mode 100644 index 0000000000..8385b6c7a9 --- /dev/null +++ b/mpcontribs-api/tests/unit/domains/test_project_group_service.py @@ -0,0 +1,144 @@ +from unittest.mock import AsyncMock, MagicMock + +import pytest +from beanie import Link, PydanticObjectId +from bson import DBRef + +from mpcontribs_api.domains.project_groups.models import ProjectGroupOut +from mpcontribs_api.domains.project_groups.service import ProjectGroupService +from mpcontribs_api.domains.projects.models import Project +from mpcontribs_api.exceptions import ConflictError, NotFoundError + +pytestmark = pytest.mark.asyncio + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_service(group: ProjectGroupOut | None, *, visible_projects: set[str] | None = None, ambiguous: bool = False): + """Build a service over stubbed repos. + + ``group`` is what the groups repo resolves to (None => not found). ``visible_projects`` gates + which project ids the projects repo reports as existing/visible. ``ambiguous`` makes identifier + resolution raise ConflictError (duplicate under the unique key). + """ + visible = visible_projects or set() + groups = AsyncMock() + projects = AsyncMock() + + if ambiguous: + groups.get_one.side_effect = ConflictError("ambiguous") + else: + groups.get_one.return_value = group + groups.get_by_id.return_value = group + # _convert_object_id is a sync repo method; keep it sync so the service gets a real id, not a coroutine. + groups._convert_object_id = MagicMock(side_effect=lambda s: PydanticObjectId(s)) + groups.add_project_refs.return_value = group + groups.delete_project_refs.return_value = group + + async def _get_project(pid, fields=None): + return {"_id": pid} if pid in visible else None + + projects.get_by_id.side_effect = _get_project + + return ProjectGroupService(groups=groups, projects=projects), groups, projects + + +def _group(project_ids: list[str] | None = None) -> ProjectGroupOut: + group = ProjectGroupOut.model_validate( + {"_id": PydanticObjectId(), "name": "g", "owner": "google:a@b.com", "projects": []} + ) + # Members are stored as Links (DBRefs); set them directly to sidestep Link revalidation. + group.projects = [Link(DBRef("projects", pid), Project) for pid in (project_ids or [])] + return group + + +# --------------------------------------------------------------------------- +# Group resolution +# --------------------------------------------------------------------------- + + +class TestGroupResolution: + async def test_add_by_id_missing_group_raises_not_found(self): + service, _, _ = _make_service(None) + with pytest.raises(NotFoundError): + await service.add_projects_by_id("0" * 24, ["mp-1"]) + + async def test_add_by_identifiers_missing_group_raises_not_found(self): + service, _, _ = _make_service(None) + with pytest.raises(NotFoundError): + await service.add_projects_by_identifiers("g", "google:a@b.com", ["mp-1"]) + + async def test_ambiguous_identifiers_propagate_conflict(self): + service, _, _ = _make_service(_group(), ambiguous=True) + with pytest.raises(ConflictError): + await service.add_projects_by_identifiers("g", "google:a@b.com", ["mp-1"]) + + +# --------------------------------------------------------------------------- +# add +# --------------------------------------------------------------------------- + + +class TestAddProjects: + async def test_valid_projects_are_added(self): + group = _group() + service, groups, _ = _make_service(group, visible_projects={"mp-1", "mp-2"}) + summary = await service.add_projects_by_id(str(group.id), ["mp-1", "mp-2"]) + assert summary.total == 2 + assert summary.succeeded == ["mp-1", "mp-2"] + assert summary.failed == [] + groups.add_project_refs.assert_awaited_once_with(group.id, ["mp-1", "mp-2"]) + + async def test_missing_project_reported_as_failure(self): + group = _group() + service, groups, _ = _make_service(group, visible_projects={"mp-1"}) + summary = await service.add_projects_by_id(str(group.id), ["mp-1", "ghost"]) + assert summary.succeeded == ["mp-1"] + assert len(summary.failed) == 1 + assert summary.failed[0].index == 1 + assert summary.failed[0].error_code == "not_found" + # only the valid id is written + groups.add_project_refs.assert_awaited_once_with(group.id, ["mp-1"]) + + async def test_no_valid_projects_skips_update(self): + group = _group() + service, groups, _ = _make_service(group, visible_projects=set()) + summary = await service.add_projects_by_id(str(group.id), ["ghost"]) + assert summary.succeeded == [] + assert len(summary.failed) == 1 + groups.add_project_refs.assert_not_awaited() + + async def test_duplicate_input_added_once(self): + group = _group() + service, groups, _ = _make_service(group, visible_projects={"mp-1"}) + summary = await service.add_projects_by_id(str(group.id), ["mp-1", "mp-1"]) + assert summary.succeeded == ["mp-1"] + groups.add_project_refs.assert_awaited_once_with(group.id, ["mp-1"]) + + +# --------------------------------------------------------------------------- +# delete +# --------------------------------------------------------------------------- + + +class TestDeleteProjects: + async def test_members_deleted_non_members_reported(self): + group = _group(["mp-1", "mp-2"]) + service, groups, _ = _make_service(group) + summary = await service.delete_projects_by_id(str(group.id), ["mp-1", "ghost"]) + assert summary.succeeded == ["mp-1"] + assert len(summary.failed) == 1 + assert summary.failed[0].index == 1 + assert summary.failed[0].error_code == "not_found" + groups.delete_project_refs.assert_awaited_once_with(group.id, ["mp-1"]) + + async def test_no_members_skips_update(self): + group = _group(["mp-1"]) + service, groups, _ = _make_service(group) + summary = await service.delete_projects_by_id(str(group.id), ["ghost"]) + assert summary.succeeded == [] + assert len(summary.failed) == 1 + groups.delete_project_refs.assert_not_awaited() From 376037ebb707c0e155ab7b7a02501f2ad135b345 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 15 Jul 2026 22:16:43 -0700 Subject: [PATCH 15/62] Added __contains filter operator --- .../mpcontribs_api/domains/_shared/filters.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/_shared/filters.py b/mpcontribs-api/src/mpcontribs_api/domains/_shared/filters.py index 027d25dfb5..4cd05d3bad 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/_shared/filters.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/_shared/filters.py @@ -2,6 +2,11 @@ from typing import Any from fastapi_filter.contrib.beanie import Filter +from fastapi_filter.contrib.beanie.filter import _odm_operator_transformer +from pydantic import ValidationInfo, field_validator + +# Register a custom __contains filter suffix to search where lists are a superset of a provided list +_odm_operator_transformer.setdefault("contains", lambda value: {"$all": value}) class BaseFilter(Filter): @@ -16,6 +21,20 @@ class BaseFilter(Filter): Domain filters should subclass this instead of fastapi-filter's ``Filter`` directly. """ + @field_validator("*", mode="before") + @classmethod + def _split_contains(cls, value: str | None, field: ValidationInfo) -> list[str] | str | None: + """Split a comma-separated ``__contains`` query string into a list. + + ``FilterDepends`` collapses list-typed filter fields to a single string query param and + relies on a before-validator to re-expand it. fastapi-filter only does this for ``__in`` + and ``__nin``; mirror it here for the ``contains`` operator so ``?tags__contains=a,c`` + parses into ``["a", "c"]``. + """ + if field.field_name is not None and field.field_name.endswith("__contains") and isinstance(value, str): + return value.split(",") if value else [] + return value + def _get_filter_conditions(self, nesting_depth: int = 1) -> list[tuple[Mapping[str, Any], Mapping[str, Any]]]: return [ ({"_id" if key == "id" else key: value for key, value in condition.items()}, options) From 38cf31fb6a2165a1c2885d66a7416f00e6cadf92 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 15 Jul 2026 22:17:32 -0700 Subject: [PATCH 16/62] Renamed 'category' to 'tags' for clearer terminology --- .../mpcontribs_api/domains/projects/models.py | 13 +++-- .../db/test_projects_repository.py | 51 +++++++++++++++++++ 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py index 728c1275b4..589ee198c3 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py @@ -51,7 +51,7 @@ class Project(BaseDocumentWithInput[ShortStr]): stats: Stats # Optional - category: ShortStr | None = None + tags: list[ShortStr] | None = None references: list[Reference] = Field(default_factory=list) long_title: str | None = None other: dict[str, Any] = Field(default_factory=dict) @@ -85,7 +85,7 @@ class ProjectOut(DocumentOut[ShortStr]): authors: str | None = None description: str | None = None title: ShortStr | None = None - category: ShortStr | None = None + tags: list[ShortStr] | None = None owner: PrefixedEmail | None = None other: dict[str, Any] | None = None is_public: bool | None = None @@ -119,10 +119,9 @@ class ProjectFilter(BaseFilter): owner__neq: PrefixedEmail | None = None owner__ilike: str | None = None - category: ShortStr | None = None - category__in: list[ShortStr] | None = None - category__neq: ShortStr | None = None - category__ilike: str | None = None + tags: list[ShortStr] | None = None # exact match of list + tags__in: list[ShortStr] | None = None # if at least one tag is present + tags__contains: list[ShortStr] | None = None # Project.tags must be a superset of these # fuzzy only long_title__ilike: str | None = None @@ -154,7 +153,7 @@ class ProjectPatch(BaseModel): title: ShortStr | None = None authors: str | None = None description: str | None = None - category: ShortStr | None = None + tags: list[ShortStr] | None = None owner: PrefixedEmail | None = None unique_identifiers: bool | None = None references: list[Reference] = Field(default_factory=list) diff --git a/mpcontribs-api/tests/integration/db/test_projects_repository.py b/mpcontribs-api/tests/integration/db/test_projects_repository.py index 6ec770c1aa..7e33766870 100644 --- a/mpcontribs-api/tests/integration/db/test_projects_repository.py +++ b/mpcontribs-api/tests/integration/db/test_projects_repository.py @@ -188,6 +188,57 @@ async def test_filter_by_id_neq_excludes(self, db): assert "filter-id-neq-drop" not in ids +# --------------------------------------------------------------------------- +# get_projects — tags filtering +# +# ``tags__contains`` maps to MongoDB ``$all``: a project matches only when its +# tags are a superset of every value supplied (the query list is a subset of +# the stored array). Contrast with ``tags__in`` ($in), which matches on any +# single overlapping tag. +# --------------------------------------------------------------------------- + + +class TestGetProjectsTagsFilter: + async def test_contains_requires_all_tags_as_subset(self, db): + from mpcontribs_api.domains.projects.models import ProjectFilter + + await _insert("tags-superset", tags=["alpha", "beta", "gamma"]) + await _insert("tags-partial", tags=["alpha", "beta"]) + await _insert("tags-none", tags=["delta"]) + page = await _repo(ADMIN).get_projects( + filter=ProjectFilter(tags__contains=["alpha", "gamma"]), + pagination=CursorParams(), + fields=None, + ) + assert {p.id for p in page.items} == {"tags-superset"} + + async def test_contains_single_tag(self, db): + from mpcontribs_api.domains.projects.models import ProjectFilter + + await _insert("tags-single-hit", tags=["alpha", "beta"]) + await _insert("tags-single-miss", tags=["beta", "gamma"]) + page = await _repo(ADMIN).get_projects( + filter=ProjectFilter(tags__contains=["alpha"]), + pagination=CursorParams(), + fields=None, + ) + assert {p.id for p in page.items} == {"tags-single-hit"} + + async def test_contains_parses_comma_string(self, db): + from mpcontribs_api.domains.projects.models import ProjectFilter + + await _insert("tags-csv-hit", tags=["alpha", "beta", "gamma"]) + await _insert("tags-csv-miss", tags=["alpha"]) + # FilterDepends collapses the list query param to a comma string; the + # BaseFilter validator must re-expand it. + page = await _repo(ADMIN).get_projects( + filter=ProjectFilter(tags__contains="alpha,beta"), + pagination=CursorParams(), + fields=None, + ) + assert {p.id for p in page.items} == {"tags-csv-hit"} + + # --------------------------------------------------------------------------- # Field projection # --------------------------------------------------------------------------- From 859f0174e2d4253f760285f76021e06e9bd2ce1f Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 15 Jul 2026 22:17:41 -0700 Subject: [PATCH 17/62] Formatting --- mpcontribs-api/src/mpcontribs_api/dependencies.py | 1 - .../src/mpcontribs_api/domains/project_groups/router.py | 4 +++- .../src/mpcontribs_api/domains/project_groups/service.py | 4 +--- mpcontribs-api/src/mpcontribs_api/exceptions.py | 1 + 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/dependencies.py b/mpcontribs-api/src/mpcontribs_api/dependencies.py index c13c84643a..0e0aa03278 100644 --- a/mpcontribs-api/src/mpcontribs_api/dependencies.py +++ b/mpcontribs-api/src/mpcontribs_api/dependencies.py @@ -71,4 +71,3 @@ def require_user(user: UserDep) -> User: if user.is_anonymous: raise AuthenticationError("authentication required") return user - diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py index a27b485b34..68ee6afdb5 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py @@ -58,7 +58,9 @@ async def get_project_group( return await repo.get_project_group(name=name, owner=owner, fields=selected) -@router.post("", response_model=ProjectGroupOut, status_code=status.HTTP_201_CREATED, dependencies=[Depends(require_user)]) +@router.post( + "", response_model=ProjectGroupOut, status_code=status.HTTP_201_CREATED, dependencies=[Depends(require_user)] +) async def insert_project_group( repo: ProjectGroupDep, project_group: ProjectGroupIn, diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py index f059ea19c2..0aa2151943 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py @@ -68,9 +68,7 @@ async def _add(self, group: ProjectGroupOut, project_ids: list[ShortStr]) -> Bul async def _delete(self, group: ProjectGroupOut, project_ids: list[ShortStr]) -> BulkWriteSummary[str]: """Delete requested members from the group; non-members are reported as failed items.""" - current = { - (link.ref.id if isinstance(link, Link) else link.id) for link in (group.projects or []) - } + current = {(link.ref.id if isinstance(link, Link) else link.id) for link in (group.projects or [])} failed: list[BulkFailure] = [] present: list[ShortStr] = [] for index, pid in enumerate(project_ids): diff --git a/mpcontribs-api/src/mpcontribs_api/exceptions.py b/mpcontribs-api/src/mpcontribs_api/exceptions.py index f6dfd32040..bcfc098235 100644 --- a/mpcontribs-api/src/mpcontribs_api/exceptions.py +++ b/mpcontribs-api/src/mpcontribs_api/exceptions.py @@ -91,6 +91,7 @@ class DownloadError(AppError): error_code = "download_error" log_level = logging.WARNING + def error_body(error_code: str, message: str, **public_context) -> dict: body: dict[str, Any] = {"error": {"code": error_code, "message": message}} if public_context: From 4aba4fdfc7d5ff63ae82432526f858e1a5a3933e Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 15 Jul 2026 22:30:14 -0700 Subject: [PATCH 18/62] Added is_public field to ProjectGroup models and fixed scope --- .../src/mpcontribs_api/domains/project_groups/models.py | 9 +++++++-- .../mpcontribs_api/domains/project_groups/repository.py | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py index 4c2546d81f..d7793250a9 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py @@ -14,6 +14,7 @@ class ProjectGroup(BaseDocumentWithInput[PydanticObjectId]): name: SearchStr = Field(max_length=50) owner: PrefixedEmail + is_public: bool = False projects: list[Link[Project]] description: str = Field(max_length=100) @@ -21,8 +22,8 @@ class Settings: name = "project_groups" indexes = [ IndexModel( - keys=[("name", ASCENDING), ("owner", ASCENDING)], - name="name_owner", + keys=[("name", ASCENDING), ("owner", ASCENDING), ("is_public", ASCENDING)], + name="name_owner_is_public", unique=True, ) ] @@ -58,6 +59,7 @@ class ProjectGroupIn(ProjectGroup): class ProjectGroupOut(DocumentOut[PydanticObjectId]): name: SearchStr | None = None owner: PrefixedEmail | None = None + is_public: bool | None = None projects: list[Link[Project]] | None = None description: str | None = None @@ -73,6 +75,7 @@ def default_fields() -> list[str]: class ProjectGroupPatch(SparseFieldsModel): name: SearchStr | None = None owner: PrefixedEmail | None = None + is_public: bool | None = None projects: list[Link[Project]] | None = None description: str | None = None @@ -96,6 +99,8 @@ class ProjectGroupFilter(BaseFilter): owner__in: list[PrefixedEmail] | None = None owner__neq: PrefixedEmail | None = None + is_public: bool | None = None + order_by: list[str] | None = None class Constants(BaseFilter.Constants): diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py index eee5c19ced..d0f96009d0 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py @@ -30,7 +30,7 @@ def _build_scope(user: User) -> dict[str, Any]: """Provides scope based on current user's permitted groups and publicly released data.""" if user.is_admin: return {} - ors: list[dict[str, Any]] = [{"is_public": True, "is_approved": True}] + ors: list[dict[str, Any]] = [{"is_public": True}] if not user.is_anonymous: ors.append({"owner": user.username}) if user.groups: From cec41adce787f3bc037a20f219010155a18c3794 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 15 Jul 2026 23:03:27 -0700 Subject: [PATCH 19/62] ProjectGroup converts from input model and ProjectGroupIn has proper fields (no id, list of str for projects) --- .../domains/project_groups/models.py | 34 +++++++++++++++---- .../db/test_project_groups_repository.py | 29 +++++++++++++++- .../db/test_project_groups_service.py | 2 +- 3 files changed, 56 insertions(+), 9 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py index d7793250a9..79d0fea8d2 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py @@ -1,6 +1,7 @@ from beanie import Link, PydanticObjectId +from bson import DBRef from bson.errors import InvalidId -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator from pymongo import ASCENDING, IndexModel from mpcontribs_api.domains._shared.filters import BaseFilter @@ -14,9 +15,9 @@ class ProjectGroup(BaseDocumentWithInput[PydanticObjectId]): name: SearchStr = Field(max_length=50) owner: PrefixedEmail - is_public: bool = False - projects: list[Link[Project]] description: str = Field(max_length=100) + is_public: bool = False + projects: list[Link[Project]] | None = None class Settings: name = "project_groups" @@ -34,12 +35,23 @@ def identifier_fields(cls) -> frozenset[str]: """A ``ProjectGroup`` is uniquely identified by its ``name`` + ``owner``.""" return frozenset({"name", "owner"}) + @classmethod + def from_input_model(cls, data: ProjectGroupIn) -> ProjectGroup: + """Build a stored group from input, assigning a fresh ``_id`` and resolving member ids to links""" + payload = data.model_dump() + project_ids = payload.pop("projects", None) or [] + payload["_id"] = PydanticObjectId() + payload["projects"] = [DBRef("projects", pid) for pid in project_ids] + return cls.model_validate(payload) + @field_validator("projects") @classmethod def _reject_duplicate_refs( cls, - value: list[Link[Project] | Project], - ) -> list[Link[Project] | Project]: + value: list[Link[Project] | Project] | None, + ) -> list[Link[Project] | Project] | None: + if value is None: + return value seen: set[ShortStr] = set() for item in value: ref_id = item.ref.id if isinstance(item, Link) else item.id @@ -52,8 +64,16 @@ def _reject_duplicate_refs( return value -class ProjectGroupIn(ProjectGroup): - pass +class ProjectGroupIn(BaseModel): + """User-supplied fields for creating a project group""" + + model_config = ConfigDict(extra="forbid") + + name: SearchStr = Field(max_length=50) + owner: PrefixedEmail + description: str = Field(max_length=100) + is_public: bool = False + projects: list[ShortStr] = Field(default_factory=list) class ProjectGroupOut(DocumentOut[PydanticObjectId]): diff --git a/mpcontribs-api/tests/integration/db/test_project_groups_repository.py b/mpcontribs-api/tests/integration/db/test_project_groups_repository.py index b247c66043..8488b85bdc 100644 --- a/mpcontribs-api/tests/integration/db/test_project_groups_repository.py +++ b/mpcontribs-api/tests/integration/db/test_project_groups_repository.py @@ -32,7 +32,6 @@ def _repo(user: User) -> ProjectGroupRepository: def _group_in(name: str, owner: str = ALICE_EMAIL, **overrides) -> ProjectGroupIn: defaults = { - "_id": PydanticObjectId(), "name": name, "owner": owner, "projects": [], @@ -157,3 +156,31 @@ async def test_no_match_returns_zero(self, db): filter=ProjectGroupFilter(owner="google:nobody@example.com") ) assert result.num_deleted == 0 + + +# --------------------------------------------------------------------------- +# insert_project_group +# +# The input model carries no ``_id`` (the server assigns the ObjectId) and takes +# plain project ids, which from_input_model resolves into stored Links/DBRefs. +# --------------------------------------------------------------------------- + + +class TestInsertProjectGroup: + async def test_assigns_object_id(self, db): + group = await _insert("ins-oid") + assert isinstance(group.id, PydanticObjectId) + + async def test_resolves_project_ids_to_links(self, db): + await _insert("ins-with-projects", projects=["mp-alpha", "mp-beta"]) + doc = await ProjectGroup.find_one(ProjectGroup.name == "ins-with-projects") + assert doc is not None + assert doc.projects is not None + assert {link.ref.collection for link in doc.projects} == {"projects"} + assert sorted(link.ref.id for link in doc.projects) == ["mp-alpha", "mp-beta"] + + async def test_empty_projects_default(self, db): + await _insert("ins-no-projects") + doc = await ProjectGroup.find_one(ProjectGroup.name == "ins-no-projects") + assert doc is not None + assert doc.projects == [] diff --git a/mpcontribs-api/tests/integration/db/test_project_groups_service.py b/mpcontribs-api/tests/integration/db/test_project_groups_service.py index d9c034f420..dfd81ddcda 100644 --- a/mpcontribs-api/tests/integration/db/test_project_groups_service.py +++ b/mpcontribs-api/tests/integration/db/test_project_groups_service.py @@ -45,7 +45,7 @@ async def _insert_project(pid: str, owner: str = ALICE_EMAIL, **overrides): async def _insert_group(name: str, owner: str = ALICE_EMAIL) -> ProjectGroup: return await ProjectGroupRepository(ADMIN).insert_project_group( - ProjectGroupIn(_id=PydanticObjectId(), name=name, owner=owner, projects=[], description="d") + ProjectGroupIn(name=name, owner=owner, projects=[], description="d") ) From 76fb5d31f2571f0c704d3407f3abdad40c08aceb Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Thu, 16 Jul 2026 09:33:50 -0700 Subject: [PATCH 20/62] Resolved issue where POST/PUT did not return a document with server-defined fields (ie. id) filled in --- .../src/mpcontribs_api/domains/_shared/models.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/_shared/models.py b/mpcontribs-api/src/mpcontribs_api/domains/_shared/models.py index d055d27a3c..7051658f9e 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/_shared/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/_shared/models.py @@ -5,7 +5,7 @@ from typing import Annotated, Any, ClassVar, Self from beanie import Document, PydanticObjectId -from pydantic import BaseModel, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, model_validator from pymongo.results import DeleteResult from mpcontribs_api import pagination @@ -59,8 +59,10 @@ class DocumentOut[TId](SparseFieldsModel): Mirrors :class:`BaseDocumentWithInput`: subclasses bind their id type as ``TId`` so each resource owns its id type, while the field (optional, since projections may omit it) and its alias wiring - are declared once here for the repository to read off any resource's output model. - """ + are declared once here for the repository to read off any resource's output model.""" + + # lets POST/PUT responses correctly bring ``_id`` into ``id``, without it ``id`` ends up as None + model_config = ConfigDict(populate_by_name=True) id: Annotated[TId | None, Field(alias="_id", serialization_alias="id")] = None From b040ad05a9bce90fab80c60a33e75d8e69d812cf Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Thu, 16 Jul 2026 09:38:17 -0700 Subject: [PATCH 21/62] Improved error catching on _shared/repo insert_one --- .../mpcontribs_api/domains/_shared/repository.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py index 76a7e8bf45..c1a65c62e0 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py @@ -14,6 +14,7 @@ from fastapi_filter.contrib.beanie import Filter from pydantic import BaseModel from pymongo.asynchronous.client_session import AsyncClientSession +from pymongo.errors import DuplicateKeyError from types_aiobotocore_s3 import S3Client from mpcontribs_api.authz import User @@ -190,16 +191,21 @@ async def list_ids(self, filter: TFilter, session: AsyncClientSession | None = N return [doc.id for doc in docs] async def insert_one(self, in_resource: TIn) -> TDoc: - """Insert a new document built from its input model, rejecting duplicate ids. + """Insert a new document built from its input model, rejecting an existing duplicate. + + Duplicates are determined by model-declared identifiers that uniquely identify a document. Args: in_resource (TIn): the validated input payload to translate and store """ document = self.document_model.from_input_model(in_resource) - existing = await self.document_model.find_one(self.document_model.id == document.id) - if existing: - raise ConflictError(f"Cannot insert document.\n Document with ID {document.id} exists") - await document.insert() + try: + await document.insert() + except DuplicateKeyError as exc: + raise ConflictError( + f"Cannot insert {self.document_model.__name__}: a conflicting document already exists", + identifiers=document.identifiers(), + ) from exc return document async def delete(self, filter: TFilter, session: AsyncClientSession | None = None) -> DeleteResponse: From 0355ea55ea31557116f4db808da125116ae092f0 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Thu, 16 Jul 2026 09:38:53 -0700 Subject: [PATCH 22/62] Improved index structure: unique key is now minimal --- .../src/mpcontribs_api/domains/project_groups/models.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py index 79d0fea8d2..2788c16aa0 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/models.py @@ -22,11 +22,15 @@ class ProjectGroup(BaseDocumentWithInput[PydanticObjectId]): class Settings: name = "project_groups" indexes = [ + IndexModel( + keys=[("name", ASCENDING), ("owner", ASCENDING)], + name="name_owner", + unique=True, + ), IndexModel( keys=[("name", ASCENDING), ("owner", ASCENDING), ("is_public", ASCENDING)], name="name_owner_is_public", - unique=True, - ) + ), ] validate_on_save = True From 0a7a2ff242f19ade91244bf93c9818962d60226c Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Thu, 16 Jul 2026 09:39:30 -0700 Subject: [PATCH 23/62] Added tests for project-group duplication --- .../db/test_project_groups_repository.py | 13 ++++ .../tests/integration/test_project_groups.py | 76 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 mpcontribs-api/tests/integration/test_project_groups.py diff --git a/mpcontribs-api/tests/integration/db/test_project_groups_repository.py b/mpcontribs-api/tests/integration/db/test_project_groups_repository.py index 8488b85bdc..82723fd30e 100644 --- a/mpcontribs-api/tests/integration/db/test_project_groups_repository.py +++ b/mpcontribs-api/tests/integration/db/test_project_groups_repository.py @@ -184,3 +184,16 @@ async def test_empty_projects_default(self, db): doc = await ProjectGroup.find_one(ProjectGroup.name == "ins-no-projects") assert doc is not None assert doc.projects == [] + + async def test_duplicate_identifiers_raise_conflict(self, db): + # A ProjectGroup's identity is name + owner, not its server-assigned _id (a fresh ObjectId is + # minted per insert). insert_one must reject a second group with the same name+owner cleanly. + await _insert("ins-dup") + with pytest.raises(ConflictError): + await _insert("ins-dup") + + async def test_same_name_different_owner_allowed(self, db): + # name alone is not the identity: the same name under a different owner is a distinct group. + await _insert("ins-shared-name", owner=ALICE_EMAIL) + other = await _insert("ins-shared-name", owner="google:bob@example.com") + assert other.owner == "google:bob@example.com" diff --git a/mpcontribs-api/tests/integration/test_project_groups.py b/mpcontribs-api/tests/integration/test_project_groups.py new file mode 100644 index 0000000000..35726048b7 --- /dev/null +++ b/mpcontribs-api/tests/integration/test_project_groups.py @@ -0,0 +1,76 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from mpcontribs_api.domains.project_groups.dependencies import get_project_group_repository +from tests.integration.conftest import AUTHED_HEADERS + +# A valid 24-char hex ObjectId string (ProjectGroupOut.id is a PydanticObjectId). +SAMPLE_OID = "6eb7cf5a86d9755df3a6c593" + + +@pytest.fixture +def group_repo(test_app): + repo = AsyncMock() + test_app.dependency_overrides[get_project_group_repository] = lambda: repo + yield repo + test_app.dependency_overrides.pop(get_project_group_repository, None) + + +# --------------------------------------------------------------------------- +# POST /api/v1/project_groups +# +# The handler returns the stored document, which FastAPI coerces into +# ProjectGroupOut with from_attributes=True. A Beanie Document exposes ``.id`` +# (not ``._id``), so the response model must populate its id by field name as +# well as the ``_id`` alias — otherwise every create/update response would +# serialise ``id: null`` and callers couldn't see what id their group got. +# --------------------------------------------------------------------------- + + +class TestInsertProjectGroupResponse: + def _body(self, **overrides): + body = { + "name": "my-group", + "owner": "google:alice@example.com", + "description": "d", + "projects": [], + } + body.update(overrides) + return body + + def _inserted(self, **overrides): + """Stand-in for the stored document: exposes ``.id`` like a Beanie Document.""" + attrs = { + "id": SAMPLE_OID, + "name": "my-group", + "owner": "google:alice@example.com", + "is_public": False, + "projects": None, + "description": "d", + } + attrs.update(overrides) + return SimpleNamespace(**attrs) + + def test_returns_201(self, client, group_repo): + group_repo.insert_project_group.return_value = self._inserted() + r = client.post("/api/v1/project_groups", json=self._body(), headers=AUTHED_HEADERS) + assert r.status_code == 201 + + def test_response_includes_generated_id(self, client, group_repo): + group_repo.insert_project_group.return_value = self._inserted() + body = client.post("/api/v1/project_groups", json=self._body(), headers=AUTHED_HEADERS).json() + assert body["id"] == SAMPLE_OID + + def test_response_echoes_full_document(self, client, group_repo): + group_repo.insert_project_group.return_value = self._inserted(name="echo-group", is_public=True) + body = client.post( + "/api/v1/project_groups", + json=self._body(name="echo-group", is_public=True), + headers=AUTHED_HEADERS, + ).json() + assert body["id"] == SAMPLE_OID + assert body["name"] == "echo-group" + assert body["owner"] == "google:alice@example.com" + assert body["is_public"] is True From 625896833f67fe4545ae74fbc519f07dbf4f2bb1 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Thu, 16 Jul 2026 09:49:27 -0700 Subject: [PATCH 24/62] Made inserting ProjectGroups a service-level concern by requiring that projects in a group during POST must be valid projects in a user's scope --- .../domains/project_groups/router.py | 9 +++-- .../domains/project_groups/service.py | 15 ++++++-- .../tests/integration/test_project_groups.py | 24 ++++++------- .../domains/test_project_group_service.py | 34 ++++++++++++++++++- 4 files changed, 64 insertions(+), 18 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py index 68ee6afdb5..70a783b526 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py @@ -62,16 +62,19 @@ async def get_project_group( "", response_model=ProjectGroupOut, status_code=status.HTTP_201_CREATED, dependencies=[Depends(require_user)] ) async def insert_project_group( - repo: ProjectGroupDep, + service: ProjectGroupServiceDep, project_group: ProjectGroupIn, ): """Insert a new project group. + Each referenced project is verified against the projects collection (scoped to the caller); + creation is rejected with 404 if any project id is unknown or not visible. + Args: - repo (ProjectGroupDep): the project group repo we depend on + service (ProjectGroupServiceDep): the project group service we depend on project_group (ProjectGroupIn): the project group to insert """ - return await repo.insert_project_group(project_group=project_group) + return await service.insert(project_group=project_group) @router.patch("/item", response_model=ProjectGroupOut, dependencies=[Depends(require_user)]) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py index 0aa2151943..5976da27ca 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py @@ -2,7 +2,7 @@ from mpcontribs_api.domains._shared.bulk import BulkFailure, BulkWriteSummary from mpcontribs_api.domains._shared.types import PrefixedEmail, SearchStr, ShortStr -from mpcontribs_api.domains.project_groups.models import ProjectGroupOut +from mpcontribs_api.domains.project_groups.models import ProjectGroup, ProjectGroupIn, ProjectGroupOut from mpcontribs_api.domains.project_groups.repository import ProjectGroupRepository from mpcontribs_api.domains.projects.repository import MongoDbProjectRepository from mpcontribs_api.exceptions import NotFoundError @@ -23,6 +23,17 @@ def __init__( self._groups = groups self._projects = projects + async def _project_exists(self, project_id: ShortStr) -> bool: + """Whether a project with ``project_id`` exists and is visible to the caller.""" + return await self._projects.get_by_id(project_id, fields=frozenset({"id"})) is not None + + async def insert(self, project_group: ProjectGroupIn) -> ProjectGroup: + """Insert a new group after verifying every referenced project exists and is visible""" + missing = [pid for pid in project_group.projects if not await self._project_exists(pid)] + if missing: + raise NotFoundError("One or more projects not found or not visible", ids=missing) + return await self._groups.insert_project_group(project_group) + async def _resolve_by_id(self, group_id: str) -> ProjectGroupOut: """Resolve a visible group by its ObjectId, or raise ``NotFoundError``.""" oid = self._groups._convert_object_id(group_id) @@ -50,7 +61,7 @@ async def _add(self, group: ProjectGroupOut, project_ids: list[ShortStr]) -> Bul failed: list[BulkFailure] = [] valid: list[ShortStr] = [] for index, pid in enumerate(project_ids): - if await self._projects.get_by_id(pid, fields=frozenset({"id"})) is None: + if not await self._project_exists(pid): failed.append( BulkFailure( index=index, diff --git a/mpcontribs-api/tests/integration/test_project_groups.py b/mpcontribs-api/tests/integration/test_project_groups.py index 35726048b7..e12ff146fd 100644 --- a/mpcontribs-api/tests/integration/test_project_groups.py +++ b/mpcontribs-api/tests/integration/test_project_groups.py @@ -3,7 +3,7 @@ import pytest -from mpcontribs_api.domains.project_groups.dependencies import get_project_group_repository +from mpcontribs_api.domains.project_groups.dependencies import get_project_group_service from tests.integration.conftest import AUTHED_HEADERS # A valid 24-char hex ObjectId string (ProjectGroupOut.id is a PydanticObjectId). @@ -11,11 +11,11 @@ @pytest.fixture -def group_repo(test_app): - repo = AsyncMock() - test_app.dependency_overrides[get_project_group_repository] = lambda: repo - yield repo - test_app.dependency_overrides.pop(get_project_group_repository, None) +def group_service(test_app): + service = AsyncMock() + test_app.dependency_overrides[get_project_group_service] = lambda: service + yield service + test_app.dependency_overrides.pop(get_project_group_service, None) # --------------------------------------------------------------------------- @@ -53,18 +53,18 @@ def _inserted(self, **overrides): attrs.update(overrides) return SimpleNamespace(**attrs) - def test_returns_201(self, client, group_repo): - group_repo.insert_project_group.return_value = self._inserted() + def test_returns_201(self, client, group_service): + group_service.insert.return_value = self._inserted() r = client.post("/api/v1/project_groups", json=self._body(), headers=AUTHED_HEADERS) assert r.status_code == 201 - def test_response_includes_generated_id(self, client, group_repo): - group_repo.insert_project_group.return_value = self._inserted() + def test_response_includes_generated_id(self, client, group_service): + group_service.insert.return_value = self._inserted() body = client.post("/api/v1/project_groups", json=self._body(), headers=AUTHED_HEADERS).json() assert body["id"] == SAMPLE_OID - def test_response_echoes_full_document(self, client, group_repo): - group_repo.insert_project_group.return_value = self._inserted(name="echo-group", is_public=True) + def test_response_echoes_full_document(self, client, group_service): + group_service.insert.return_value = self._inserted(name="echo-group", is_public=True) body = client.post( "/api/v1/project_groups", json=self._body(name="echo-group", is_public=True), diff --git a/mpcontribs-api/tests/unit/domains/test_project_group_service.py b/mpcontribs-api/tests/unit/domains/test_project_group_service.py index 8385b6c7a9..d1de793a28 100644 --- a/mpcontribs-api/tests/unit/domains/test_project_group_service.py +++ b/mpcontribs-api/tests/unit/domains/test_project_group_service.py @@ -4,7 +4,7 @@ from beanie import Link, PydanticObjectId from bson import DBRef -from mpcontribs_api.domains.project_groups.models import ProjectGroupOut +from mpcontribs_api.domains.project_groups.models import ProjectGroupIn, ProjectGroupOut from mpcontribs_api.domains.project_groups.service import ProjectGroupService from mpcontribs_api.domains.projects.models import Project from mpcontribs_api.exceptions import ConflictError, NotFoundError @@ -55,6 +55,38 @@ def _group(project_ids: list[str] | None = None) -> ProjectGroupOut: return group +# --------------------------------------------------------------------------- +# insert +# --------------------------------------------------------------------------- + + +class TestInsert: + def _payload(self, projects: list[str]) -> ProjectGroupIn: + return ProjectGroupIn(name="g", owner="google:a@b.com", description="d", projects=projects) + + async def test_all_projects_valid_inserts(self): + service, groups, _ = _make_service(None, visible_projects={"mp-1", "mp-2"}) + groups.insert_project_group.return_value = "stored" + payload = self._payload(["mp-1", "mp-2"]) + result = await service.insert(payload) + assert result == "stored" + groups.insert_project_group.assert_awaited_once_with(payload) + + async def test_missing_project_raises_not_found_and_skips_insert(self): + service, groups, _ = _make_service(None, visible_projects={"mp-1"}) + with pytest.raises(NotFoundError) as exc: + await service.insert(self._payload(["mp-1", "ghost"])) + assert exc.value.context["ids"] == ["ghost"] + groups.insert_project_group.assert_not_awaited() + + async def test_empty_projects_inserts_without_validation(self): + service, groups, projects = _make_service(None) + payload = self._payload([]) + await service.insert(payload) + projects.get_by_id.assert_not_awaited() + groups.insert_project_group.assert_awaited_once_with(payload) + + # --------------------------------------------------------------------------- # Group resolution # --------------------------------------------------------------------------- From ff4ba2c852ba8a5c81dac7dd227837d6285d6f21 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Thu, 16 Jul 2026 13:22:39 -0700 Subject: [PATCH 25/62] Added settings for controlling initiative permissions --- mpcontribs-api/src/mpcontribs_api/config.py | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/mpcontribs-api/src/mpcontribs_api/config.py b/mpcontribs-api/src/mpcontribs_api/config.py index 3995eebb0e..17175a8d0c 100644 --- a/mpcontribs-api/src/mpcontribs_api/config.py +++ b/mpcontribs-api/src/mpcontribs_api/config.py @@ -157,6 +157,26 @@ def _clamp_concurrency(self): return self +class InitiativeSettings(BaseModel): + """Limits governing user-owned initiatives.""" + + max_unapproved_per_owner: int = Field( + default=3, + description="Maximum number of unapproved initiatives a single owner may have at once. Enforced on create.", + ) + max_projects_per_unapproved: int = Field( + default=2, + description="Maximum number of projects that may be assigned to an unapproved initiative. Enforced when a " + "project's initiative is set via PATCH.", + ) + + +class DomainSettings(BaseModel): + """Settings to configure the domain logic of MPContribs""" + + initiatives: InitiativeSettings = Field(default_factory=InitiativeSettings) + + class Settings(BaseSettings): model_config = SettingsConfigDict( env_file=".env", @@ -179,6 +199,9 @@ class Settings(BaseSettings): # MPContribs_otel__* otel: ObservabilitySettings = Field(default_factory=ObservabilitySettings) + # MPContribs_domain_* + domain: DomainSettings = Field(default_factory=DomainSettings) + # SMTP Settings mail_default_sender: str = Field( description="SMTP Server to send out notifications on new projects and other important moments" From 66a784e28eea00185199c66b5d9a574fa923b478 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Thu, 16 Jul 2026 13:23:30 -0700 Subject: [PATCH 26/62] Added flexibility to roles, allowing prefixing with resource --- mpcontribs-api/src/mpcontribs_api/authz.py | 34 +++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/mpcontribs-api/src/mpcontribs_api/authz.py b/mpcontribs-api/src/mpcontribs_api/authz.py index 2a56f7dc65..27fd78de63 100644 --- a/mpcontribs-api/src/mpcontribs_api/authz.py +++ b/mpcontribs-api/src/mpcontribs_api/authz.py @@ -33,6 +33,9 @@ ADMIN_GROUP = settings.mongo.admin_group +# prefix to user roles to disambiguate from project roles, which are bare ids +INITIATIVE_ROLE_PREFIX = "initiative:" + class User(BaseModel): """User definition derived from request headers. @@ -64,7 +67,29 @@ def is_anonymous(self) -> bool: def is_admin(self) -> bool: return (not self.is_anonymous) and (ADMIN_GROUP in self.groups) - def has_role(self, role: str) -> bool: + @property + def project_roles(self) -> list[str]: + return [ + role[len(INITIATIVE_ROLE_PREFIX) :] for role in self.groups if not role.startswith(INITIATIVE_ROLE_PREFIX) + ] + + @property + def initiative_roles(self) -> list[str]: + """The initiative slugs this user collaborates on, decoded from their ``initiative:`` roles.""" + return [role[len(INITIATIVE_ROLE_PREFIX) :] for role in self.groups if role.startswith(INITIATIVE_ROLE_PREFIX)] + + def has_role(self, role: str, *, resource: str | None = None) -> bool: + """Determine whether a user has a role assigned to them. + + Specifying resource as: + - ``INITIATIVE_ROLE_PREFIX`` looks for roles scoped to initiatives + - "project" looks for roles scoped to projects (no actual prefix implementation yet) + - None looks for roles by matching the entire string + """ + if resource == INITIATIVE_ROLE_PREFIX[:-1]: + return role in self.initiative_roles + if resource == "project": + return role in self.project_roles return role in self.groups @property @@ -75,6 +100,13 @@ def writable_projects(self) -> frozenset[str]: # exclude the admin sentinel so it never leaks into a $in / membership test return frozenset(g for g in self.groups if g != ADMIN_GROUP) + def can_manage(self, id: str, resource: str) -> bool: + """Determines whether a user can manage a resource. + + If the user is known and either an admin or has a valid role assigned, they can manage + """ + return (not self.is_anonymous) and (self.is_admin or self.has_role(role=id, resource=resource)) + def can_write(self, project: str) -> bool: """Single source of truth for write authorization.""" return self.is_admin or project in self.writable_projects From 1d1b71ac46ae901f800cc131d782e42ea0d11249 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Thu, 16 Jul 2026 13:25:23 -0700 Subject: [PATCH 27/62] Added Initiative domain, for a canonical grouping of projects (ie ALab) --- .../src/mpcontribs_api/api/v1/router.py | 2 + mpcontribs-api/src/mpcontribs_api/app.py | 2 + .../domains/initiatives/__init__.py | 0 .../domains/initiatives/dependencies.py | 13 ++ .../domains/initiatives/models.py | 151 ++++++++++++++ .../domains/initiatives/repository.py | 140 +++++++++++++ .../domains/initiatives/router.py | 79 ++++++++ .../db/test_initiatives_repository.py | 184 ++++++++++++++++++ .../db/test_initiatives_service.py | 177 +++++++++++++++++ .../tests/integration/test_initiatives.py | 135 +++++++++++++ 10 files changed, 883 insertions(+) create mode 100644 mpcontribs-api/src/mpcontribs_api/domains/initiatives/__init__.py create mode 100644 mpcontribs-api/src/mpcontribs_api/domains/initiatives/dependencies.py create mode 100644 mpcontribs-api/src/mpcontribs_api/domains/initiatives/models.py create mode 100644 mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py create mode 100644 mpcontribs-api/src/mpcontribs_api/domains/initiatives/router.py create mode 100644 mpcontribs-api/tests/integration/db/test_initiatives_repository.py create mode 100644 mpcontribs-api/tests/integration/db/test_initiatives_service.py create mode 100644 mpcontribs-api/tests/integration/test_initiatives.py diff --git a/mpcontribs-api/src/mpcontribs_api/api/v1/router.py b/mpcontribs-api/src/mpcontribs_api/api/v1/router.py index 530751946d..511632c048 100644 --- a/mpcontribs-api/src/mpcontribs_api/api/v1/router.py +++ b/mpcontribs-api/src/mpcontribs_api/api/v1/router.py @@ -2,6 +2,7 @@ from mpcontribs_api.domains.attachments.router import router as attachments_router from mpcontribs_api.domains.contributions.router import router as contributions_router +from mpcontribs_api.domains.initiatives.router import router as initiatives_router from mpcontribs_api.domains.limits.router import router as limits_router from mpcontribs_api.domains.project_groups.router import router as project_groups_router from mpcontribs_api.domains.projects.router import router as projects_router @@ -12,6 +13,7 @@ router.include_router(attachments_router, prefix="/attachments", tags=["attachments"]) router.include_router(contributions_router, prefix="/contributions", tags=["contributions"]) +router.include_router(initiatives_router, prefix="/initiatives", tags=["initiatives"]) router.include_router(limits_router, prefix="/limits", tags=["limits"]) router.include_router(project_groups_router, prefix="/project_groups", tags=["project_groups"]) router.include_router(projects_router, prefix="/projects", tags=["projects"]) diff --git a/mpcontribs-api/src/mpcontribs_api/app.py b/mpcontribs-api/src/mpcontribs_api/app.py index de13195b31..e48efa9d4b 100644 --- a/mpcontribs-api/src/mpcontribs_api/app.py +++ b/mpcontribs-api/src/mpcontribs_api/app.py @@ -21,6 +21,7 @@ from mpcontribs_api.domains.attachments.models import Attachment from mpcontribs_api.domains.contributions.models import Contribution from mpcontribs_api.domains.healthcheck.router import router as healthcheck_router +from mpcontribs_api.domains.initiatives.models import Initiative from mpcontribs_api.domains.project_groups.models import ProjectGroup from mpcontribs_api.domains.projects.models import Project from mpcontribs_api.domains.structures.models import Structure @@ -65,6 +66,7 @@ async def _setup_mongo(app: FastAPI, settings: Settings, stack: AsyncExitStack) document_models=[ Project, ProjectGroup, + Initiative, Contribution, Attachment, Structure, diff --git a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/__init__.py b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/dependencies.py b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/dependencies.py new file mode 100644 index 0000000000..a81d8c8d5d --- /dev/null +++ b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/dependencies.py @@ -0,0 +1,13 @@ +from typing import Annotated + +from fastapi import Depends + +from mpcontribs_api.dependencies import UserDep +from mpcontribs_api.domains.initiatives.repository import InitiativeRepository + + +def get_initiative_repository(user: UserDep) -> InitiativeRepository: + return InitiativeRepository(user) + + +InitiativeDep = Annotated[InitiativeRepository, Depends(get_initiative_repository)] diff --git a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/models.py b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/models.py new file mode 100644 index 0000000000..49914166fc --- /dev/null +++ b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/models.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import re +from typing import Annotated, Self + +from beanie import PydanticObjectId +from bson.errors import InvalidId +from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, field_validator, model_validator +from pymongo import ASCENDING, IndexModel + +from mpcontribs_api.domains._shared.filters import BaseFilter +from mpcontribs_api.domains._shared.models import BaseDocumentWithInput, DocumentOut +from mpcontribs_api.domains._shared.types import NFKCStr, PrefixedEmail +from mpcontribs_api.exceptions import ValidationError +from mpcontribs_api.projection import SparseFieldsModel + +# A URL-safe, human-readable slug +# Also carried in user.groups like ``initiative:`` +_SLUG_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") + + +def _validate_slug(v: str) -> str: + v = v.strip().lower() + if not _SLUG_RE.match(v): + raise ValidationError( + "slug must be lowercase alphanumeric words separated by single hyphens, e.g. 'battery-genome-2025'", + slug=v, + ) + return v + + +Slug = Annotated[str, Field(min_length=3, max_length=50), BeforeValidator(_validate_slug)] + + +class Initiative(BaseDocumentWithInput[PydanticObjectId]): + """A canonical, authoritative grouping of projects into a larger organizational effort. + + Unlike an ad-hoc ``ProjectGroup`` (many-to-many, user-curated), an initiative is the single + canonical parent of its member projects: a project points at *at most one* initiative via + ``Project.initiative``. Membership is therefore derived from the projects collection — an + initiative stores no project list of its own. + + Collaborator rights are drawn from the caller's roles, mirroring how projects use groups: a + user may manage an initiative (add projects, patch it) if they own it, are an admin, or carry + the ``initiative:`` role. + """ + + slug: Slug + name: NFKCStr = Field(max_length=100) + owner: PrefixedEmail + is_public: bool = False + is_approved: bool = False + + class Settings: + name = "initiatives" + keep_nulls = False + indexes = [ + IndexModel(keys=[("slug", ASCENDING)], name="slug", unique=True), + IndexModel( + keys=[("owner", ASCENDING), ("is_approved", ASCENDING), ("is_public", ASCENDING)], + name="owner_is_approved_is_public", + ), + ] + validate_on_save = True + + @classmethod + def identifier_fields(cls) -> frozenset[str]: + """An ``Initiative`` is uniquely identified by its globally-unique ``slug``.""" + return frozenset({"slug"}) + + @model_validator(mode="after") + def _public_requires_approved(self) -> Self: + """An initiative cannot be public until it has been approved.""" + if self.is_public and not self.is_approved: + raise ValidationError("an initiative cannot be public until it is approved", slug=self.slug) + return self + + +class InitiativeIn(BaseModel): + """User-supplied fields for creating an initiative. + + ``owner`` is forced to the caller and ``is_public`` / ``is_approved`` always start ``False`` + (an admin approves later), so none of them are part of the input contract. + """ + + model_config = ConfigDict(extra="forbid") + + slug: Slug + name: NFKCStr = Field(max_length=100) + + +class InitiativeOut(DocumentOut[PydanticObjectId]): + slug: Slug | None = None + name: NFKCStr | None = None + owner: PrefixedEmail | None = None + is_public: bool | None = None + is_approved: bool | None = None + + @staticmethod + def default_fields() -> list[str]: + return ["slug", "name", "owner", "is_public", "is_approved"] + + +class InitiativePatch(SparseFieldsModel): + """Partial update to an initiative. + + ``slug`` and ``owner`` are immutable and intentionally absent. ``is_approved`` is admin-only + (enforced in the repository), and the ``is_public`` ⇒ ``is_approved`` invariant is re-checked + there against the resulting state, since a partial ``$set`` bypasses the document validator. + """ + + name: NFKCStr | None = Field(default=None, max_length=100) + is_public: bool | None = None + is_approved: bool | None = None + + +class InitiativeFilter(BaseFilter): + id: PydanticObjectId | None = None + id__in: list[PydanticObjectId] | None = None + id__neq: PydanticObjectId | None = None + + slug: Slug | None = None + slug__in: list[Slug] | None = None + slug__neq: Slug | None = None + + name: NFKCStr | None = None + name__in: list[NFKCStr] | None = None + name__neq: NFKCStr | None = None + + owner: PrefixedEmail | None = None + owner__in: list[PrefixedEmail] | None = None + owner__neq: PrefixedEmail | None = None + + is_public: bool | None = None + is_approved: bool | None = None + + order_by: list[str] | None = None + + class Constants(BaseFilter.Constants): + model = Initiative + + @field_validator("id", mode="before") + @classmethod + def convert_str_to_oid(cls, v: str): + try: + return PydanticObjectId(v) + except InvalidId as err: + raise ValidationError( + "Invalid ObjectId format. Must be 12-byte input or a 24-character hex string", + oid=v, + ) from err diff --git a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py new file mode 100644 index 0000000000..bce9f706b1 --- /dev/null +++ b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py @@ -0,0 +1,140 @@ +from typing import Any + +from beanie import PydanticObjectId +from pymongo.errors import DuplicateKeyError + +from mpcontribs_api.authz import User +from mpcontribs_api.config import get_settings +from mpcontribs_api.domains._shared.models import DeleteResponse +from mpcontribs_api.domains._shared.repository import MongoDbRepository +from mpcontribs_api.domains.initiatives.models import ( + Initiative, + InitiativeFilter, + InitiativeIn, + InitiativeOut, + InitiativePatch, +) +from mpcontribs_api.exceptions import ConflictError, NotFoundError, PermissionError, ValidationError +from mpcontribs_api.pagination import CursorParams, Page + + +class InitiativeRepository( + MongoDbRepository[Initiative, InitiativeIn, InitiativeOut, InitiativeFilter, InitiativePatch] +): + document_model = Initiative + out_model = InitiativeOut + + def __init__(self, user: User) -> None: + super().__init__(user) + self._user = user + self._limits = get_settings().domain.initiatives + + @staticmethod + def _build_scope(user: User) -> dict[str, Any]: + """Scope reads to what the caller may see: public+approved, owned, or collaborated-on.""" + if user.is_admin: + return {} + ors: list[dict[str, Any]] = [{"is_public": True, "is_approved": True}] + if not user.is_anonymous: + ors.append({"owner": user.username}) + slugs = user.initiative_roles + if slugs: + ors.append({"slug": {"$in": sorted(slugs)}}) + return {"$or": ors} + + async def get_initiatives( + self, + pagination: CursorParams, + filter: InitiativeFilter, + fields: frozenset[str] | None, + ) -> Page[InitiativeOut]: + """Return a scoped, filtered, paginated page of initiatives. See ``get_many``.""" + return await self.get_many(pagination=pagination, filter=filter, fields=fields) + + async def get_initiative(self, slug: str, fields: frozenset[str] | None) -> InitiativeOut | None: + """Return the single scoped initiative identified by ``slug``. See ``get_one``.""" + return await self.get_one({"slug": slug}, fields) + + async def resolve_visible(self, slug: str) -> Initiative | None: + """Return the full scoped initiative document for ``slug`` (or None), for write-path checks.""" + return await self.document_model.find_one(self._scope, self.document_model.slug == slug) + + async def insert_initiative(self, data: InitiativeIn) -> Initiative: + """Create an initiative owned by the caller, enforcing the per-owner unapproved quota. + + ``owner`` is forced to the caller and the initiative starts unapproved and private. A + non-admin who already owns ``max_unapproved_per_owner`` unapproved initiatives is rejected + with 409. A duplicate ``slug`` (globally unique) is also a 409. + """ + if self._user.username is None: + raise PermissionError(required_role="authenticated") + + if not self._user.is_admin: + unapproved = await self.document_model.find( + self.document_model.owner == self._user.username, + self.document_model.is_approved == False, # noqa: E712 — Beanie needs the value, not `is` + ).count() + if unapproved >= self._limits.max_unapproved_per_owner: + raise ConflictError( + "owner already has the maximum number of unapproved initiatives", + limit=self._limits.max_unapproved_per_owner, + ) + + # ``BaseDocumentWithInput`` makes ``id`` required (no auto-default), so mint the ObjectId + # here — as ``ProjectGroup.from_input_model`` does — and force owner/flags server-side. + initiative = self.document_model.model_validate( + { + "_id": PydanticObjectId(), + "slug": data.slug, + "name": data.name, + "owner": self._user.username, + } + ) + try: + await initiative.insert() + except DuplicateKeyError as exc: # unique slug index + raise ConflictError("an initiative with this slug already exists", slug=data.slug) from exc + return initiative + + async def patch_initiative(self, slug: str, update: InitiativePatch) -> Initiative: + """Patch a scoped initiative by ``slug``, enforcing manage rights and approval rules. + + - The caller must be able to *manage* the initiative (owner/collaborator/admin); mere + visibility (e.g. a public initiative) is not enough. + - Only an admin may change ``is_approved``. + - The resulting state must satisfy ``is_public ⇒ is_approved`` (re-checked here because a + partial ``$set`` does not run the document validator). + """ + existing = await self.resolve_visible(slug) + if existing is None: + raise NotFoundError("Initiative not found", slug=slug) + if not ( + self._user.can_manage(id=existing.slug, resource="initiative") + or self._user.username == self.document_model.owner + ): + raise PermissionError(required_role="initiative-owner-collaborator-or-admin") + + data = update.model_dump(exclude_unset=True) + if "is_approved" in data and not self._user.is_admin: + raise PermissionError(required_role="admin") + + resulting_approved = data.get("is_approved", existing.is_approved) + resulting_public = data.get("is_public", existing.is_public) + if resulting_public and not resulting_approved: + raise ValidationError("an initiative cannot be public until it is approved", slug=slug) + + return await self.patch(existing.id, update) + + async def delete_initiative(self, slug: str) -> DeleteResponse: + """Delete a scoped initiative by ``slug``. Restricted to the owner or an admin. + + Collaborators may contribute projects but may not dissolve the effort. Deleting an + initiative does not touch member projects; their ``initiative`` link simply dangles until + re-pointed (reads resolve a missing link to null). + """ + existing = await self.resolve_visible(slug) + if existing is None: + raise NotFoundError("Initiative not found", slug=slug) + if not (self._user.is_admin or existing.owner == self._user.username): + raise PermissionError(required_role="owner-or-admin") + return await self.delete_by_id(existing.id) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/router.py b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/router.py new file mode 100644 index 0000000000..844818fd50 --- /dev/null +++ b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/router.py @@ -0,0 +1,79 @@ +from typing import Annotated + +from fastapi import APIRouter, Depends, Response, status +from fastapi_filter import FilterDepends + +from mpcontribs_api.dependencies import require_user +from mpcontribs_api.domains._shared.types import FieldSelector +from mpcontribs_api.domains.initiatives.dependencies import InitiativeDep +from mpcontribs_api.domains.initiatives.models import ( + InitiativeFilter, + InitiativeIn, + InitiativeOut, + InitiativePatch, +) +from mpcontribs_api.pagination import CursorParams + +router = APIRouter() + + +@router.get("") +async def get_initiatives( + repo: InitiativeDep, + pagination: Annotated[CursorParams, Depends()], + filter: InitiativeFilter = FilterDepends(InitiativeFilter), + fields: FieldSelector = InitiativeOut.default_fields(), +): + """Return paginated initiatives matching a filter, scoped to the caller.""" + selected = InitiativeOut.parse_fields(fields) + return await repo.get_initiatives(pagination=pagination, filter=filter, fields=selected) + + +@router.get("/{slug}") +async def get_initiative( + repo: InitiativeDep, + slug: str, + fields: FieldSelector = InitiativeOut.default_fields(), +): + """Return the single initiative identified by ``slug``, scoped to the caller.""" + selected = InitiativeOut.parse_fields(fields) + return await repo.get_initiative(slug=slug, fields=selected) + + +@router.post( + "", response_model=InitiativeOut, status_code=status.HTTP_201_CREATED, dependencies=[Depends(require_user)] +) +async def insert_initiative( + repo: InitiativeDep, + initiative: InitiativeIn, +): + """Create a new initiative owned by the caller. + + Starts unapproved and private. Rejected with 409 if the caller already owns the maximum number + of unapproved initiatives, or if the slug is already taken. + """ + return await repo.insert_initiative(data=initiative) + + +@router.patch("/{slug}", response_model=InitiativeOut, dependencies=[Depends(require_user)]) +async def patch_initiative( + repo: InitiativeDep, + slug: str, + update: InitiativePatch, +): + """Partially update the initiative identified by ``slug``. + + Requires manage rights (owner/collaborator/admin). ``is_approved`` is admin-only, and an + initiative cannot be made public until it is approved. + """ + return await repo.patch_initiative(slug=slug, update=update) + + +@router.delete("/{slug}", status_code=status.HTTP_204_NO_CONTENT, dependencies=[Depends(require_user)]) +async def delete_initiative( + repo: InitiativeDep, + slug: str, +): + """Delete the initiative identified by ``slug``. Restricted to its owner or an admin.""" + await repo.delete_initiative(slug=slug) + return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/mpcontribs-api/tests/integration/db/test_initiatives_repository.py b/mpcontribs-api/tests/integration/db/test_initiatives_repository.py new file mode 100644 index 0000000000..f36f2a995a --- /dev/null +++ b/mpcontribs-api/tests/integration/db/test_initiatives_repository.py @@ -0,0 +1,184 @@ +import pytest + +from mpcontribs_api.authz import User +from mpcontribs_api.config import get_settings +from mpcontribs_api.domains.initiatives.models import Initiative, InitiativeIn, InitiativePatch +from mpcontribs_api.domains.initiatives.repository import InitiativeRepository +from mpcontribs_api.exceptions import ConflictError, NotFoundError, PermissionError, ValidationError + +# Share the session event loop (see the projects repo test for why). +pytestmark = [pytest.mark.db, pytest.mark.asyncio(loop_scope="session")] + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +ADMIN = User(username="google:admin@example.com", groups=frozenset({"admin"})) +ALICE = User(username="google:alice@example.com", groups=frozenset({"mp-team"})) +BOB = User(username="google:bob@example.com", groups=frozenset({"mp-team"})) +ANON = User() + +ALICE_EMAIL = "google:alice@example.com" +BOB_EMAIL = "google:bob@example.com" + + +def _repo(user: User) -> InitiativeRepository: + return InitiativeRepository(user) + + +def _collaborator(slug: str, username: str = BOB_EMAIL) -> User: + """A user whose role grants them collaborator rights on ``slug``.""" + return User(username=username, groups=frozenset({f"initiative:{slug}"})) + + +async def _insert(slug: str, owner_user: User = ALICE, name: str = "An Initiative") -> Initiative: + return await _repo(owner_user).insert_initiative(InitiativeIn(slug=slug, name=name)) + + +async def _approve(slug: str) -> Initiative: + return await _repo(ADMIN).patch_initiative(slug, InitiativePatch(is_approved=True)) + + +# --------------------------------------------------------------------------- +# Create + owner forcing +# --------------------------------------------------------------------------- + + +class TestInsert: + async def test_forces_owner_and_starts_private_unapproved(self, db): + created = await _insert("battery-genome", ALICE) + assert created.owner == ALICE_EMAIL + assert created.is_public is False + assert created.is_approved is False + + async def test_duplicate_slug_is_conflict(self, db): + await _insert("dup-slug", ALICE) + with pytest.raises(ConflictError): + await _insert("dup-slug", BOB) # globally unique, even across owners + + async def test_anonymous_cannot_create(self, db): + with pytest.raises(PermissionError): + await _repo(ANON).insert_initiative(InitiativeIn(slug="anon-init", name="x")) + + async def test_invalid_slug_rejected(self, db): + with pytest.raises(ValidationError): + InitiativeIn(slug="Not A Slug!", name="x") + + +class TestUnapprovedPerOwnerLimit: + async def test_owner_capped_at_configured_unapproved(self, db): + limit = get_settings().initiatives.max_unapproved_per_owner + for i in range(limit): + await _insert(f"cap-{i}", ALICE) + with pytest.raises(ConflictError): + await _insert("cap-over", ALICE) + + async def test_approved_do_not_count_against_quota(self, db): + limit = get_settings().initiatives.max_unapproved_per_owner + for i in range(limit): + await _insert(f"quota-{i}", ALICE) + await _approve("quota-0") # frees a slot + # A fresh unapproved initiative now fits again. + assert await _insert("quota-extra", ALICE) is not None + + async def test_admin_is_exempt(self, db): + limit = get_settings().initiatives.max_unapproved_per_owner + for i in range(limit + 2): + await _repo(ADMIN).insert_initiative(InitiativeIn(slug=f"admin-{i}", name="x")) + + +# --------------------------------------------------------------------------- +# Approval + public invariant +# --------------------------------------------------------------------------- + + +class TestApprovalAndPublic: + async def test_only_admin_may_approve(self, db): + await _insert("approve-me", ALICE) + with pytest.raises(PermissionError): + await _repo(ALICE).patch_initiative("approve-me", InitiativePatch(is_approved=True)) + approved = await _approve("approve-me") + assert approved.is_approved is True + + async def test_cannot_make_public_while_unapproved(self, db): + await _insert("public-fail", ALICE) + with pytest.raises(ValidationError): + await _repo(ALICE).patch_initiative("public-fail", InitiativePatch(is_public=True)) + + async def test_public_allowed_once_approved(self, db): + await _insert("public-ok", ALICE) + await _approve("public-ok") + patched = await _repo(ALICE).patch_initiative("public-ok", InitiativePatch(is_public=True)) + assert patched.is_public is True + + async def test_admin_can_approve_and_publish_together(self, db): + await _insert("publish-both", ALICE) + patched = await _repo(ADMIN).patch_initiative( + "publish-both", InitiativePatch(is_approved=True, is_public=True) + ) + assert patched.is_approved is True and patched.is_public is True + + +# --------------------------------------------------------------------------- +# Manage rights (patch) + read scope +# --------------------------------------------------------------------------- + + +class TestManageAndScope: + async def test_owner_can_rename(self, db): + await _insert("rename-me", ALICE) + patched = await _repo(ALICE).patch_initiative("rename-me", InitiativePatch(name="Renamed")) + assert patched.name == "Renamed" + + async def test_collaborator_can_patch(self, db): + await _insert("collab-patch", ALICE) + patched = await _repo(_collaborator("collab-patch")).patch_initiative( + "collab-patch", InitiativePatch(name="By Collaborator") + ) + assert patched.name == "By Collaborator" + + async def test_visible_but_unmanaged_cannot_patch(self, db): + # An approved+public initiative is visible to everyone, but a stranger still cannot manage it. + await _insert("visible-public", ALICE) + await _approve("visible-public") + await _repo(ALICE).patch_initiative("visible-public", InitiativePatch(is_public=True)) + stranger = User(username="google:carol@example.com", groups=frozenset()) + with pytest.raises(PermissionError): + await _repo(stranger).patch_initiative("visible-public", InitiativePatch(name="hijack")) + + async def test_private_unapproved_scope(self, db): + await _insert("scoped-priv", ALICE) + assert await _repo(ALICE).get_initiative("scoped-priv", fields=None) is not None # owner + assert await _repo(ADMIN).get_initiative("scoped-priv", fields=None) is not None # admin + assert await _repo(_collaborator("scoped-priv")).get_initiative("scoped-priv", fields=None) is not None + assert await _repo(ANON).get_initiative("scoped-priv", fields=None) is None # anon + assert await _repo(BOB).get_initiative("scoped-priv", fields=None) is None # unrelated user + + async def test_public_approved_visible_to_anon(self, db): + await _insert("scoped-pub", ALICE) + await _approve("scoped-pub") + await _repo(ALICE).patch_initiative("scoped-pub", InitiativePatch(is_public=True)) + assert await _repo(ANON).get_initiative("scoped-pub", fields=None) is not None + + +# --------------------------------------------------------------------------- +# Delete (owner or admin only) +# --------------------------------------------------------------------------- + + +class TestDelete: + async def test_owner_can_delete(self, db): + await _insert("del-owner", ALICE) + result = await _repo(ALICE).delete_initiative("del-owner") + assert result.num_deleted == 1 + assert await _repo(ADMIN).get_initiative("del-owner", fields=None) is None + + async def test_collaborator_cannot_delete(self, db): + await _insert("del-collab", ALICE) + with pytest.raises(PermissionError): + await _repo(_collaborator("del-collab")).delete_initiative("del-collab") + + async def test_missing_is_not_found(self, db): + with pytest.raises(NotFoundError): + await _repo(ADMIN).delete_initiative("nope-missing") diff --git a/mpcontribs-api/tests/integration/db/test_initiatives_service.py b/mpcontribs-api/tests/integration/db/test_initiatives_service.py new file mode 100644 index 0000000000..c10fbaf6e8 --- /dev/null +++ b/mpcontribs-api/tests/integration/db/test_initiatives_service.py @@ -0,0 +1,177 @@ +import pytest +from beanie import Link + +from mpcontribs_api.authz import User +from mpcontribs_api.config import get_settings +from mpcontribs_api.domains.initiatives.models import InitiativeIn, InitiativePatch +from mpcontribs_api.domains.initiatives.repository import InitiativeRepository +from mpcontribs_api.domains.projects.models import Project, ProjectIn, ProjectPatch, Stats +from mpcontribs_api.domains.projects.repository import MongoDbProjectRepository +from mpcontribs_api.domains.projects.service import ProjectInitiativeService +from mpcontribs_api.exceptions import ConflictError, NotFoundError, PermissionError + +pytestmark = [pytest.mark.db, pytest.mark.asyncio(loop_scope="session")] + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +ADMIN = User(username="google:admin@example.com", groups=frozenset({"admin"})) +ALICE = User(username="google:alice@example.com", groups=frozenset({"mp-team"})) +CAROL = User(username="google:carol@example.com", groups=frozenset()) + +ALICE_EMAIL = "google:alice@example.com" +BOB_EMAIL = "google:bob@example.com" +CAROL_EMAIL = "google:carol@example.com" +STATS = Stats(columns=0, contributions=0, tables=0, structures=0, attachments=0, size=0.0) + + +def _service(user: User) -> ProjectInitiativeService: + return ProjectInitiativeService( + projects=MongoDbProjectRepository(user), + initiatives=InitiativeRepository(user), + ) + + +def _collaborator(slug: str, username: str = BOB_EMAIL) -> User: + return User(username=username, groups=frozenset({f"initiative:{slug}"})) + + +async def _insert_project(pid: str, owner: str = ALICE_EMAIL) -> Project: + return await MongoDbProjectRepository(ADMIN).insert_project( + ProjectIn( + _id=pid, + title=pid[:30], + authors="Author", + description="desc", + owner=owner, + unique_identifiers=True, + stats=STATS, + ) + ) + + +async def _insert_initiative(slug: str, owner_user: User = ALICE): + return await InitiativeRepository(owner_user).insert_initiative(InitiativeIn(slug=slug, name="Init")) + + +def _assigned_id(project: Project): + """The initiative _id a returned project points at, or None.""" + link = project.initiative + if link is None: + return None + return link.ref.id if isinstance(link, Link) else link.id + + +# --------------------------------------------------------------------------- +# Happy-path assignment +# --------------------------------------------------------------------------- + + +class TestAssign: + async def test_owner_of_both_can_assign(self, db): + await _insert_project("proj-a", owner=ALICE_EMAIL) + init = await _insert_initiative("init-a", ALICE) + updated = await _service(ALICE).patch("proj-a", ProjectPatch(initiative="init-a")) + assert _assigned_id(updated) == init.id + + async def test_collaborator_can_assign_own_project(self, db): + await _insert_project("proj-b", owner=BOB_EMAIL) + init = await _insert_initiative("init-collab", ALICE) + bob = _collaborator("init-collab") + updated = await _service(bob).patch("proj-b", ProjectPatch(initiative="init-collab")) + assert _assigned_id(updated) == init.id + + async def test_plain_patch_passes_through_untouched(self, db): + await _insert_project("proj-plain", owner=ALICE_EMAIL) + init = await _insert_initiative("init-plain", ALICE) + await _service(ALICE).patch("proj-plain", ProjectPatch(initiative="init-plain")) + # A patch that does not mention `initiative` must not disturb the existing assignment. + updated = await _service(ALICE).patch("proj-plain", ProjectPatch(title="new-title")) + assert updated.title == "new-title" + assert _assigned_id(updated) == init.id + + async def test_unassign_clears_link(self, db): + await _insert_project("proj-un", owner=ALICE_EMAIL) + await _insert_initiative("init-un", ALICE) + await _service(ALICE).patch("proj-un", ProjectPatch(initiative="init-un")) + updated = await _service(ALICE).patch("proj-un", ProjectPatch(initiative=None)) + assert _assigned_id(updated) is None + + +# --------------------------------------------------------------------------- +# Both-rights enforcement +# --------------------------------------------------------------------------- + + +class TestBothRights: + async def test_visible_but_unmanaged_initiative_rejected(self, db): + # Carol owns her project (project-write ok) and can *see* this public+approved initiative, + # but she neither owns nor collaborates on it, so she still cannot assign to it. + await _insert_project("proj-c", owner=CAROL_EMAIL) + await _insert_initiative("init-c", ALICE) + await InitiativeRepository(ADMIN).patch_initiative( + "init-c", InitiativePatch(is_approved=True, is_public=True) + ) + with pytest.raises(PermissionError): + await _service(CAROL).patch("proj-c", ProjectPatch(initiative="init-c")) + + async def test_invisible_initiative_is_not_found(self, db): + # Alice's private initiative is invisible to Carol, so it reads as not-found (not a 403). + await _insert_project("proj-c2", owner=CAROL_EMAIL) + await _insert_initiative("init-priv", ALICE) + with pytest.raises(NotFoundError): + await _service(CAROL).patch("proj-c2", ProjectPatch(initiative="init-priv")) + + async def test_manager_without_project_write_rejected(self, db): + # Alice manages the initiative but cannot see/write Bob's private project. + await _insert_project("proj-bob", owner=BOB_EMAIL) + await _insert_initiative("init-d", ALICE) + with pytest.raises(NotFoundError): + await _service(ALICE).patch("proj-bob", ProjectPatch(initiative="init-d")) + + async def test_assign_to_missing_initiative_is_not_found(self, db): + await _insert_project("proj-ghost", owner=ALICE_EMAIL) + with pytest.raises(NotFoundError): + await _service(ALICE).patch("proj-ghost", ProjectPatch(initiative="ghost-init")) + + +# --------------------------------------------------------------------------- +# Member cap on unapproved initiatives +# --------------------------------------------------------------------------- + + +class TestMemberCap: + async def test_unapproved_capped_at_configured_members(self, db): + cap = get_settings().initiatives.max_projects_per_unapproved + await _insert_initiative("init-cap", ALICE) + for i in range(cap): + await _insert_project(f"cap-proj-{i}", owner=ALICE_EMAIL) + await _service(ALICE).patch(f"cap-proj-{i}", ProjectPatch(initiative="init-cap")) + await _insert_project("cap-proj-over", owner=ALICE_EMAIL) + with pytest.raises(ConflictError): + await _service(ALICE).patch("cap-proj-over", ProjectPatch(initiative="init-cap")) + + async def test_reassigning_existing_member_is_idempotent(self, db): + cap = get_settings().initiatives.max_projects_per_unapproved + await _insert_initiative("init-idem", ALICE) + for i in range(cap): + await _insert_project(f"idem-proj-{i}", owner=ALICE_EMAIL) + await _service(ALICE).patch(f"idem-proj-{i}", ProjectPatch(initiative="init-idem")) + # At the cap, re-assigning a project that is already a member must not trip the limit. + again = await _service(ALICE).patch("idem-proj-0", ProjectPatch(initiative="init-idem")) + assert again.initiative is not None + + async def test_approved_initiative_has_no_member_cap(self, db): + cap = get_settings().initiatives.max_projects_per_unapproved + await _insert_initiative("init-approved", ALICE) + await InitiativeRepository(ADMIN).patch_initiative("init-approved", InitiativePatch(is_approved=True)) + for i in range(cap + 2): # comfortably past the unapproved cap + await _insert_project(f"appr-proj-{i}", owner=ALICE_EMAIL) + await _service(ALICE).patch(f"appr-proj-{i}", ProjectPatch(initiative="init-approved")) + count = await MongoDbProjectRepository(ADMIN).count_initiative_members( + initiative_id=(await InitiativeRepository(ADMIN).resolve_visible("init-approved")).id, # type: ignore[union-attr] + exclude_project_id=None, + ) + assert count == cap + 2 diff --git a/mpcontribs-api/tests/integration/test_initiatives.py b/mpcontribs-api/tests/integration/test_initiatives.py new file mode 100644 index 0000000000..c0eba7a471 --- /dev/null +++ b/mpcontribs-api/tests/integration/test_initiatives.py @@ -0,0 +1,135 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from mpcontribs_api.domains.initiatives.dependencies import get_initiative_repository +from mpcontribs_api.exceptions import NotFoundError +from mpcontribs_api.pagination import Page +from tests.integration.conftest import AUTHED_HEADERS, FORCE_ANON_HEADERS + +SAMPLE_OID = "6eb7cf5a86d9755df3a6c593" + + +@pytest.fixture +def initiative_repo(test_app): + repo = AsyncMock() + test_app.dependency_overrides[get_initiative_repository] = lambda: repo + yield repo + test_app.dependency_overrides.pop(get_initiative_repository, None) + + +def _stored(**overrides): + """Stand-in for the stored document: exposes ``.id`` like a Beanie Document.""" + attrs = { + "id": SAMPLE_OID, + "slug": "battery-genome", + "name": "Battery Genome", + "owner": "google:alice@example.com", + "is_public": False, + "is_approved": False, + } + attrs.update(overrides) + return SimpleNamespace(**attrs) + + +# --------------------------------------------------------------------------- +# POST /api/v1/initiatives +# --------------------------------------------------------------------------- + + +class TestInsert: + def test_returns_201_and_echoes_id(self, client, initiative_repo): + initiative_repo.insert_initiative.return_value = _stored() + r = client.post( + "/api/v1/initiatives", + json={"slug": "battery-genome", "name": "Battery Genome"}, + headers=AUTHED_HEADERS, + ) + assert r.status_code == 201 + assert r.json()["id"] == SAMPLE_OID + + def test_anonymous_rejected_401(self, client, initiative_repo): + r = client.post( + "/api/v1/initiatives", + json={"slug": "battery-genome", "name": "Battery Genome"}, + headers=FORCE_ANON_HEADERS, + ) + assert r.status_code == 401 + + def test_invalid_slug_returns_422(self, client, initiative_repo): + r = client.post( + "/api/v1/initiatives", + json={"slug": "Not A Slug!", "name": "x"}, + headers=AUTHED_HEADERS, + ) + assert r.status_code == 422 + + +# --------------------------------------------------------------------------- +# GET /api/v1/initiatives (+ /{slug}) +# --------------------------------------------------------------------------- + + +class TestGet: + def test_list_returns_200(self, client, initiative_repo): + initiative_repo.get_initiatives.return_value = Page(items=[], next_cursor=None) + r = client.get("/api/v1/initiatives", headers=AUTHED_HEADERS) + assert r.status_code == 200 + + def test_get_by_slug_returns_200(self, client, initiative_repo): + initiative_repo.get_initiative.return_value = _stored() + r = client.get("/api/v1/initiatives/battery-genome", headers=AUTHED_HEADERS) + assert r.status_code == 200 + assert r.json()["slug"] == "battery-genome" + + +# --------------------------------------------------------------------------- +# PATCH /api/v1/initiatives/{slug} +# --------------------------------------------------------------------------- + + +class TestPatch: + def test_patch_returns_200(self, client, initiative_repo): + initiative_repo.patch_initiative.return_value = _stored(name="Renamed") + r = client.patch( + "/api/v1/initiatives/battery-genome", + json={"name": "Renamed"}, + headers=AUTHED_HEADERS, + ) + assert r.status_code == 200 + assert r.json()["name"] == "Renamed" + + def test_anonymous_rejected_401(self, client, initiative_repo): + r = client.patch( + "/api/v1/initiatives/battery-genome", + json={"name": "Renamed"}, + headers=FORCE_ANON_HEADERS, + ) + assert r.status_code == 401 + + def test_not_found_propagates_404(self, client, initiative_repo): + initiative_repo.patch_initiative.side_effect = NotFoundError("nope") + r = client.patch( + "/api/v1/initiatives/missing", + json={"name": "Renamed"}, + headers=AUTHED_HEADERS, + ) + assert r.status_code == 404 + + +# --------------------------------------------------------------------------- +# DELETE /api/v1/initiatives/{slug} +# --------------------------------------------------------------------------- + + +class TestDelete: + def test_delete_returns_204(self, client, initiative_repo): + initiative_repo.delete_initiative.return_value = None + r = client.delete("/api/v1/initiatives/battery-genome", headers=AUTHED_HEADERS) + assert r.status_code == 204 + assert r.content == b"" + + def test_anonymous_rejected_401(self, client, initiative_repo): + r = client.delete("/api/v1/initiatives/battery-genome", headers=FORCE_ANON_HEADERS) + assert r.status_code == 401 From 2d1c361c4546686753dc3535dd0e8a3ee0815149 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Thu, 16 Jul 2026 13:26:08 -0700 Subject: [PATCH 28/62] Added initiative as a field on models and implemented logic for managing project-initiative interactions --- .../domains/projects/dependencies.py | 12 +++ .../mpcontribs_api/domains/projects/models.py | 8 ++ .../domains/projects/repository.py | 45 ++++++++++- .../mpcontribs_api/domains/projects/router.py | 13 +++- .../domains/projects/service.py | 76 +++++++++++++++++++ .../tests/integration/db/conftest.py | 10 ++- .../db/test_project_groups_repository.py | 13 +++- .../db/test_project_groups_service.py | 12 ++- .../tests/integration/test_projects.py | 33 +++++--- 9 files changed, 201 insertions(+), 21 deletions(-) create mode 100644 mpcontribs-api/src/mpcontribs_api/domains/projects/service.py diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/dependencies.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/dependencies.py index 94535cc67d..e1d791ea21 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/dependencies.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/dependencies.py @@ -3,9 +3,11 @@ from fastapi import Depends from mpcontribs_api.dependencies import UserDep +from mpcontribs_api.domains.initiatives.repository import InitiativeRepository from mpcontribs_api.domains.projects.repository import ( MongoDbProjectRepository, ) +from mpcontribs_api.domains.projects.service import ProjectService def get_scoped_projects(user: UserDep) -> MongoDbProjectRepository: @@ -13,3 +15,13 @@ def get_scoped_projects(user: UserDep) -> MongoDbProjectRepository: ProjectDep = Annotated[MongoDbProjectRepository, Depends(get_scoped_projects)] + + +def get_project_service(user: UserDep) -> ProjectService: + return ProjectService( + projects=MongoDbProjectRepository(user), + initiatives=InitiativeRepository(user), + ) + + +ProjectServiceDep = Annotated[ProjectService, Depends(get_project_service)] diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py index 589ee198c3..d8b307b026 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py @@ -2,12 +2,14 @@ from typing import Any, Literal +from beanie import Link from pydantic import BaseModel, ConfigDict, Field, HttpUrl from mpcontribs_api import pagination from mpcontribs_api.domains._shared.filters import BaseFilter from mpcontribs_api.domains._shared.models import BaseDocumentWithInput, DocumentOut from mpcontribs_api.domains._shared.types import PrefixedEmail, ShortStr +from mpcontribs_api.domains.initiatives.models import Initiative class Column(BaseModel): @@ -60,6 +62,8 @@ class Project(BaseDocumentWithInput[ShortStr]): is_approved: bool = False license: Literal["CCA4", "CCPD"] | None = None + initiative: Link[Initiative] | None = None + # Empty method for now. Keeping for business logic later @classmethod def from_input_model(cls, data: ProjectIn) -> Project: @@ -96,6 +100,7 @@ class ProjectOut(DocumentOut[ShortStr]): stats: Stats | None = None columns: list[Column] | None = None license: Literal["CCA4", "CCPD"] | None = None + initiative: Link[Initiative] | None = None @staticmethod def default_fields() -> list[str]: @@ -163,3 +168,6 @@ class ProjectPatch(BaseModel): is_public: bool | None = None is_approved: bool | None = None license: Literal["CCA4", "CCPD"] | None = None + + # str here, but ProjectService coerces to a Link + initiative: str | None = None diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py index 000be07dca..00e8373824 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py @@ -1,5 +1,9 @@ from typing import Any +from beanie import PydanticObjectId, UpdateResponse +from beanie.operators import Set +from bson import DBRef + from mpcontribs_api.authz import User from mpcontribs_api.domains._shared.repository import MongoDbRepository from mpcontribs_api.domains.projects.models import ( @@ -9,7 +13,7 @@ ProjectOut, ProjectPatch, ) -from mpcontribs_api.exceptions import PermissionError +from mpcontribs_api.exceptions import NotFoundError, PermissionError from mpcontribs_api.pagination import CursorParams @@ -133,3 +137,42 @@ async def upsert_project_by_id(self, id: str, data: ProjectIn) -> Project: # New project: the caller owns it, regardless of the submitted owner. project.owner = self._user.username return await project.save() + + async def set_initiative(self, id: str, ref: DBRef | None) -> Project: + """Set a scoped project's canonical initiative link. + + The link is written as-is (a ``DBRef`` into ``initiatives`` or ``None``); all authorization + and limit checks are the caller's (see ``ProjectInitiativeService``). Scoping ensures a + project the caller cannot see is reported as not found rather than silently missed. + + Args: + id (str): the id of the project to update + ref (DBRef | None): the initiative reference to assign, or None to unassign + """ + query = self.document_model.find_one(self._scope, self.document_model.id == id).update( + Set({"initiative": ref}), + response_type=UpdateResponse.NEW_DOCUMENT, + ) + updated = await query # pyright: ignore[reportGeneralTypeIssues] # beanie UpdateQuery is awaitable + if updated is None: + raise NotFoundError(self._not_found(id)) + return updated + + async def count_initiative_members(self, initiative_id: PydanticObjectId, exclude_project_id: str | None) -> int: + """Count projects assigned to an initiative, ignoring user scope. + + The unapproved-initiative member limit is an integrity constraint on the initiative's true + size, so it must count every member regardless of who can see them — a scoped count could + let a collaborator overshoot the cap with projects they cannot see. ``exclude_project_id`` + drops the project being (re)assigned so re-assigning an existing member is idempotent and + never trips the limit. + + Args: + initiative_id (PydanticObjectId): the initiative whose members to count + exclude_project_id (str | None): a project id to exclude from the count, if any + """ + collection = self.document_model.get_pymongo_collection() + query: dict[str, Any] = {"initiative.$id": initiative_id} + if exclude_project_id is not None: + query["_id"] = {"$ne": exclude_project_id} + return await collection.count_documents(query) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/router.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/router.py index 33dce7ebd7..56d453a948 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/router.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/router.py @@ -6,7 +6,7 @@ from mpcontribs_api.dependencies import require_user from mpcontribs_api.domains._shared.types import FieldSelector -from mpcontribs_api.domains.projects.dependencies import ProjectDep +from mpcontribs_api.domains.projects.dependencies import ProjectDep, ProjectServiceDep from mpcontribs_api.domains.projects.models import ( ProjectFilter, ProjectIn, @@ -83,7 +83,7 @@ async def upsert_project_by_id( @router.patch("/{id}", response_model=ProjectOut, dependencies=[Depends(require_user)]) async def patch_project_by_id( - repo: ProjectDep, + service: ProjectServiceDep, id: str, update: ProjectPatch, ): @@ -91,8 +91,13 @@ async def patch_project_by_id( Note: overwrites fields with given values - arrays are not appended to. + The ``initiative`` field carries an initiative ``slug`` (or ``null`` to unassign). Setting it + is gated by the assignment service: the caller must be able to manage the target initiative + (owner/collaborator/admin) and an unapproved initiative may not exceed its member cap. Plain + field patches take the fast path straight to the repository. + Args: - repo (ProjectDep): the project repo we depend on + service (ProjectServiceDep): the project assignment service we depend on id (str): the id of the project to update update (ProjectPatch): the partial update to apply - unset fields are dropped - Note: If fields are intentionally set to None, None is applied to the field. @@ -100,7 +105,7 @@ async def patch_project_by_id( Returns: ProjectOut: the full Project with updates applied """ - return await repo.patch_project_by_id(id=id, update=update) + return await service.patch(id=id, update=update) @router.delete("/{id}", status_code=status.HTTP_204_NO_CONTENT, dependencies=[Depends(require_user)]) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py new file mode 100644 index 0000000000..f95fbacb49 --- /dev/null +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py @@ -0,0 +1,76 @@ +from bson import DBRef + +from mpcontribs_api.config import get_settings +from mpcontribs_api.domains.initiatives.repository import InitiativeRepository +from mpcontribs_api.domains.projects.models import Project, ProjectPatch +from mpcontribs_api.domains.projects.repository import MongoDbProjectRepository +from mpcontribs_api.exceptions import ConflictError, NotFoundError, PermissionError + + +class ProjectService: + """Coordinates assigning a project to its canonical initiative across the two collections.""" + + def __init__( + self, + projects: MongoDbProjectRepository, + initiatives: InitiativeRepository, + ) -> None: + self._projects = projects + self._initiatives = initiatives + self._limits = get_settings().domain.initiatives + + async def patch(self, id: str, update: ProjectPatch) -> Project: + """Apply a project patch, routing an ``initiative`` change through the assignment checks. + + ``initiative`` carries the target initiative's ``slug`` (or ``null`` to unassign). It is + split out of the patch so it never reaches the raw ``$set`` as a bare string; any remaining + fields are applied first, then the initiative link is set. + """ + if "initiative" not in update.model_fields_set: + return await self._projects.patch_project_by_id(id=id, update=update) + + data = update.model_dump(exclude_unset=True) + slug = data.pop("initiative", None) + + # Resolve the target link (and run the both-rights + limit checks) before touching anything. + ref = await self._resolve_initiative_assignment(project_id=id, slug=slug) + + # Apply any co-submitted plain fields first, so a single request can rename and assign. + if data: + await self._projects.patch_project_by_id(id=id, update=ProjectPatch(**data)) + return await self._projects.set_initiative(id=id, ref=ref) + + async def _resolve_initiative_assignment(self, project_id: str, slug: str | None) -> DBRef | None: + """Validate an initiative assignment and return the link to store (or None to unassign). + + Unassigning needs only project-write access (already enforced downstream). Assigning + additionally requires that the caller can manage the target initiative and that an + unapproved target has room under its member cap. + """ + if slug is None: + return None + + initiative = await self._initiatives.resolve_visible(slug) + if initiative is None: + raise NotFoundError("Initiative not found or not visible", slug=slug) + + if not self._initiatives._user.can_manage(id=initiative.slug, resource="initiative"): + raise PermissionError( + message="user does not have adequate acceess to this resource", + required_role="initiative-owner-collaborator-or-admin", + resource_id=initiative.slug, + ) + + if not initiative.is_approved: + members = await self._projects.count_initiative_members( + initiative_id=initiative.id, + exclude_project_id=project_id, + ) + if members >= self._limits.max_projects_per_unapproved: + raise ConflictError( + message="unapproved initiative already has the maximum number of assigned projects", + slug=slug, + limit=self._limits.max_projects_per_unapproved, + ) + + return DBRef("initiatives", initiative.id) diff --git a/mpcontribs-api/tests/integration/db/conftest.py b/mpcontribs-api/tests/integration/db/conftest.py index f4f8caa08f..841b0fc3d4 100644 --- a/mpcontribs-api/tests/integration/db/conftest.py +++ b/mpcontribs-api/tests/integration/db/conftest.py @@ -6,6 +6,7 @@ from mpcontribs_api.config import get_settings from mpcontribs_api.domains.attachments.models import Attachment from mpcontribs_api.domains.contributions.models import Contribution +from mpcontribs_api.domains.initiatives.models import Initiative from mpcontribs_api.domains.project_groups.models import ProjectGroup from mpcontribs_api.domains.projects.models import Project from mpcontribs_api.domains.structures.models import Structure @@ -67,7 +68,7 @@ async def db(mongo_client): database = mongo_client[settings.mongo.db_name] await init_beanie( database=database, - document_models=[Project, ProjectGroup, Contribution, Structure, Table, Attachment], + document_models=[Project, ProjectGroup, Initiative, Contribution, Structure, Table, Attachment], ) yield database @@ -105,3 +106,10 @@ async def clean_project_groups(db): await db["project_groups"].delete_many({}) yield await db["project_groups"].delete_many({}) + + +@pytest_asyncio.fixture(autouse=True) +async def clean_initiatives(db): + await db["initiatives"].delete_many({}) + yield + await db["initiatives"].delete_many({}) diff --git a/mpcontribs-api/tests/integration/db/test_project_groups_repository.py b/mpcontribs-api/tests/integration/db/test_project_groups_repository.py index 82723fd30e..fc19448017 100644 --- a/mpcontribs-api/tests/integration/db/test_project_groups_repository.py +++ b/mpcontribs-api/tests/integration/db/test_project_groups_repository.py @@ -111,8 +111,17 @@ async def test_duplicate_identifiers_raise_conflict(self, db): {"_id": PydanticObjectId(), "name": "dup", "owner": ALICE_EMAIL, "projects": [], "description": "d"}, ] ) - with pytest.raises(ConflictError): - await _repo(ADMIN).delete_project_group(name="dup", owner=ALICE_EMAIL) + try: + with pytest.raises(ConflictError): + await _repo(ADMIN).delete_project_group(name="dup", owner=ALICE_EMAIL) + finally: + # Restore the unique index we dropped so order-dependent tests that rely on it (e.g. the + # insert-duplicate guard) still see it. Planted duplicates must go first, or the unique + # index rebuild would fail. + await db["project_groups"].delete_many({"name": "dup"}) + await db["project_groups"].create_index( + [("name", 1), ("owner", 1)], name="name_owner", unique=True + ) # --------------------------------------------------------------------------- diff --git a/mpcontribs-api/tests/integration/db/test_project_groups_service.py b/mpcontribs-api/tests/integration/db/test_project_groups_service.py index dfd81ddcda..5aac87b567 100644 --- a/mpcontribs-api/tests/integration/db/test_project_groups_service.py +++ b/mpcontribs-api/tests/integration/db/test_project_groups_service.py @@ -117,8 +117,16 @@ async def test_ambiguous_identifiers_raise_conflict(self, db): {"_id": PydanticObjectId(), "name": "dup", "owner": ALICE_EMAIL, "projects": [], "description": "d"}, ] ) - with pytest.raises(ConflictError): - await _service().add_projects_by_identifiers("dup", ALICE_EMAIL, []) + try: + with pytest.raises(ConflictError): + await _service().add_projects_by_identifiers("dup", ALICE_EMAIL, []) + finally: + # Restore the unique index we dropped so order-dependent tests that rely on it still see + # it. Planted duplicates must go first, or the unique index rebuild would fail. + await db["project_groups"].delete_many({"name": "dup"}) + await db["project_groups"].create_index( + [("name", 1), ("owner", 1)], name="name_owner", unique=True + ) # --------------------------------------------------------------------------- diff --git a/mpcontribs-api/tests/integration/test_projects.py b/mpcontribs-api/tests/integration/test_projects.py index 0f98857923..e157665880 100644 --- a/mpcontribs-api/tests/integration/test_projects.py +++ b/mpcontribs-api/tests/integration/test_projects.py @@ -1,6 +1,8 @@ +from unittest.mock import AsyncMock + import pytest -from mpcontribs_api.domains.projects.dependencies import get_scoped_projects +from mpcontribs_api.domains.projects.dependencies import get_project_initiative_service, get_scoped_projects from mpcontribs_api.domains.projects.models import ProjectOut, Stats from mpcontribs_api.exceptions import ConflictError, NotFoundError from mpcontribs_api.pagination import Page @@ -37,6 +39,15 @@ def project_repo(test_app, mock_project_repo): test_app.dependency_overrides.pop(get_scoped_projects, None) +@pytest.fixture +def project_service(test_app): + """Override the assignment service the PATCH route depends on with an async mock.""" + service = AsyncMock() + test_app.dependency_overrides[get_project_initiative_service] = lambda: service + yield service + test_app.dependency_overrides.pop(get_project_initiative_service, None) + + # --------------------------------------------------------------------------- # GET /api/v1/projects # --------------------------------------------------------------------------- @@ -154,8 +165,8 @@ def test_no_fields_param_uses_default_fields(self, client, project_repo): class TestPatchProject: - def test_valid_patch_returns_200(self, client, project_repo): - project_repo.patch_project_by_id.return_value = SAMPLE_PROJECT + def test_valid_patch_returns_200(self, client, project_service): + project_service.patch.return_value = SAMPLE_PROJECT r = client.patch( "/api/v1/projects/mp-sample", json={"title": "Updated Title"}, @@ -163,9 +174,9 @@ def test_valid_patch_returns_200(self, client, project_repo): ) assert r.status_code == 200 - def test_patch_response_is_project_out(self, client, project_repo): + def test_patch_response_is_project_out(self, client, project_service): updated = ProjectOut(id="mp-sample", title="Updated Title") - project_repo.patch_project_by_id.return_value = updated + project_service.patch.return_value = updated body = client.patch( "/api/v1/projects/mp-sample", json={"title": "Updated Title"}, @@ -173,8 +184,8 @@ def test_patch_response_is_project_out(self, client, project_repo): ).json() assert body["title"] == "Updated Title" - def test_not_found_returns_404(self, client, project_repo): - project_repo.patch_project_by_id.side_effect = NotFoundError("not found") + def test_not_found_returns_404(self, client, project_service): + project_service.patch.side_effect = NotFoundError("not found") r = client.patch( "/api/v1/projects/missing", json={"title": "x" * 5}, @@ -182,7 +193,7 @@ def test_not_found_returns_404(self, client, project_repo): ) assert r.status_code == 404 - def test_invalid_title_too_short_returns_422(self, client, project_repo): + def test_invalid_title_too_short_returns_422(self, client, project_service): r = client.patch( "/api/v1/projects/mp-sample", json={"title": "ab"}, @@ -190,14 +201,14 @@ def test_invalid_title_too_short_returns_422(self, client, project_repo): ) assert r.status_code == 422 - def test_id_and_update_forwarded_to_repo(self, client, project_repo): - project_repo.patch_project_by_id.return_value = SAMPLE_PROJECT + def test_id_and_update_forwarded_to_service(self, client, project_service): + project_service.patch.return_value = SAMPLE_PROJECT client.patch( "/api/v1/projects/mp-sample", json={"title": "New Name"}, headers=AUTHED_HEADERS, ) - _, kwargs = project_repo.patch_project_by_id.call_args + _, kwargs = project_service.patch.call_args assert kwargs["id"] == "mp-sample" assert kwargs["update"].title == "New Name" From aced4c160712893954ebe7c19e6eb4b11532c1f2 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Thu, 16 Jul 2026 17:01:46 -0700 Subject: [PATCH 29/62] Removed user from repo fields --- .../src/mpcontribs_api/domains/_shared/repository.py | 1 + .../src/mpcontribs_api/domains/initiatives/repository.py | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py index c1a65c62e0..b5e276f4d2 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py @@ -56,6 +56,7 @@ def __init__(self, user: User) -> None: Args: user (User): the current user requesting resources """ + self._user = user self._scope = self._build_scope(user) @staticmethod diff --git a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py index bce9f706b1..6c373a5221 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py @@ -26,7 +26,6 @@ class InitiativeRepository( def __init__(self, user: User) -> None: super().__init__(user) - self._user = user self._limits = get_settings().domain.initiatives @staticmethod @@ -109,8 +108,7 @@ async def patch_initiative(self, slug: str, update: InitiativePatch) -> Initiati if existing is None: raise NotFoundError("Initiative not found", slug=slug) if not ( - self._user.can_manage(id=existing.slug, resource="initiative") - or self._user.username == self.document_model.owner + self._user.can_manage(id=existing.slug, resource="initiative") or self._user.username == existing.owner ): raise PermissionError(required_role="initiative-owner-collaborator-or-admin") From ccfd1f9d212ec459f24f6625d573437467f9468c Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Thu, 16 Jul 2026 17:08:28 -0700 Subject: [PATCH 30/62] Automatically set caller as owner of ProjectGroup on Insert --- .../src/mpcontribs_api/domains/project_groups/service.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py index 5976da27ca..b882208aed 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py @@ -28,7 +28,13 @@ async def _project_exists(self, project_id: ShortStr) -> bool: return await self._projects.get_by_id(project_id, fields=frozenset({"id"})) is not None async def insert(self, project_group: ProjectGroupIn) -> ProjectGroup: - """Insert a new group after verifying every referenced project exists and is visible""" + """Insert a new group after verifying every referenced project exists and is visible. + + Non-admins are set as owner automatically, while admins can specify owners. + """ + user = self._groups._user + if not user.is_admin: + project_group = project_group.model_copy(update={"owner": user.username}) missing = [pid for pid in project_group.projects if not await self._project_exists(pid)] if missing: raise NotFoundError("One or more projects not found or not visible", ids=missing) From b4ff3b023191a970fb41bd2cb8395bdb7c450daa Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Thu, 16 Jul 2026 17:19:11 -0700 Subject: [PATCH 31/62] Require users inserting components to have at least one project assigned to them --- .../src/mpcontribs_api/domains/structures/router.py | 4 ++-- mpcontribs-api/src/mpcontribs_api/domains/tables/router.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/structures/router.py b/mpcontribs-api/src/mpcontribs_api/domains/structures/router.py index 803047648f..efa86a2e38 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/structures/router.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/structures/router.py @@ -4,7 +4,7 @@ from fastapi.responses import StreamingResponse from fastapi_filter import FilterDepends -from mpcontribs_api.dependencies import S3Dep, require_user +from mpcontribs_api.dependencies import S3Dep, require_user, require_writer from mpcontribs_api.domains._shared.bulk import BulkWriteSummary from mpcontribs_api.domains._shared.models import ComponentDeleteResponse from mpcontribs_api.domains._shared.types import ( @@ -68,7 +68,7 @@ async def download_structure( ) -@router.post("", response_model=BulkWriteSummary[StructureOut], dependencies=[Depends(require_user)]) +@router.post("", response_model=BulkWriteSummary[StructureOut], dependencies=[Depends(require_writer)]) async def insert_structures( service: StructureServiceDep, structures: list[StructureIn], diff --git a/mpcontribs-api/src/mpcontribs_api/domains/tables/router.py b/mpcontribs-api/src/mpcontribs_api/domains/tables/router.py index 703f0c6faa..658a650575 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/tables/router.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/tables/router.py @@ -4,7 +4,7 @@ from fastapi.responses import StreamingResponse from fastapi_filter import FilterDepends -from mpcontribs_api.dependencies import S3Dep, require_user +from mpcontribs_api.dependencies import S3Dep, require_user, require_writer from mpcontribs_api.domains._shared.bulk import BulkWriteSummary from mpcontribs_api.domains._shared.models import ComponentDeleteResponse from mpcontribs_api.domains._shared.types import ( @@ -68,7 +68,7 @@ async def download_table( ) -@router.post("", response_model=BulkWriteSummary[Table], dependencies=[Depends(require_user)]) +@router.post("", response_model=BulkWriteSummary[Table], dependencies=[Depends(require_writer)]) async def insert_tables( service: TableServiceDep, tables: list[TableIn], From 4e175e7a6450ff84270299c7224fa0b38f80735a Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Thu, 16 Jul 2026 17:19:40 -0700 Subject: [PATCH 32/62] require_writer requires a user have at least one project assigned to them --- mpcontribs-api/src/mpcontribs_api/dependencies.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/mpcontribs-api/src/mpcontribs_api/dependencies.py b/mpcontribs-api/src/mpcontribs_api/dependencies.py index 0e0aa03278..e145920f70 100644 --- a/mpcontribs-api/src/mpcontribs_api/dependencies.py +++ b/mpcontribs-api/src/mpcontribs_api/dependencies.py @@ -8,7 +8,7 @@ from types_aiobotocore_s3 import S3Client from mpcontribs_api.authz import User -from mpcontribs_api.exceptions import AuthenticationError +from mpcontribs_api.exceptions import AuthenticationError, PermissionError def get_db(request: Request) -> AsyncDatabase: @@ -71,3 +71,16 @@ def require_user(user: UserDep) -> User: if user.is_anonymous: raise AuthenticationError("authentication required") return user + + +def require_writer(user: UserDep) -> User: + """Require an authenticated caller who can write to at least one project. + + Controls access to creating components if you do not have contributions to attach them to. + Helps to limit orphanned components + """ + if user.is_anonymous: + raise AuthenticationError("authentication required") + if not (user.is_admin or user.writable_projects): + raise PermissionError("write access to at least one project is required") + return user From 5fe29d37f7865a408b1743ebe9b16ec77b196958 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Thu, 16 Jul 2026 17:21:12 -0700 Subject: [PATCH 33/62] Added parsing user roles for project-group: strings --- mpcontribs-api/src/mpcontribs_api/authz.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/mpcontribs-api/src/mpcontribs_api/authz.py b/mpcontribs-api/src/mpcontribs_api/authz.py index 27fd78de63..99d6713ace 100644 --- a/mpcontribs-api/src/mpcontribs_api/authz.py +++ b/mpcontribs-api/src/mpcontribs_api/authz.py @@ -36,6 +36,9 @@ # prefix to user roles to disambiguate from project roles, which are bare ids INITIATIVE_ROLE_PREFIX = "initiative:" +# prefix for project-group roles: a group's _id (an ObjectId hex string) is granted as ``project-group:`` +PROJECT_GROUP_ROLE_PREFIX = "project-group:" + class User(BaseModel): """User definition derived from request headers. @@ -78,6 +81,16 @@ def initiative_roles(self) -> list[str]: """The initiative slugs this user collaborates on, decoded from their ``initiative:`` roles.""" return [role[len(INITIATIVE_ROLE_PREFIX) :] for role in self.groups if role.startswith(INITIATIVE_ROLE_PREFIX)] + @property + def project_group_roles(self) -> list[str]: + """The project-group ids this user may access, decoded from their ``project-group:`` roles. + + Values are the raw hex strings; callers that query by ``_id`` must convert them + """ + return [ + role[len(PROJECT_GROUP_ROLE_PREFIX) :] for role in self.groups if role.startswith(PROJECT_GROUP_ROLE_PREFIX) + ] + def has_role(self, role: str, *, resource: str | None = None) -> bool: """Determine whether a user has a role assigned to them. From bd8ac131c9940c4ad4bd92035b008acc62529979 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Thu, 16 Jul 2026 17:22:24 -0700 Subject: [PATCH 34/62] Improved handling of user roles and permissions along with informative errors --- .../domains/project_groups/repository.py | 31 ++++++++-- .../domains/projects/repository.py | 56 +++++++++++++++---- .../domains/projects/service.py | 3 +- 3 files changed, 73 insertions(+), 17 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py index d0f96009d0..f615923a39 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py @@ -3,6 +3,7 @@ from beanie import PydanticObjectId, UpdateResponse from beanie.operators import AddToSet, Pull from bson import DBRef +from bson.errors import InvalidId from pymongo.asynchronous.client_session import AsyncClientSession from mpcontribs_api.authz import User @@ -16,6 +17,7 @@ ProjectGroupOut, ProjectGroupPatch, ) +from mpcontribs_api.exceptions import NotFoundError, PermissionError from mpcontribs_api.pagination import CursorParams, Page @@ -27,14 +29,20 @@ class ProjectGroupRepository( @staticmethod def _build_scope(user: User) -> dict[str, Any]: - """Provides scope based on current user's permitted groups and publicly released data.""" + """Scope reads to what the caller may see: public groups, ones they own, or ones granted.""" if user.is_admin: return {} ors: list[dict[str, Any]] = [{"is_public": True}] if not user.is_anonymous: ors.append({"owner": user.username}) - if user.groups: - ors.append({"_id": {"$in": sorted(user.groups)}}) + granted: list[PydanticObjectId] = [] + for raw in user.project_group_roles: + try: + granted.append(PydanticObjectId(raw)) + except InvalidId: + continue + if granted: + ors.append({"_id": {"$in": sorted(granted)}}) return {"$or": ors} async def get_project_groups( @@ -71,11 +79,22 @@ async def patch_project_group( return await self.patch_one({"name": name, "owner": owner}, update) async def delete_project_group(self, name: SearchStr, owner: PrefixedEmail) -> DeleteResponse: - """Delete the single project group identified by ``name`` + ``owner``. See ``delete_one``.""" - return await self.delete_one({"name": name, "owner": owner}) + """Delete the single project group identified by ``name`` + ``owner``.""" + oid = await self._resolve_one_id({"name": name, "owner": owner}) + if oid is None: + raise NotFoundError(f"{self.document_model.__name__} not found", name=name, owner=owner) + if not (self._user.is_admin or owner == self._user.username): + raise PermissionError(required_role="owner-or-admin") + return await self.delete_by_id(oid) async def delete_project_groups(self, filter: ProjectGroupFilter) -> DeleteResponse: - """Bulk-delete every scoped project group matching ``filter``. See ``delete``.""" + """Bulk-delete project groups matching ``filter``, restricted to the caller's own. + + A non-admin's bulk delete is scoped to their own groups (overriding any ``owner`` in the + filter) so it can never remove public groups belonging to others. See ``delete``. + """ + if not self._user.is_admin: + filter.owner = self._user.username return await self.delete(filter) async def add_project_refs( diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py index 00e8373824..3a403e6f7b 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py @@ -13,7 +13,7 @@ ProjectOut, ProjectPatch, ) -from mpcontribs_api.exceptions import NotFoundError, PermissionError +from mpcontribs_api.exceptions import NotFoundError, PermissionError, ValidationError from mpcontribs_api.pagination import CursorParams @@ -33,10 +33,6 @@ class MongoDbProjectRepository(MongoDbRepository[Project, ProjectIn, ProjectOut, document_model = Project out_model = ProjectOut - def __init__(self, user: User) -> None: - super().__init__(user) - self._user = user - @staticmethod def _build_scope(user: User) -> dict[str, Any]: """Provides scope based on current user's permitted groups and publicly released data.""" @@ -91,24 +87,55 @@ async def insert_project(self, project: ProjectIn) -> Project: return await self.insert_one(project) async def patch_project_by_id(self, id: str, update: ProjectPatch) -> Project: - """Partially update a project by id, scoped to the current user. See ``patch``.""" + """Partially update a scoped project by id, enforcing approval rules. + + - Only an admin may change ``is_approved``. + - Resulting state must satisfy is_public <-> is_approved condition + + The ``initiative`` field is split out upstream in ``ProjectService.patch``, so it never + reaches this method. + """ + data = update.model_dump(exclude_unset=True) + if "is_approved" in data and not self._user.is_admin: + raise PermissionError(required_role="admin") + + existing = await self.document_model.find_one(self._scope, self.document_model.id == id) + if existing is None: + raise NotFoundError(self._not_found(id)) + + resulting_approved = data.get("is_approved", existing.is_approved) + resulting_public = data.get("is_public", existing.is_public) + if resulting_public and not resulting_approved: + raise ValidationError("a project cannot be public until it is approved", id=id) + return await self.patch(id, update) async def delete_project_by_id(self, id: str) -> None: - """Delete a project by id, scoped to the current user. See ``delete_by_id``.""" + """Delete a scoped project by id. Restricted to the owner or an admin. + + Visibility (public/approved or group membership) is not enough to delete: a project can + only be dissolved by its owner (or an admin). A caller who cannot see the project gets a + 404; a caller who can see it but does not own it gets a 403. + """ + existing = await self.document_model.find_one(self._scope, self.document_model.id == id) + if existing is None: + raise NotFoundError(self._not_found(id)) + if not (self._user.is_admin or existing.owner == self._user.username): + raise PermissionError(required_role="owner-or-admin") await self.delete_by_id(id) async def upsert_project_by_id(self, id: str, data: ProjectIn) -> Project: """Upsert a project by provided id, authorized to the current user. Update the document if the id exists, otherwise insert a new one under that id. - Authorization (the read scope is for visibility, not write access, so it is not - reused here): - **Existing project:** only its ``owner`` or an admin may overwrite it. The stored - ``owner`` is preserved — ownership cannot be reassigned through the request body. + ``owner`` is preserved - ownership cannot be reassigned through the request body. - **New project:** ``owner`` is forced to the caller, ignoring any body value. + an existing project keeps its stored approval and a new one starts unapproved. The resulting + document must also satisfy ``is_public ⇒ is_approved``. + Note: relies on the path param ``id`` for identity, not the body's id. Args: @@ -133,9 +160,18 @@ async def upsert_project_by_id(self, id: str, data: ProjectIn) -> Project: raise PermissionError(required_role="owner-or-admin") # Ownership is immutable via upsert; keep the original owner. project.owner = existing.owner + # Approval is admin-only; a non-admin keeps the project's stored approval state. + if not self._user.is_admin: + project.is_approved = existing.is_approved else: # New project: the caller owns it, regardless of the submitted owner. project.owner = self._user.username + # Approval is admin-only; a non-admin's new project always starts unapproved. + if not self._user.is_admin: + project.is_approved = False + + if project.is_public and not project.is_approved: + raise ValidationError("a project cannot be public until it is approved", id=id) return await project.save() async def set_initiative(self, id: str, ref: DBRef | None) -> Project: diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py index f95fbacb49..95c8120689 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py @@ -54,7 +54,8 @@ async def _resolve_initiative_assignment(self, project_id: str, slug: str | None if initiative is None: raise NotFoundError("Initiative not found or not visible", slug=slug) - if not self._initiatives._user.can_manage(id=initiative.slug, resource="initiative"): + user = self._initiatives._user + if not (user.can_manage(id=initiative.slug, resource="initiative") or initiative.owner == user.username): raise PermissionError( message="user does not have adequate acceess to this resource", required_role="initiative-owner-collaborator-or-admin", From 4db18945974ec486870543b18a78ede57335db92 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Thu, 16 Jul 2026 17:22:42 -0700 Subject: [PATCH 35/62] Initatives and ProjectGroup testing --- .../db/test_initiatives_repository.py | 61 ++++++++- .../db/test_initiatives_service.py | 38 +++++- .../db/test_project_groups_repository.py | 71 ++++++++++- .../db/test_project_groups_service.py | 15 +++ .../db/test_projects_repository.py | 118 +++++++++++++++++- .../integration/test_component_routes.py | 32 +++++ .../tests/integration/test_projects.py | 6 +- .../unit/domains/test_initiatives_models.py | 100 +++++++++++++++ .../domains/test_project_group_service.py | 5 + 9 files changed, 431 insertions(+), 15 deletions(-) create mode 100644 mpcontribs-api/tests/unit/domains/test_initiatives_models.py diff --git a/mpcontribs-api/tests/integration/db/test_initiatives_repository.py b/mpcontribs-api/tests/integration/db/test_initiatives_repository.py index f36f2a995a..c4aec88ccd 100644 --- a/mpcontribs-api/tests/integration/db/test_initiatives_repository.py +++ b/mpcontribs-api/tests/integration/db/test_initiatives_repository.py @@ -2,9 +2,15 @@ from mpcontribs_api.authz import User from mpcontribs_api.config import get_settings -from mpcontribs_api.domains.initiatives.models import Initiative, InitiativeIn, InitiativePatch +from mpcontribs_api.domains.initiatives.models import ( + Initiative, + InitiativeFilter, + InitiativeIn, + InitiativePatch, +) from mpcontribs_api.domains.initiatives.repository import InitiativeRepository from mpcontribs_api.exceptions import ConflictError, NotFoundError, PermissionError, ValidationError +from mpcontribs_api.pagination import CursorParams # Share the session event loop (see the projects repo test for why). pytestmark = [pytest.mark.db, pytest.mark.asyncio(loop_scope="session")] @@ -68,14 +74,14 @@ async def test_invalid_slug_rejected(self, db): class TestUnapprovedPerOwnerLimit: async def test_owner_capped_at_configured_unapproved(self, db): - limit = get_settings().initiatives.max_unapproved_per_owner + limit = get_settings().domain.initiatives.max_unapproved_per_owner for i in range(limit): await _insert(f"cap-{i}", ALICE) with pytest.raises(ConflictError): await _insert("cap-over", ALICE) async def test_approved_do_not_count_against_quota(self, db): - limit = get_settings().initiatives.max_unapproved_per_owner + limit = get_settings().domain.initiatives.max_unapproved_per_owner for i in range(limit): await _insert(f"quota-{i}", ALICE) await _approve("quota-0") # frees a slot @@ -83,7 +89,7 @@ async def test_approved_do_not_count_against_quota(self, db): assert await _insert("quota-extra", ALICE) is not None async def test_admin_is_exempt(self, db): - limit = get_settings().initiatives.max_unapproved_per_owner + limit = get_settings().domain.initiatives.max_unapproved_per_owner for i in range(limit + 2): await _repo(ADMIN).insert_initiative(InitiativeIn(slug=f"admin-{i}", name="x")) @@ -182,3 +188,50 @@ async def test_collaborator_cannot_delete(self, db): async def test_missing_is_not_found(self, db): with pytest.raises(NotFoundError): await _repo(ADMIN).delete_initiative("nope-missing") + + +# --------------------------------------------------------------------------- +# Listing + filtering (scoped) +# --------------------------------------------------------------------------- + + +class TestListAndFilter: + async def test_list_scoped_to_caller(self, db): + await _insert("mine-1", ALICE) + await _insert("bobs-1", BOB) # Bob's private initiative, invisible to Alice + page = await _repo(ALICE).get_initiatives(CursorParams(), InitiativeFilter(), fields=None) + slugs = {i.slug for i in page.items} + assert "mine-1" in slugs + assert "bobs-1" not in slugs + + async def test_filter_by_is_approved(self, db): + await _insert("appr-1", ALICE) + await _insert("unappr-1", ALICE) + await _approve("appr-1") + page = await _repo(ADMIN).get_initiatives(CursorParams(), InitiativeFilter(is_approved=True), fields=None) + slugs = {i.slug for i in page.items} + assert "appr-1" in slugs + assert "unappr-1" not in slugs + + async def test_filter_by_owner(self, db): + await _insert("owned-alice", ALICE) + await _insert("owned-bob", BOB) + page = await _repo(ADMIN).get_initiatives(CursorParams(), InitiativeFilter(owner=BOB_EMAIL), fields=None) + assert {i.slug for i in page.items} == {"owned-bob"} + + +# --------------------------------------------------------------------------- +# Admin bypass +# --------------------------------------------------------------------------- + + +class TestAdminBypass: + async def test_admin_can_patch_non_owned(self, db): + await _insert("admin-patch", ALICE) + patched = await _repo(ADMIN).patch_initiative("admin-patch", InitiativePatch(name="Admin Renamed")) + assert patched.name == "Admin Renamed" + + async def test_admin_can_delete_non_owned(self, db): + await _insert("admin-del", ALICE) + result = await _repo(ADMIN).delete_initiative("admin-del") + assert result.num_deleted == 1 diff --git a/mpcontribs-api/tests/integration/db/test_initiatives_service.py b/mpcontribs-api/tests/integration/db/test_initiatives_service.py index c10fbaf6e8..cf83f42e53 100644 --- a/mpcontribs-api/tests/integration/db/test_initiatives_service.py +++ b/mpcontribs-api/tests/integration/db/test_initiatives_service.py @@ -7,7 +7,7 @@ from mpcontribs_api.domains.initiatives.repository import InitiativeRepository from mpcontribs_api.domains.projects.models import Project, ProjectIn, ProjectPatch, Stats from mpcontribs_api.domains.projects.repository import MongoDbProjectRepository -from mpcontribs_api.domains.projects.service import ProjectInitiativeService +from mpcontribs_api.domains.projects.service import ProjectService from mpcontribs_api.exceptions import ConflictError, NotFoundError, PermissionError pytestmark = [pytest.mark.db, pytest.mark.asyncio(loop_scope="session")] @@ -27,8 +27,8 @@ STATS = Stats(columns=0, contributions=0, tables=0, structures=0, attachments=0, size=0.0) -def _service(user: User) -> ProjectInitiativeService: - return ProjectInitiativeService( +def _service(user: User) -> ProjectService: + return ProjectService( projects=MongoDbProjectRepository(user), initiatives=InitiativeRepository(user), ) @@ -144,7 +144,7 @@ async def test_assign_to_missing_initiative_is_not_found(self, db): class TestMemberCap: async def test_unapproved_capped_at_configured_members(self, db): - cap = get_settings().initiatives.max_projects_per_unapproved + cap = get_settings().domain.initiatives.max_projects_per_unapproved await _insert_initiative("init-cap", ALICE) for i in range(cap): await _insert_project(f"cap-proj-{i}", owner=ALICE_EMAIL) @@ -154,7 +154,7 @@ async def test_unapproved_capped_at_configured_members(self, db): await _service(ALICE).patch("cap-proj-over", ProjectPatch(initiative="init-cap")) async def test_reassigning_existing_member_is_idempotent(self, db): - cap = get_settings().initiatives.max_projects_per_unapproved + cap = get_settings().domain.initiatives.max_projects_per_unapproved await _insert_initiative("init-idem", ALICE) for i in range(cap): await _insert_project(f"idem-proj-{i}", owner=ALICE_EMAIL) @@ -164,7 +164,7 @@ async def test_reassigning_existing_member_is_idempotent(self, db): assert again.initiative is not None async def test_approved_initiative_has_no_member_cap(self, db): - cap = get_settings().initiatives.max_projects_per_unapproved + cap = get_settings().domain.initiatives.max_projects_per_unapproved await _insert_initiative("init-approved", ALICE) await InitiativeRepository(ADMIN).patch_initiative("init-approved", InitiativePatch(is_approved=True)) for i in range(cap + 2): # comfortably past the unapproved cap @@ -175,3 +175,29 @@ async def test_approved_initiative_has_no_member_cap(self, db): exclude_project_id=None, ) assert count == cap + 2 + + +# --------------------------------------------------------------------------- +# Admin bypass + unassignment rights +# --------------------------------------------------------------------------- + + +class TestAdminAndUnassign: + async def test_admin_can_assign_to_any_initiative(self, db): + # Alice's private initiative is manageable by an admin even though the admin holds no role. + await _insert_project("adm-proj", owner=ALICE_EMAIL) + init = await _insert_initiative("adm-init", ALICE) + updated = await _service(ADMIN).patch("adm-proj", ProjectPatch(initiative="adm-init")) + assert _assigned_id(updated) == init.id + + async def test_project_owner_can_unassign_without_initiative_rights(self, db): + # A collaborator assigns Bob's project; Bob, lacking any initiative role, can still detach + # his own project — unassignment needs only project-write access. + await _insert_project("detach-proj", owner=BOB_EMAIL) + await _insert_initiative("detach-init", ALICE) + await _service(_collaborator("detach-init", username=BOB_EMAIL)).patch( + "detach-proj", ProjectPatch(initiative="detach-init") + ) + bob_plain = User(username=BOB_EMAIL, groups=frozenset()) + updated = await _service(bob_plain).patch("detach-proj", ProjectPatch(initiative=None)) + assert _assigned_id(updated) is None diff --git a/mpcontribs-api/tests/integration/db/test_project_groups_repository.py b/mpcontribs-api/tests/integration/db/test_project_groups_repository.py index fc19448017..26928cfb9a 100644 --- a/mpcontribs-api/tests/integration/db/test_project_groups_repository.py +++ b/mpcontribs-api/tests/integration/db/test_project_groups_repository.py @@ -9,7 +9,8 @@ ProjectGroupPatch, ) from mpcontribs_api.domains.project_groups.repository import ProjectGroupRepository -from mpcontribs_api.exceptions import ConflictError, NotFoundError, ValidationError +from mpcontribs_api.exceptions import ConflictError, NotFoundError, PermissionError, ValidationError +from mpcontribs_api.pagination import CursorParams # Share the session event loop (see the projects repo test for why). pytestmark = [pytest.mark.db, pytest.mark.asyncio(loop_scope="session")] @@ -21,9 +22,11 @@ ADMIN = User(username="google:admin@example.com", groups=frozenset({"admin"})) ALICE = User(username="google:alice@example.com", groups=frozenset({"mp-team"})) +BOB = User(username="google:bob@example.com", groups=frozenset()) ANON = User() ALICE_EMAIL = "google:alice@example.com" +BOB_EMAIL = "google:bob@example.com" def _repo(user: User) -> ProjectGroupRepository: @@ -69,6 +72,50 @@ async def test_out_of_scope_returns_none(self, db): assert found is None +# --------------------------------------------------------------------------- +# project-group: role scoping +# --------------------------------------------------------------------------- + + +def _role_user(group_id, username: str = "google:carol@example.com") -> User: + """A non-owner authenticated user granted access to one group via its project-group role.""" + return User(username=username, groups=frozenset({f"project-group:{group_id}"})) + + +class TestGroupRoleScope: + async def test_role_grants_visibility(self, db): + group = await _insert("role-vis") # Alice's private group + found = await _repo(_role_user(group.id)).get_project_group(name="role-vis", owner=ALICE_EMAIL, fields=None) + assert found is not None + assert found.id == group.id + + async def test_without_role_not_visible(self, db): + await _insert("role-none") + found = await _repo(BOB).get_project_group(name="role-none", owner=ALICE_EMAIL, fields=None) + assert found is None + + async def test_malformed_role_is_ignored(self, db): + await _insert("role-bad") + member = User(username="google:carol@example.com", groups=frozenset({"project-group:not-an-oid"})) + # A malformed role id must not raise; it simply grants nothing. + found = await _repo(member).get_project_group(name="role-bad", owner=ALICE_EMAIL, fields=None) + assert found is None + + async def test_role_appears_in_listing(self, db): + group = await _insert("role-list") + page = await _repo(_role_user(group.id)).get_project_groups( + pagination=CursorParams(), filter=ProjectGroupFilter(), fields=None + ) + assert group.id in {g.id for g in page.items} + + async def test_role_grants_scope_but_not_delete(self, db): + # Scope makes the group visible, but deletion remains owner-or-admin (403 for a role holder). + group = await _insert("role-del") + with pytest.raises(PermissionError): + await _repo(_role_user(group.id)).delete_project_group(name="role-del", owner=ALICE_EMAIL) + assert await ProjectGroup.find_one(ProjectGroup.name == "role-del") is not None + + # --------------------------------------------------------------------------- # delete_one (identifier-keyed, single-resource, raises) # --------------------------------------------------------------------------- @@ -93,6 +140,18 @@ async def test_out_of_scope_raises_not_found(self, db): # ...and it is untouched. assert await ProjectGroup.find_one(ProjectGroup.name == "del-scoped") is not None + async def test_owner_can_delete_own(self, db): + await _insert("del-own", owner=ALICE_EMAIL) + result = await _repo(ALICE).delete_project_group(name="del-own", owner=ALICE_EMAIL) + assert result.num_deleted == 1 + + async def test_visible_public_non_owner_forbidden(self, db): + # Bob can *see* Alice's public group but does not own it → 403, and it is left intact. + await _insert("del-pub", owner=ALICE_EMAIL, is_public=True) + with pytest.raises(PermissionError): + await _repo(BOB).delete_project_group(name="del-pub", owner=ALICE_EMAIL) + assert await ProjectGroup.find_one(ProjectGroup.name == "del-pub") is not None + async def test_wrong_identifier_keys_raise_validation(self, db): with pytest.raises(ValidationError): await _repo(ADMIN).delete_one({"name": "x"}) # missing 'owner' @@ -166,6 +225,16 @@ async def test_no_match_returns_zero(self, db): ) assert result.num_deleted == 0 + async def test_non_admin_bulk_restricted_to_own(self, db): + # A broad filter from a non-admin is pinned to their own groups: a public group owned by + # someone else must survive even though the filter would otherwise match it. + await _insert("own-bulk", owner=ALICE_EMAIL, is_public=True) + await _insert("other-bulk", owner=BOB_EMAIL, is_public=True) + result = await _repo(ALICE).delete_project_groups(filter=ProjectGroupFilter(is_public=True)) + assert result.num_deleted == 1 + assert await ProjectGroup.find_one(ProjectGroup.name == "own-bulk") is None + assert await ProjectGroup.find_one(ProjectGroup.name == "other-bulk") is not None + # --------------------------------------------------------------------------- # insert_project_group diff --git a/mpcontribs-api/tests/integration/db/test_project_groups_service.py b/mpcontribs-api/tests/integration/db/test_project_groups_service.py index 5aac87b567..af7d188f85 100644 --- a/mpcontribs-api/tests/integration/db/test_project_groups_service.py +++ b/mpcontribs-api/tests/integration/db/test_project_groups_service.py @@ -134,6 +134,21 @@ async def test_ambiguous_identifiers_raise_conflict(self, db): # --------------------------------------------------------------------------- +class TestInsert: + async def test_non_admin_owner_forced_to_caller(self, db): + # Alice submits Bob as owner; the caller's identity must win so she can manage the group. + group = await _service(ALICE).insert( + ProjectGroupIn(name="ins-forced", owner=BOB_EMAIL, projects=[], description="d") + ) + assert group.owner == ALICE_EMAIL + + async def test_admin_may_set_owner_on_behalf(self, db): + group = await _service(ADMIN).insert( + ProjectGroupIn(name="ins-onbehalf", owner=BOB_EMAIL, projects=[], description="d") + ) + assert group.owner == BOB_EMAIL + + class TestDelete: async def test_delete_by_id_unlinks_project(self, db): group = await _insert_group("rm-id") diff --git a/mpcontribs-api/tests/integration/db/test_projects_repository.py b/mpcontribs-api/tests/integration/db/test_projects_repository.py index 7e33766870..f824d01c9e 100644 --- a/mpcontribs-api/tests/integration/db/test_projects_repository.py +++ b/mpcontribs-api/tests/integration/db/test_projects_repository.py @@ -3,7 +3,7 @@ from mpcontribs_api.authz import User from mpcontribs_api.domains.projects.models import Project, ProjectIn, ProjectOut, ProjectPatch, Stats from mpcontribs_api.domains.projects.repository import MongoDbProjectRepository -from mpcontribs_api.exceptions import ConflictError, NotFoundError +from mpcontribs_api.exceptions import ConflictError, NotFoundError, PermissionError, ValidationError from mpcontribs_api.pagination import CursorParams # All tests in this module share the session event loop so they can reuse the @@ -365,6 +365,38 @@ async def test_delete_nonexistent_throws_error(self, db): with pytest.raises(NotFoundError, match="not found"): await _repo(ADMIN).delete_project_by_id(id="ghost-id") + async def test_owner_can_delete_own_project(self, db): + await _insert("del-own", owner="google:alice@example.com") + await _repo(ALICE).delete_project_by_id(id="del-own") + assert await Project.find_one(Project.id == "del-own") is None + + async def test_admin_can_delete_any_project(self, db): + await _insert("del-admin", owner="google:alice@example.com") + await _repo(ADMIN).delete_project_by_id(id="del-admin") + assert await Project.find_one(Project.id == "del-admin") is None + + async def test_group_member_non_owner_cannot_delete(self, db): + # A user whose group contains the project slug can *see* it, but only the owner may delete. + member = User(username="google:carol@example.com", groups=frozenset({"del-grp"})) + await _insert("del-grp", owner="google:alice@example.com") + with pytest.raises(PermissionError): + await _repo(member).delete_project_by_id(id="del-grp") + assert await Project.find_one(Project.id == "del-grp") is not None + + async def test_visible_public_non_owner_cannot_delete(self, db): + # BOB can see the public+approved project but does not own it → 403, not a silent success. + await _insert("del-pub", owner="google:alice@example.com", is_public=True, is_approved=True) + with pytest.raises(PermissionError): + await _repo(BOB).delete_project_by_id(id="del-pub") + assert await Project.find_one(Project.id == "del-pub") is not None + + async def test_out_of_scope_delete_not_found(self, db): + # BOB cannot see Alice's private project → 404 (existence is not leaked as a 403). + await _insert("del-hidden", owner="google:alice@example.com", is_public=False) + with pytest.raises(NotFoundError): + await _repo(BOB).delete_project_by_id(id="del-hidden") + assert await Project.find_one(Project.id == "del-hidden") is not None + # --------------------------------------------------------------------------- # upsert_project_by_id @@ -438,3 +470,87 @@ async def test_update_preserves_original_owner(self, db): await _repo(ALICE).upsert_project_by_id(id="auth-preserve", data=data) found = await Project.find_one(Project.id == "auth-preserve") assert found.owner == "google:alice@example.com" + + +# --------------------------------------------------------------------------- +# is_approved is admin-only (patch + upsert) +# --------------------------------------------------------------------------- + + +class TestApprovalIsAdminOnly: + async def test_non_admin_cannot_patch_is_approved(self, db): + await _insert("appr-patch", owner="google:alice@example.com") + with pytest.raises(PermissionError): + await _repo(ALICE).patch_project_by_id(id="appr-patch", update=ProjectPatch(is_approved=True)) + found = await Project.find_one(Project.id == "appr-patch") + assert found.is_approved is False + + async def test_admin_can_patch_is_approved(self, db): + await _insert("appr-patch-admin", owner="google:alice@example.com") + await _repo(ADMIN).patch_project_by_id(id="appr-patch-admin", update=ProjectPatch(is_approved=True)) + found = await Project.find_one(Project.id == "appr-patch-admin") + assert found.is_approved is True + + async def test_non_admin_upsert_new_project_stays_unapproved(self, db): + # Body asks for approval; a non-admin's new project must start unapproved. + data = _project_in("appr-new", owner="google:bob@example.com", is_approved=True) + await _repo(BOB).upsert_project_by_id(id="appr-new", data=data) + found = await Project.find_one(Project.id == "appr-new") + assert found.is_approved is False + + async def test_non_admin_upsert_cannot_change_existing_approval(self, db): + # Admin seeds an approved project; the owner (non-admin) cannot un-approve it via PUT. + await _insert("appr-existing", owner="google:alice@example.com", is_approved=True) + data = _project_in("appr-existing", owner="google:alice@example.com", is_approved=False) + await _repo(ALICE).upsert_project_by_id(id="appr-existing", data=data) + found = await Project.find_one(Project.id == "appr-existing") + assert found.is_approved is True + + async def test_admin_upsert_can_approve(self, db): + data = _project_in("appr-admin-new", is_approved=True) + await _repo(ADMIN).upsert_project_by_id(id="appr-admin-new", data=data) + found = await Project.find_one(Project.id == "appr-admin-new") + assert found.is_approved is True + + +# --------------------------------------------------------------------------- +# a project cannot be public unless approved +# --------------------------------------------------------------------------- + + +class TestPublicRequiresApproved: + async def test_patch_public_on_unapproved_rejected(self, db): + await _insert("pub-unappr", owner="google:alice@example.com", is_approved=False) + with pytest.raises(ValidationError, match="approved"): + await _repo(ADMIN).patch_project_by_id(id="pub-unappr", update=ProjectPatch(is_public=True)) + found = await Project.find_one(Project.id == "pub-unappr") + assert found.is_public is False + + async def test_patch_public_and_approved_together_succeeds(self, db): + await _insert("pub-both", owner="google:alice@example.com", is_approved=False) + await _repo(ADMIN).patch_project_by_id( + id="pub-both", update=ProjectPatch(is_public=True, is_approved=True) + ) + found = await Project.find_one(Project.id == "pub-both") + assert found.is_public is True + assert found.is_approved is True + + async def test_patch_public_on_approved_succeeds(self, db): + await _insert("pub-approved", owner="google:alice@example.com", is_approved=True) + await _repo(ADMIN).patch_project_by_id(id="pub-approved", update=ProjectPatch(is_public=True)) + found = await Project.find_one(Project.id == "pub-approved") + assert found.is_public is True + + async def test_upsert_public_unapproved_rejected(self, db): + # A non-admin's public+approved body is coerced to unapproved, which then violates the invariant. + data = _project_in("pub-upsert", owner="google:bob@example.com", is_public=True, is_approved=True) + with pytest.raises(ValidationError, match="approved"): + await _repo(BOB).upsert_project_by_id(id="pub-upsert", data=data) + assert await Project.find_one(Project.id == "pub-upsert") is None + + async def test_admin_upsert_public_approved_succeeds(self, db): + data = _project_in("pub-upsert-admin", is_public=True, is_approved=True) + await _repo(ADMIN).upsert_project_by_id(id="pub-upsert-admin", data=data) + found = await Project.find_one(Project.id == "pub-upsert-admin") + assert found.is_public is True + assert found.is_approved is True diff --git a/mpcontribs-api/tests/integration/test_component_routes.py b/mpcontribs-api/tests/integration/test_component_routes.py index c1b49cd8ba..f47fb1ae64 100644 --- a/mpcontribs-api/tests/integration/test_component_routes.py +++ b/mpcontribs-api/tests/integration/test_component_routes.py @@ -331,3 +331,35 @@ def test_structures_get_still_open_to_anon(self, client, structure_service): structure_service.get_many.return_value = Page(items=[], next_cursor=None) r = client.get("/api/v1/structures", headers=FORCE_ANON_HEADERS) assert r.status_code == 200 + + +# =========================================================================== +# Component inserts require the caller be a writer of at least one project +# (authenticated alone is not enough — require_writer). +# =========================================================================== + +# Authenticated, but carries no groups -> no writable projects. Override the default +# groups header (AUTHED_HEADERS sets mp-team) to empty so the caller is a non-writer. +NON_WRITER_HEADERS = { + "x-consumer-username": "google:nogroups@example.com", + "x-authenticated-groups": "", +} + + +class TestComponentInsertRequiresWriter: + def test_structures_post_non_writer_403(self, client, structure_service): + r = client.post("/api/v1/structures", json=[], headers=NON_WRITER_HEADERS) + assert r.status_code == 403 + structure_service.insert.assert_not_called() + + def test_tables_post_non_writer_403(self, client, table_service): + r = client.post("/api/v1/tables", json=[], headers=NON_WRITER_HEADERS) + assert r.status_code == 403 + table_service.insert.assert_not_called() + + def test_structures_post_writer_allowed(self, client, structure_service): + # The default AUTHED_HEADERS identity carries the mp-team group -> writer. + structure_service.insert.return_value = {"total": 0, "succeeded": [], "failed": []} + r = client.post("/api/v1/structures", json=[]) + assert r.status_code == 200 + structure_service.insert.assert_awaited_once() diff --git a/mpcontribs-api/tests/integration/test_projects.py b/mpcontribs-api/tests/integration/test_projects.py index e157665880..a7ff999b55 100644 --- a/mpcontribs-api/tests/integration/test_projects.py +++ b/mpcontribs-api/tests/integration/test_projects.py @@ -2,7 +2,7 @@ import pytest -from mpcontribs_api.domains.projects.dependencies import get_project_initiative_service, get_scoped_projects +from mpcontribs_api.domains.projects.dependencies import get_project_service, get_scoped_projects from mpcontribs_api.domains.projects.models import ProjectOut, Stats from mpcontribs_api.exceptions import ConflictError, NotFoundError from mpcontribs_api.pagination import Page @@ -43,9 +43,9 @@ def project_repo(test_app, mock_project_repo): def project_service(test_app): """Override the assignment service the PATCH route depends on with an async mock.""" service = AsyncMock() - test_app.dependency_overrides[get_project_initiative_service] = lambda: service + test_app.dependency_overrides[get_project_service] = lambda: service yield service - test_app.dependency_overrides.pop(get_project_initiative_service, None) + test_app.dependency_overrides.pop(get_project_service, None) # --------------------------------------------------------------------------- diff --git a/mpcontribs-api/tests/unit/domains/test_initiatives_models.py b/mpcontribs-api/tests/unit/domains/test_initiatives_models.py new file mode 100644 index 0000000000..956c82c137 --- /dev/null +++ b/mpcontribs-api/tests/unit/domains/test_initiatives_models.py @@ -0,0 +1,100 @@ +import pytest +from beanie import PydanticObjectId +from pydantic import ValidationError as PydanticValidationError + +from mpcontribs_api.domains.initiatives.models import ( + Initiative, + InitiativeIn, + InitiativeOut, + InitiativePatch, +) +from mpcontribs_api.exceptions import ValidationError + +OID = PydanticObjectId() +OWNER = "google:alice@example.com" + + +def _init(**overrides) -> Initiative: + payload = {"_id": OID, "slug": "battery-genome", "name": "Battery Genome", "owner": OWNER} + payload.update(overrides) + return Initiative.model_validate(payload) + + +# --------------------------------------------------------------------------- +# Slug validation / normalisation +# --------------------------------------------------------------------------- + + +class TestSlug: + def test_lowercases_and_strips(self): + assert _init(slug=" Battery-Genome-2025 ").slug == "battery-genome-2025" + + @pytest.mark.parametrize("bad", ["has space", "under_score", "trailing-", "-leading", "sym!bol", "Dou--ble"]) + def test_rejects_malformed_slug(self, bad): + with pytest.raises(ValidationError): + _init(slug=bad) + + def test_rejects_too_short_via_length(self): + # "ab" is well-formed but below the 3-char minimum, so the length constraint rejects it. + with pytest.raises(PydanticValidationError): + _init(slug="ab") + + +# --------------------------------------------------------------------------- +# Initiative document invariants +# --------------------------------------------------------------------------- + + +class TestInitiative: + def test_defaults_private_unapproved(self): + init = _init() + assert init.is_public is False + assert init.is_approved is False + + def test_public_and_approved_ok(self): + init = _init(is_public=True, is_approved=True) + assert init.is_public is True + + def test_public_without_approved_rejected(self): + with pytest.raises(ValidationError): + _init(is_public=True, is_approved=False) + + +# --------------------------------------------------------------------------- +# InitiativeIn (create contract) +# --------------------------------------------------------------------------- + + +class TestInitiativeIn: + def test_minimal_valid(self): + data = InitiativeIn(slug="battery-genome", name="Battery Genome") + assert data.slug == "battery-genome" + + @pytest.mark.parametrize("field", ["owner", "is_public", "is_approved", "id", "unknown"]) + def test_forbids_server_controlled_and_unknown_fields(self, field): + # owner and the flags are forced server-side; none are part of the input contract. + with pytest.raises(PydanticValidationError): + InitiativeIn(slug="battery-genome", name="Battery Genome", **{field: "x"}) + + def test_normalises_slug(self): + assert InitiativeIn(slug="Battery-Genome", name="x").slug == "battery-genome" + + +# --------------------------------------------------------------------------- +# InitiativeOut / InitiativePatch shape +# --------------------------------------------------------------------------- + + +class TestOutAndPatch: + def test_out_populates_id_from_alias(self): + out = InitiativeOut.model_validate({"_id": OID, "slug": "battery-genome"}) + assert out.id == OID + + def test_patch_tracks_only_set_fields(self): + patch = InitiativePatch(name="Renamed") + assert patch.model_dump(exclude_unset=True) == {"name": "Renamed"} + + def test_patch_has_no_slug_or_owner_field(self): + # slug and owner are immutable, so they are not part of the patch surface. + assert "slug" not in InitiativePatch.model_fields + assert "owner" not in InitiativePatch.model_fields diff --git a/mpcontribs-api/tests/unit/domains/test_project_group_service.py b/mpcontribs-api/tests/unit/domains/test_project_group_service.py index d1de793a28..d5ab9ef0de 100644 --- a/mpcontribs-api/tests/unit/domains/test_project_group_service.py +++ b/mpcontribs-api/tests/unit/domains/test_project_group_service.py @@ -4,6 +4,7 @@ from beanie import Link, PydanticObjectId from bson import DBRef +from mpcontribs_api.authz import User from mpcontribs_api.domains.project_groups.models import ProjectGroupIn, ProjectGroupOut from mpcontribs_api.domains.project_groups.service import ProjectGroupService from mpcontribs_api.domains.projects.models import Project @@ -27,6 +28,10 @@ def _make_service(group: ProjectGroupOut | None, *, visible_projects: set[str] | visible = visible_projects or set() groups = AsyncMock() projects = AsyncMock() + # insert() forces owner to the caller for non-admins; give the stub an admin user so these + # payload-identity assertions exercise the pass-through path (owner-forcing is covered end-to-end + # in the db service test). + groups._user = User(username="google:admin@example.com", groups=frozenset({"admin"})) if ambiguous: groups.get_one.side_effect = ConflictError("ambiguous") From 6c1b1d32e04e2a0161faac9bd0a6f8411022611d Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Fri, 17 Jul 2026 08:53:56 -0700 Subject: [PATCH 36/62] Added mp_category for MP defined categories. Improve ProjectIn to exclude server-defined fields --- .../mpcontribs_api/domains/projects/models.py | 46 ++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py index d8b307b026..fe03054ee4 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py @@ -24,12 +24,12 @@ def segments(self) -> tuple[str, ...]: class Stats(BaseModel): - columns: int - contributions: int - tables: int - structures: int - attachments: int - size: float + columns: int = 0 + contributions: int = 0 + tables: int = 0 + structures: int = 0 + attachments: int = 0 + size: float = 0 class Reference(BaseModel): @@ -50,10 +50,11 @@ class Project(BaseDocumentWithInput[ShortStr]): description: str owner: PrefixedEmail unique_identifiers: bool - stats: Stats # Optional + stats: Stats = Field(default_factory=Stats) tags: list[ShortStr] | None = None + mp_category: str | None = None references: list[Reference] = Field(default_factory=list) long_title: str | None = None other: dict[str, Any] = Field(default_factory=dict) @@ -64,7 +65,6 @@ class Project(BaseDocumentWithInput[ShortStr]): initiative: Link[Initiative] | None = None - # Empty method for now. Keeping for business logic later @classmethod def from_input_model(cls, data: ProjectIn) -> Project: return cls(**data.model_dump()) @@ -77,6 +77,10 @@ def decode_cursor(cursor: str) -> str: """ return pagination.decode_cursor(cursor) + @classmethod + def server_managed_fields(cls) -> tuple: + return ("is_public", "is_approved", "stats", "mp_category") + class Settings: name = "projects" keep_nulls = False @@ -90,6 +94,7 @@ class ProjectOut(DocumentOut[ShortStr]): description: str | None = None title: ShortStr | None = None tags: list[ShortStr] | None = None + mp_category: str | None = None owner: PrefixedEmail | None = None other: dict[str, Any] | None = None is_public: bool | None = None @@ -128,6 +133,11 @@ class ProjectFilter(BaseFilter): tags__in: list[ShortStr] | None = None # if at least one tag is present tags__contains: list[ShortStr] | None = None # Project.tags must be a superset of these + mp_category: str | None = None + mp_category__in: list[str] | None = None + mp_category__neq: str | None = None + mp_category__ilike: str | None = None + # fuzzy only long_title__ilike: str | None = None @@ -145,11 +155,25 @@ class Constants(BaseFilter.Constants): model = Project -# Keeping for business logic separation. May have specific implementation later -class ProjectIn(Project): +class ProjectIn(BaseModel): """Representation of user-supplied input.""" - pass + id: ShortStr + title: ShortStr + authors: str + description: str + owner: PrefixedEmail + unique_identifiers: bool + + # Optional + tags: list[ShortStr] | None = None + references: list[Reference] = Field(default_factory=list) + long_title: str | None = None + other: dict[str, Any] = Field(default_factory=dict) + columns: list[Column] = Field(default_factory=list) + license: Literal["CCA4", "CCPD"] | None = None + + initiative: Link[Initiative] | None = None class ProjectPatch(BaseModel): From e3c3f16c27bfe3a304b1fe97101ebc1877505e63 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Fri, 17 Jul 2026 08:54:14 -0700 Subject: [PATCH 37/62] Exclude server-managed fields from patching --- .../domains/projects/repository.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py index 3a403e6f7b..6bd677024a 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py @@ -130,11 +130,11 @@ async def upsert_project_by_id(self, id: str, data: ProjectIn) -> Project: Update the document if the id exists, otherwise insert a new one under that id. - **Existing project:** only its ``owner`` or an admin may overwrite it. The stored - ``owner`` is preserved - ownership cannot be reassigned through the request body. - - **New project:** ``owner`` is forced to the caller, ignoring any body value. - - an existing project keeps its stored approval and a new one starts unapproved. The resulting - document must also satisfy ``is_public ⇒ is_approved``. + ``owner`` and all server-managed fields (see ``Project.server_managed_fields``) are + preserved - ``ProjectIn`` cannot carry them, so a PUT never resets approval, publication, + or stats. + - **New project:** ``owner`` is forced to the caller; server-managed fields keep their + defaults Note: relies on the path param ``id`` for identity, not the body's id. @@ -160,9 +160,9 @@ async def upsert_project_by_id(self, id: str, data: ProjectIn) -> Project: raise PermissionError(required_role="owner-or-admin") # Ownership is immutable via upsert; keep the original owner. project.owner = existing.owner - # Approval is admin-only; a non-admin keeps the project's stored approval state. - if not self._user.is_admin: - project.is_approved = existing.is_approved + # make sure a full replacement doesn't overwrite server-defined fields + for field in self.document_model.server_managed_fields(): + setattr(project, field, getattr(existing, field)) else: # New project: the caller owns it, regardless of the submitted owner. project.owner = self._user.username From 20961e7216834ffff45fdac744ffba955de19c91 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Fri, 17 Jul 2026 08:54:45 -0700 Subject: [PATCH 38/62] Syncing tests with new Project models and methods --- .../db/test_initiatives_service.py | 6 +- .../db/test_project_groups_service.py | 6 +- .../db/test_projects_repository.py | 108 ++++++++++-------- .../tests/integration/test_projects.py | 6 +- .../unit/domains/test_projects_models.py | 18 +-- 5 files changed, 75 insertions(+), 69 deletions(-) diff --git a/mpcontribs-api/tests/integration/db/test_initiatives_service.py b/mpcontribs-api/tests/integration/db/test_initiatives_service.py index cf83f42e53..061f8acad0 100644 --- a/mpcontribs-api/tests/integration/db/test_initiatives_service.py +++ b/mpcontribs-api/tests/integration/db/test_initiatives_service.py @@ -5,7 +5,7 @@ from mpcontribs_api.config import get_settings from mpcontribs_api.domains.initiatives.models import InitiativeIn, InitiativePatch from mpcontribs_api.domains.initiatives.repository import InitiativeRepository -from mpcontribs_api.domains.projects.models import Project, ProjectIn, ProjectPatch, Stats +from mpcontribs_api.domains.projects.models import Project, ProjectIn, ProjectPatch from mpcontribs_api.domains.projects.repository import MongoDbProjectRepository from mpcontribs_api.domains.projects.service import ProjectService from mpcontribs_api.exceptions import ConflictError, NotFoundError, PermissionError @@ -24,7 +24,6 @@ ALICE_EMAIL = "google:alice@example.com" BOB_EMAIL = "google:bob@example.com" CAROL_EMAIL = "google:carol@example.com" -STATS = Stats(columns=0, contributions=0, tables=0, structures=0, attachments=0, size=0.0) def _service(user: User) -> ProjectService: @@ -41,13 +40,12 @@ def _collaborator(slug: str, username: str = BOB_EMAIL) -> User: async def _insert_project(pid: str, owner: str = ALICE_EMAIL) -> Project: return await MongoDbProjectRepository(ADMIN).insert_project( ProjectIn( - _id=pid, + id=pid, title=pid[:30], authors="Author", description="desc", owner=owner, unique_identifiers=True, - stats=STATS, ) ) diff --git a/mpcontribs-api/tests/integration/db/test_project_groups_service.py b/mpcontribs-api/tests/integration/db/test_project_groups_service.py index af7d188f85..4c9d5acb45 100644 --- a/mpcontribs-api/tests/integration/db/test_project_groups_service.py +++ b/mpcontribs-api/tests/integration/db/test_project_groups_service.py @@ -5,7 +5,7 @@ from mpcontribs_api.domains.project_groups.models import ProjectGroup, ProjectGroupIn from mpcontribs_api.domains.project_groups.repository import ProjectGroupRepository from mpcontribs_api.domains.project_groups.service import ProjectGroupService -from mpcontribs_api.domains.projects.models import ProjectIn, Stats +from mpcontribs_api.domains.projects.models import ProjectIn from mpcontribs_api.domains.projects.repository import MongoDbProjectRepository from mpcontribs_api.exceptions import ConflictError, NotFoundError @@ -22,7 +22,6 @@ ALICE_EMAIL = "google:alice@example.com" BOB_EMAIL = "google:bob@example.com" -STATS = Stats(columns=0, contributions=0, tables=0, structures=0, attachments=0, size=0.0) def _service(user: User = ADMIN) -> ProjectGroupService: @@ -31,13 +30,12 @@ def _service(user: User = ADMIN) -> ProjectGroupService: async def _insert_project(pid: str, owner: str = ALICE_EMAIL, **overrides): payload = { - "_id": pid, + "id": pid, "title": pid[:30], "authors": "Author", "description": "desc", "owner": owner, "unique_identifiers": True, - "stats": STATS, } payload.update(overrides) return await MongoDbProjectRepository(ADMIN).insert_project(ProjectIn(**payload)) diff --git a/mpcontribs-api/tests/integration/db/test_projects_repository.py b/mpcontribs-api/tests/integration/db/test_projects_repository.py index f824d01c9e..ece6566b7e 100644 --- a/mpcontribs-api/tests/integration/db/test_projects_repository.py +++ b/mpcontribs-api/tests/integration/db/test_projects_repository.py @@ -16,8 +16,6 @@ # Helpers # --------------------------------------------------------------------------- -STATS = Stats(columns=0, contributions=0, tables=0, structures=0, attachments=0, size=0.0) - ADMIN = User(username="google:admin@example.com", groups=frozenset({"admin"})) ALICE = User(username="google:alice@example.com", groups=frozenset({"mp-team"})) ANON = User() @@ -28,22 +26,36 @@ def _repo(user: User) -> MongoDbProjectRepository: def _project_in(id: str, **overrides) -> ProjectIn: + """Build a user-supplied ``ProjectIn`` (content fields only — no server-managed fields).""" defaults = { - "_id": id, + "id": id, "title": id[:30], "authors": "Test Author", "description": "Test description", "owner": "google:alice@example.com", "unique_identifiers": True, - "stats": STATS, } defaults.update(overrides) return ProjectIn(**defaults) async def _insert(id: str, **overrides) -> Project: - project_in = _project_in(id, **overrides) - return await _repo(ADMIN).insert_project(project_in) + """Seed a Project document directly. + + ``ProjectIn`` no longer carries server-managed fields, so seeding a specific ``is_public`` / + ``is_approved`` / ``stats`` state (as the scope tests need) is done by building the stored + ``Project`` and inserting it, bypassing the input contract. + """ + defaults = { + "_id": id, + "title": id[:30], + "authors": "Test Author", + "description": "Test description", + "owner": "google:alice@example.com", + "unique_identifiers": True, + } + defaults.update(overrides) + return await Project(**defaults).insert() # --------------------------------------------------------------------------- @@ -53,25 +65,22 @@ async def _insert(id: str, **overrides) -> Project: class TestInsertProject: async def test_inserted_project_is_retrievable(self, db): - await _insert("ins-basic") + await _repo(ADMIN).insert_project(_project_in("ins-basic")) found = await Project.find_one(Project.id == "ins-basic") assert found is not None assert found.id == "ins-basic" async def test_duplicate_id_raises_conflict(self, db): - await _insert("ins-dup") + await _repo(ADMIN).insert_project(_project_in("ins-dup")) with pytest.raises(ConflictError): - await _insert("ins-dup") + await _repo(ADMIN).insert_project(_project_in("ins-dup")) - async def test_default_not_public(self, db): - await _insert("ins-priv") + async def test_insert_defaults_private_and_unapproved(self, db): + # ProjectIn carries no is_public/is_approved, so an inserted project is private and unapproved. + await _repo(ADMIN).insert_project(_project_in("ins-priv")) found = await Project.find_one(Project.id == "ins-priv") assert found.is_public is False - - async def test_explicit_public(self, db): - await _insert("ins-pub", is_public=True, is_approved=True) - found = await Project.find_one(Project.id == "ins-pub") - assert found.is_public is True + assert found.is_approved is False # --------------------------------------------------------------------------- @@ -473,7 +482,7 @@ async def test_update_preserves_original_owner(self, db): # --------------------------------------------------------------------------- -# is_approved is admin-only (patch + upsert) +# is_approved is admin-only (via PATCH — ProjectIn cannot carry it) # --------------------------------------------------------------------------- @@ -491,30 +500,9 @@ async def test_admin_can_patch_is_approved(self, db): found = await Project.find_one(Project.id == "appr-patch-admin") assert found.is_approved is True - async def test_non_admin_upsert_new_project_stays_unapproved(self, db): - # Body asks for approval; a non-admin's new project must start unapproved. - data = _project_in("appr-new", owner="google:bob@example.com", is_approved=True) - await _repo(BOB).upsert_project_by_id(id="appr-new", data=data) - found = await Project.find_one(Project.id == "appr-new") - assert found.is_approved is False - - async def test_non_admin_upsert_cannot_change_existing_approval(self, db): - # Admin seeds an approved project; the owner (non-admin) cannot un-approve it via PUT. - await _insert("appr-existing", owner="google:alice@example.com", is_approved=True) - data = _project_in("appr-existing", owner="google:alice@example.com", is_approved=False) - await _repo(ALICE).upsert_project_by_id(id="appr-existing", data=data) - found = await Project.find_one(Project.id == "appr-existing") - assert found.is_approved is True - - async def test_admin_upsert_can_approve(self, db): - data = _project_in("appr-admin-new", is_approved=True) - await _repo(ADMIN).upsert_project_by_id(id="appr-admin-new", data=data) - found = await Project.find_one(Project.id == "appr-admin-new") - assert found.is_approved is True - # --------------------------------------------------------------------------- -# a project cannot be public unless approved +# a project cannot be public unless approved (enforced on PATCH) # --------------------------------------------------------------------------- @@ -541,16 +529,38 @@ async def test_patch_public_on_approved_succeeds(self, db): found = await Project.find_one(Project.id == "pub-approved") assert found.is_public is True - async def test_upsert_public_unapproved_rejected(self, db): - # A non-admin's public+approved body is coerced to unapproved, which then violates the invariant. - data = _project_in("pub-upsert", owner="google:bob@example.com", is_public=True, is_approved=True) - with pytest.raises(ValidationError, match="approved"): - await _repo(BOB).upsert_project_by_id(id="pub-upsert", data=data) - assert await Project.find_one(Project.id == "pub-upsert") is None - async def test_admin_upsert_public_approved_succeeds(self, db): - data = _project_in("pub-upsert-admin", is_public=True, is_approved=True) - await _repo(ADMIN).upsert_project_by_id(id="pub-upsert-admin", data=data) - found = await Project.find_one(Project.id == "pub-upsert-admin") +# --------------------------------------------------------------------------- +# upsert (PUT) cannot set server-managed fields; it preserves them on update +# --------------------------------------------------------------------------- + + +class TestUpsertServerManagedFields: + async def test_new_project_is_private_and_unapproved(self, db): + # ProjectIn has no is_public/is_approved, so a new PUT project starts safe by default. + await _repo(BOB).upsert_project_by_id(id="srv-new", data=_project_in("srv-new")) + found = await Project.find_one(Project.id == "srv-new") + assert found.is_public is False + assert found.is_approved is False + + async def test_admin_upsert_cannot_approve_via_body(self, db): + # Approval is PATCH-only even for an admin; a PUT can never approve a project. + await _repo(ADMIN).upsert_project_by_id(id="srv-admin-new", data=_project_in("srv-admin-new")) + found = await Project.find_one(Project.id == "srv-admin-new") + assert found.is_approved is False + + async def test_update_preserves_public_and_approved(self, db): + # A full-replace PUT by the owner must not wipe server-managed publication/approval. + await _insert("srv-preserve", owner="google:alice@example.com", is_public=True, is_approved=True) + data = _project_in("srv-preserve", owner="google:alice@example.com", title="Renamed Title") + await _repo(ALICE).upsert_project_by_id(id="srv-preserve", data=data) + found = await Project.find_one(Project.id == "srv-preserve") + assert found.title == "Renamed Title" # content fields still update assert found.is_public is True assert found.is_approved is True + + async def test_update_preserves_stats(self, db): + await _insert("srv-stats", owner="google:alice@example.com", stats=Stats(contributions=7)) + await _repo(ALICE).upsert_project_by_id(id="srv-stats", data=_project_in("srv-stats")) + found = await Project.find_one(Project.id == "srv-stats") + assert found.stats.contributions == 7 diff --git a/mpcontribs-api/tests/integration/test_projects.py b/mpcontribs-api/tests/integration/test_projects.py index a7ff999b55..e5fcf39a19 100644 --- a/mpcontribs-api/tests/integration/test_projects.py +++ b/mpcontribs-api/tests/integration/test_projects.py @@ -244,13 +244,12 @@ def test_id_forwarded_to_repo(self, client, project_repo): class TestUpsertProject: def _valid_body(self, **overrides): body = { - "_id": "mp-sample", + "id": "mp-sample", "title": "Test Project", "authors": "Alice", "description": "A project", "owner": "google:alice@example.com", "unique_identifiers": True, - "stats": {"columns": 0, "contributions": 0, "tables": 0, "structures": 0, "attachments": 0, "size": 0.0}, } body.update(overrides) return body @@ -280,13 +279,12 @@ def test_missing_required_field_returns_422(self, client, project_repo): class TestProjectMutationsRequireAuth: def _body(self): return { - "_id": "mp-sample", + "id": "mp-sample", "title": "Test Project", "authors": "Alice", "description": "A project", "owner": "google:alice@example.com", "unique_identifiers": True, - "stats": {"columns": 0, "contributions": 0, "tables": 0, "structures": 0, "attachments": 0, "size": 0.0}, } def test_anonymous_put_returns_401(self, client, project_repo): diff --git a/mpcontribs-api/tests/unit/domains/test_projects_models.py b/mpcontribs-api/tests/unit/domains/test_projects_models.py index b874091e73..70f1cab883 100644 --- a/mpcontribs-api/tests/unit/domains/test_projects_models.py +++ b/mpcontribs-api/tests/unit/domains/test_projects_models.py @@ -59,9 +59,15 @@ def test_zero_values_allowed(self): stats = Stats(columns=0, contributions=0, tables=0, structures=0, attachments=0, size=0.0) assert stats.contributions == 0 - def test_missing_field_raises(self): - with pytest.raises(PydanticValidationError): - Stats(columns=1, contributions=2, tables=3, structures=4, attachments=5) # missing size + def test_fields_default_to_zero(self): + # Stats is server-computed and every field defaults to zero, so an empty Stats is valid. + stats = Stats() + assert stats.columns == 0 + assert stats.contributions == 0 + assert stats.tables == 0 + assert stats.structures == 0 + assert stats.attachments == 0 + assert stats.size == 0.0 # --------------------------------------------------------------------------- @@ -196,19 +202,15 @@ def test_invalid_license_raises(self): # --------------------------------------------------------------------------- -VALID_STATS = Stats(columns=0, contributions=0, tables=0, structures=0, attachments=0, size=0.0) - - class TestProjectFromInputModel: def _make_input(self, **overrides): defaults = { - "_id": "test-proj", + "id": "test-proj", "title": "Test Project", "authors": "Alice, Bob", "description": "A test project", "owner": "google:alice@example.com", "unique_identifiers": True, - "stats": VALID_STATS, } defaults.update(overrides) return ProjectIn(**defaults) From 975fe9db8c060cdb86505b83d81d436ca3782f41 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Fri, 17 Jul 2026 10:42:36 -0700 Subject: [PATCH 39/62] Switched tags to be SearchStr to improve usability --- .../mpcontribs_api/domains/projects/models.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py index fe03054ee4..08d1e08b5e 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py @@ -8,7 +8,7 @@ from mpcontribs_api import pagination from mpcontribs_api.domains._shared.filters import BaseFilter from mpcontribs_api.domains._shared.models import BaseDocumentWithInput, DocumentOut -from mpcontribs_api.domains._shared.types import PrefixedEmail, ShortStr +from mpcontribs_api.domains._shared.types import PrefixedEmail, SearchStr, ShortStr from mpcontribs_api.domains.initiatives.models import Initiative @@ -53,7 +53,7 @@ class Project(BaseDocumentWithInput[ShortStr]): # Optional stats: Stats = Field(default_factory=Stats) - tags: list[ShortStr] | None = None + tags: list[SearchStr] | None = None mp_category: str | None = None references: list[Reference] = Field(default_factory=list) long_title: str | None = None @@ -93,7 +93,7 @@ class ProjectOut(DocumentOut[ShortStr]): authors: str | None = None description: str | None = None title: ShortStr | None = None - tags: list[ShortStr] | None = None + tags: list[SearchStr] | None = None mp_category: str | None = None owner: PrefixedEmail | None = None other: dict[str, Any] | None = None @@ -129,9 +129,9 @@ class ProjectFilter(BaseFilter): owner__neq: PrefixedEmail | None = None owner__ilike: str | None = None - tags: list[ShortStr] | None = None # exact match of list - tags__in: list[ShortStr] | None = None # if at least one tag is present - tags__contains: list[ShortStr] | None = None # Project.tags must be a superset of these + tags: list[SearchStr] | None = None # exact match of list + tags__in: list[SearchStr] | None = None # if at least one tag is present + tags__contains: list[SearchStr] | None = None # Project.tags must be a superset of these mp_category: str | None = None mp_category__in: list[str] | None = None @@ -166,7 +166,7 @@ class ProjectIn(BaseModel): unique_identifiers: bool # Optional - tags: list[ShortStr] | None = None + tags: list[SearchStr] | None = None references: list[Reference] = Field(default_factory=list) long_title: str | None = None other: dict[str, Any] = Field(default_factory=dict) @@ -182,7 +182,7 @@ class ProjectPatch(BaseModel): title: ShortStr | None = None authors: str | None = None description: str | None = None - tags: list[ShortStr] | None = None + tags: list[SearchStr] | None = None owner: PrefixedEmail | None = None unique_identifiers: bool | None = None references: list[Reference] = Field(default_factory=list) From 22207a9503efa822e8bc139c9254d5c1a96648d9 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Fri, 17 Jul 2026 10:43:46 -0700 Subject: [PATCH 40/62] Tests for tags as SearchStr --- .../unit/domains/test_search_str_tags.py | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 mpcontribs-api/tests/unit/domains/test_search_str_tags.py diff --git a/mpcontribs-api/tests/unit/domains/test_search_str_tags.py b/mpcontribs-api/tests/unit/domains/test_search_str_tags.py new file mode 100644 index 0000000000..01ae17359d --- /dev/null +++ b/mpcontribs-api/tests/unit/domains/test_search_str_tags.py @@ -0,0 +1,128 @@ +"""Fuzz the ``SearchStr`` normalizer behind ``tags`` (and the other places it is reused). + +``tags`` is ``list[SearchStr]`` on ``ProjectIn`` / ``ProjectOut`` / ``ProjectPatch`` and on every +``ProjectFilter`` operator (``tags`` / ``tags__in`` / ``tags__contains``); ``SearchStr`` also backs +other fields such as ``ProjectGroupFilter.name``. ``SearchStr`` runs ``_nfkc_casefold`` — NFKC +compatibility fold, whitespace strip, then casefold — so a stored tag and a query tag always +collapse to the same bytes. These tests hit the tricky unicode edges and prove the normalizer is +applied identically wherever it is declared. + +All non-ASCII codepoints use ``\\u`` escapes so the source is unambiguous byte-for-byte. +""" + +import random + +import pytest +from pydantic import TypeAdapter + +from mpcontribs_api.domains._shared.types import SearchStr +from mpcontribs_api.domains.project_groups.models import ProjectGroupFilter +from mpcontribs_api.domains.projects.models import ProjectFilter, ProjectIn, ProjectPatch + +_search = TypeAdapter(SearchStr) + + +# (id, raw, expected) — each row targets a distinct edge of NFKC + strip + casefold. +_CASES = [ + ("ascii_casefold", "BandGap", "bandgap"), + ("hyphen_preserved", "Band-Gap", "band-gap"), # SearchStr does not strip punctuation + ("eszett_grows_length", "Straße", "strasse"), # U+00DF casefolds to "ss" (grows length) + ("greek_final_sigma", "ΟΔΟΣ", "οδοσ"), # Σ -> σ (not ς) + ("ligature_fi", "file", "file"), # NFKC decomposes the fi ligature + ("micro_sign_to_mu", "µ", "μ"), # MICRO SIGN -> GREEK SMALL LETTER MU + ("kelvin_sign", "K", "k"), # KELVIN SIGN -> latin k + ("fullwidth_to_ascii", "ＡＢ", "ab"), # fullwidth A B -> ascii + ("superscript_digit", "m²", "m2"), # m² -> m2 + ("combining_composes", "é", "é"), # e + COMBINING ACUTE -> precomposed é + ("roman_numeral", "Ⅷ", "viii"), # ROMAN NUMERAL EIGHT -> viii + ("nbsp_trimmed", " tag ", "tag"), # NBSP folds to space, then strip + ("mixed_whitespace_trimmed", " Tag\n", "tag"), + ("turkish_dotted_I", "İ", "i̇"), # İ casefolds to i + COMBINING DOT ABOVE + ("empty_after_strip", " ", ""), # no min length: whitespace-only -> "" +] + + +@pytest.mark.parametrize("raw,expected", [(r, e) for _, r, e in _CASES], ids=[i for i, _, _ in _CASES]) +def test_searchstr_normalization(raw, expected): + out = _search.validate_python(raw) + assert out == expected + # every realistic tag must be a stable key: re-folding it changes nothing + assert _search.validate_python(out) == out + + +def test_searchstr_fuzz_output_is_stripped_and_casefolded(): + """Whatever the input's unicode form, the output is always trimmed and fully casefolded. + + These are the invariants that hold universally. Idempotency does *not* hold for every input -- + see ``test_searchstr_casefold_expansion_breaks_idempotency`` -- so it is asserted only for the + realistic cases above, not fuzzed here. + + Seeded so the run is deterministic. Draws from ranges that break naive normalizers (latin-1 + supplement, combining marks, greek, fullwidth, ligatures, roman numerals, and whitespace). + """ + rng = random.Random(1729) + pool = ( + [chr(c) for c in range(0x20, 0x7F)] # ascii printable + + [chr(c) for c in range(0xA0, 0x100)] # latin-1 supplement (µ, ß, é, NBSP, ...) + + [chr(c) for c in range(0x300, 0x370)] # combining marks + + [chr(c) for c in range(0x391, 0x3CA)] # greek letters + + [chr(c) for c in range(0xFF01, 0xFF5F)] # fullwidth forms + + [chr(c) for c in range(0xFB00, 0xFB07)] # latin ligatures + + [chr(c) for c in range(0x2160, 0x2180)] # roman numerals + + ["\t", "\n", "\r", "\x20", " ", " "] # tab/nl/cr/space/NBSP/em-space + ) + for _ in range(2000): + raw = "".join(rng.choice(pool) for _ in range(rng.randint(0, 8))) + out = _search.validate_python(raw) + assert out == out.strip(), f"leaked surrounding whitespace for {raw!r}" + assert out == out.casefold(), f"not casefold-stable for {raw!r}" + + +# ligature + uppercase + trailing NBSP -> "file": exercises fold, casefold, and trim at once. +_MESSY_TAG = " fiLE " + +_TAG_FIELD_EXTRACTORS = [ + ( + "project_in_tags", + lambda t: ProjectIn( + id="proj-x", + title="title-x", + authors="a", + description="d", + owner="google:a@b.com", + unique_identifiers=True, + tags=[t], + ).tags, + ), + ("project_patch_tags", lambda t: ProjectPatch(tags=[t]).tags), + ("filter_tags", lambda t: ProjectFilter(tags=[t]).tags), + ("filter_tags__in", lambda t: ProjectFilter(tags__in=[t]).tags__in), + ("filter_tags__contains", lambda t: ProjectFilter(tags__contains=[t]).tags__contains), + # "etc.": the same SearchStr normalizer, reused on a non-tag list field on another model. + ("project_group_filter_name__in", lambda t: ProjectGroupFilter(name__in=[t]).name__in), +] + + +@pytest.mark.parametrize( + "extract", [e for _, e in _TAG_FIELD_EXTRACTORS], ids=[i for i, _ in _TAG_FIELD_EXTRACTORS] +) +def test_searchstr_normalized_across_models(extract): + assert extract(_MESSY_TAG) == ["file"] + + +@pytest.mark.xfail( + strict=True, + reason="_nfkc_casefold is not idempotent when casefold expands a char sitting before a combining mark", +) +def test_searchstr_casefold_expansion_breaks_idempotency(): + """Documents a real edge: a casefold-expanding char (ß -> ss) followed by a combining mark. + + NFKC runs before casefold, so ``ß`` + combining circumflex stays decomposed through the first + fold (-> ``ss`` + circumflex). Re-folding then NFKC-composes ``s`` + circumflex into ``ŝ``, so + the value is not stable under a second pass. Because ``ProjectOut.tags`` is also + ``list[SearchStr]``, a stored tag re-normalizes on read and can round-trip to a different + string. xfail(strict) so this flips to a failure the moment the normalizer is made idempotent + (e.g. a trailing NFKC pass after casefold). + """ + once = _search.validate_python("ß̂") # eszett + COMBINING CIRCUMFLEX ACCENT + assert _search.validate_python(once) == once From fbe2a5e78f3557ae0720f3566f60a0b690c3fcc7 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Fri, 17 Jul 2026 10:44:06 -0700 Subject: [PATCH 41/62] Fixed bug in role parsing resulting in improper roles --- mpcontribs-api/src/mpcontribs_api/authz.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/authz.py b/mpcontribs-api/src/mpcontribs_api/authz.py index 99d6713ace..3dfc72b0b9 100644 --- a/mpcontribs-api/src/mpcontribs_api/authz.py +++ b/mpcontribs-api/src/mpcontribs_api/authz.py @@ -39,6 +39,10 @@ # prefix for project-group roles: a group's _id (an ObjectId hex string) is granted as ``project-group:`` PROJECT_GROUP_ROLE_PREFIX = "project-group:" +# A role carrying one of these prefixes is scoped to a non-project resource; a role with none of +# them is a bare project id. +_RESOURCE_ROLE_PREFIXES = (INITIATIVE_ROLE_PREFIX, PROJECT_GROUP_ROLE_PREFIX) + class User(BaseModel): """User definition derived from request headers. @@ -72,9 +76,12 @@ def is_admin(self) -> bool: @property def project_roles(self) -> list[str]: - return [ - role[len(INITIATIVE_ROLE_PREFIX) :] for role in self.groups if not role.startswith(INITIATIVE_ROLE_PREFIX) - ] + """The project ids this user carries, from their bare (unprefixed) roles. + + Resource-scoped roles (``initiative:``, ``project-group:``) and the admin sentinel are + excluded, leaving only bare project ids. + """ + return [role for role in self.groups if role != ADMIN_GROUP and not role.startswith(_RESOURCE_ROLE_PREFIXES)] @property def initiative_roles(self) -> list[str]: @@ -96,7 +103,7 @@ def has_role(self, role: str, *, resource: str | None = None) -> bool: Specifying resource as: - ``INITIATIVE_ROLE_PREFIX`` looks for roles scoped to initiatives - - "project" looks for roles scoped to projects (no actual prefix implementation yet) + - "project" looks for bare (unprefixed) project roles - None looks for roles by matching the entire string """ if resource == INITIATIVE_ROLE_PREFIX[:-1]: @@ -110,8 +117,9 @@ def writable_projects(self) -> frozenset[str]: """Projects this user may write to. Admins are unbounded (handled by can_write)""" if self.is_anonymous: return frozenset() - # exclude the admin sentinel so it never leaks into a $in / membership test - return frozenset(g for g in self.groups if g != ADMIN_GROUP) + # only bare project roles are writable projects; the admin sentinel and resource-scoped + # roles (initiative:/project-group:) must never leak into a $in / membership test + return frozenset(self.project_roles) def can_manage(self, id: str, resource: str) -> bool: """Determines whether a user can manage a resource. From 47eec4f2f07411a2a9394b25c0a2140f75bbbcc9 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Fri, 17 Jul 2026 10:45:40 -0700 Subject: [PATCH 42/62] Removed requirements/ --- mpcontribs-api/requirements/deployment.txt | 551 --------------------- 1 file changed, 551 deletions(-) delete mode 100644 mpcontribs-api/requirements/deployment.txt diff --git a/mpcontribs-api/requirements/deployment.txt b/mpcontribs-api/requirements/deployment.txt deleted file mode 100644 index 7056ece33a..0000000000 --- a/mpcontribs-api/requirements/deployment.txt +++ /dev/null @@ -1,551 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --output-file=MPContribs/mpcontribs-api/requirements/deployment.txt MPContribs/mpcontribs-api/pyproject.toml python/requirements.txt -# -anyio==4.13.0 - # via jupyter-server -apispec==5.2.2 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -argon2-cffi==25.1.0 - # via - # jupyter-server - # notebook -argon2-cffi-bindings==25.1.0 - # via argon2-cffi -arrow==1.4.0 - # via isoduration -asn1crypto==1.5.1 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -asttokens==3.0.1 - # via stack-data -atlasq-tschaume==0.11.1.dev2 - # via flask-mongorest-mpcontribs -attrs==26.1.0 - # via - # jsonschema - # referencing -backports-zstd==1.5.0 - # via flask-compress -beautifulsoup4==4.14.3 - # via nbconvert -bibtexparser==1.4.4 - # via pymatgen-core -bleach[css]==6.3.0 - # via nbconvert -blinker==1.9.0 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -boltons==25.0.0 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -boto3==1.43.19 - # via flask-mongorest-mpcontribs -botocore==1.43.19 - # via - # boto3 - # s3transfer -brotli==1.2.0 - # via flask-compress -bytecode==0.17.0 - # via ddtrace -certifi==2026.5.20 - # via requests -cffi==2.0.0 - # via - # argon2-cffi-bindings - # cryptography -charset-normalizer==3.4.7 - # via requests -click==8.4.1 - # via - # flask - # rq -comm==0.2.3 - # via ipykernel -contourpy==1.3.3 - # via matplotlib -cramjam==2.11.0 - # via python-snappy -crontab==1.0.5 - # via rq-scheduler -cryptography==48.0.0 - # via pyopenssl -css-html-js-minify==2.5.5 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -cycler==0.12.1 - # via matplotlib -dateparser==1.4.0 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -ddtrace==4.3.0 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -debugpy==1.8.21 - # via ipykernel -decorator==5.3.1 - # via ipython -defusedxml==0.7.1 - # via nbconvert -dnspython==2.8.0 - # via - # mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) - # pymongo -entrypoints==0.4 - # via jupyter-client -envier==0.6.1 - # via ddtrace -executing==2.2.1 - # via stack-data -fastjsonschema==2.21.2 - # via nbformat -fastnumbers==5.1.1 - # via flask-mongorest-mpcontribs -filetype==1.2.0 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -flasgger-tschaume==0.9.7 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -flask==2.2.5 - # via - # flasgger-tschaume - # flask-compress - # flask-marshmallow - # flask-mongoengine-tschaume - # flask-rq2 - # flask-sse -flask-compress==1.24 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -flask-marshmallow==1.4.0 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -flask-mongoengine-tschaume==1.1.0 - # via flask-mongorest-mpcontribs -flask-mongorest-mpcontribs==3.3.0 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -flask-rq2==18.3 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -flask-sse==1.0.0 - # via flask-mongorest-mpcontribs -flatten-dict==0.5.0 - # via flask-mongorest-mpcontribs -flexcache==0.3 - # via pint -flexparser==0.4 - # via pint -fonttools==4.63.0 - # via matplotlib -fqdn==1.5.1 - # via jsonschema -freezegun==1.5.5 - # via rq-scheduler -gevent==26.5.0 - # via gunicorn -greenlet==3.5.1 - # via gevent -gunicorn[gevent]==24.1.1 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -idna==3.17 - # via - # anyio - # jsonschema - # requests -ipykernel==6.29.5 - # via - # nbclassic - # notebook -ipython==9.14.0 - # via ipykernel -ipython-genutils==0.2.0 - # via - # nbclassic - # notebook -ipython-pygments-lexers==1.1.1 - # via ipython -isoduration==20.11.0 - # via jsonschema -itsdangerous==2.2.0 - # via flask -jedi==0.20.0 - # via ipython -jinja2==3.1.6 - # via - # flask - # jupyter-server - # mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) - # nbconvert - # notebook -jmespath==1.1.0 - # via - # boto3 - # botocore -joblib==1.5.3 - # via pymatgen-core -json2html==1.3.0 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -jsonpointer==3.1.1 - # via jsonschema -jsonschema[format-nongpl]==4.26.0 - # via - # flasgger-tschaume - # jupyter-events - # nbformat -jsonschema-specifications==2025.9.1 - # via jsonschema -jupyter-client==7.4.9 - # via - # ipykernel - # jupyter-server - # nbclient - # notebook -jupyter-core==5.9.1 - # via - # ipykernel - # jupyter-client - # jupyter-server - # nbclient - # nbconvert - # nbformat - # notebook -jupyter-events==0.12.1 - # via jupyter-server -jupyter-server==2.19.0 - # via notebook-shim -jupyter-server-terminals==0.5.4 - # via jupyter-server -jupyterlab-pygments==0.3.0 - # via nbconvert -kiwisolver==1.5.0 - # via matplotlib -lark==1.3.1 - # via rfc3987-syntax -lxml==6.1.1 - # via pymatgen-core -markupsafe==3.0.3 - # via - # jinja2 - # nbconvert - # werkzeug -marshmallow==3.26.2 - # via - # flask-marshmallow - # marshmallow-mongoengine - # mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -marshmallow-mongoengine==0.31.2 - # via flask-mongorest-mpcontribs -matplotlib==3.10.9 - # via - # -r python/requirements.txt - # pymatgen-core -matplotlib-inline==0.2.2 - # via - # ipykernel - # ipython -mimerender-pr36==0.0.2 - # via flask-mongorest-mpcontribs -mistune==3.2.1 - # via - # flasgger-tschaume - # nbconvert -mongoengine==0.29.3 - # via - # atlasq-tschaume - # flask-mongoengine-tschaume - # marshmallow-mongoengine -monty==2026.5.18 - # via pymatgen-core -more-itertools==11.1.0 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -mpmath==1.3.0 - # via sympy -narwhals==2.22.0 - # via plotly -nbclassic==1.3.3 - # via notebook -nbclient==0.10.4 - # via nbconvert -nbconvert==7.17.1 - # via - # jupyter-server - # notebook -nbformat==5.10.4 - # via - # jupyter-server - # mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) - # nbclient - # nbconvert - # notebook -nest-asyncio==1.6.0 - # via - # ipykernel - # jupyter-client - # nbclassic - # notebook -networkx==3.6.1 - # via pymatgen-core -notebook==6.5.7 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -notebook-shim==0.2.4 - # via nbclassic -numpy==2.4.6 - # via - # -r python/requirements.txt - # contourpy - # matplotlib - # monty - # mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) - # pandas - # pymatgen-core - # scipy - # spglib -opentelemetry-api==1.42.1 - # via ddtrace -orjson==3.11.9 - # via - # flask-mongorest-mpcontribs - # pymatgen-core -overrides==7.7.0 - # via jupyter-server -packaging==26.2 - # via - # gunicorn - # ipykernel - # jupyter-events - # jupyter-server - # marshmallow - # matplotlib - # nbconvert - # plotly -palettable==3.3.3 - # via pymatgen-core -pandas==3.0.3 - # via - # -r python/requirements.txt - # pymatgen-core -pandocfilters==1.5.1 - # via nbconvert -parso==0.8.7 - # via jedi -pexpect==4.9.0 - # via ipython -pillow==12.2.0 - # via matplotlib -pint==0.25.3 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -platformdirs==4.10.0 - # via - # jupyter-core - # pint -plotly==6.7.0 - # via pymatgen-core -prometheus-client==0.25.0 - # via - # jupyter-server - # notebook -prompt-toolkit==3.0.52 - # via ipython -psutil==7.2.2 - # via - # ipykernel - # ipython -psycopg2-binary==2.9.12 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -ptyprocess==0.7.0 - # via - # pexpect - # terminado -pure-eval==0.2.3 - # via stack-data -pycparser==3.0 - # via cffi -pygments==2.20.0 - # via - # ipython - # ipython-pygments-lexers - # nbconvert -pymatgen==2026.5.4 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -pymatgen-core==2026.5.18 - # via pymatgen -pymongo==4.17.0 - # via - # flask-mongorest-mpcontribs - # mongoengine -pyopenssl==26.2.0 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -pyparsing==3.3.2 - # via - # bibtexparser - # matplotlib -python-dateutil==2.9.0.post0 - # via - # arrow - # botocore - # dateparser - # flask-mongorest-mpcontribs - # freezegun - # jupyter-client - # matplotlib - # pandas - # rq-scheduler -python-json-logger==4.1.0 - # via jupyter-events -python-mimeparse==2.0.0 - # via mimerender-pr36 -python-snappy==0.7.3 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -pytz==2026.2 - # via dateparser -pyyaml==6.0.3 - # via - # flasgger-tschaume - # jupyter-events -pyzmq==27.1.0 - # via - # ipykernel - # jupyter-client - # jupyter-server - # notebook -redis==8.0.0 - # via - # flask-rq2 - # flask-sse - # rq -referencing==0.37.0 - # via - # jsonschema - # jsonschema-specifications - # jupyter-events -regex==2026.5.9 - # via dateparser -requests==2.34.2 - # via - # atlasq-tschaume - # pymatgen-core -rfc3339-validator==0.1.4 - # via - # jsonschema - # jupyter-events -rfc3986-validator==0.1.1 - # via - # jsonschema - # jupyter-events -rfc3987-syntax==1.1.0 - # via jsonschema -rpds-py==2026.5.1 - # via - # jsonschema - # referencing -rq==2.3.2 - # via - # flask-rq2 - # mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) - # rq-scheduler -rq-scheduler==0.14.0 - # via flask-rq2 -ruamel-yaml==0.19.1 - # via monty -s3transfer==0.18.0 - # via boto3 -scipy==1.17.1 - # via - # -r python/requirements.txt - # pymatgen-core -send2trash==2.1.0 - # via - # jupyter-server - # notebook -setproctitle==1.3.7 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -six==1.17.0 - # via - # flasgger-tschaume - # flask-sse - # python-dateutil - # rfc3339-validator -soupsieve==2.8.4 - # via beautifulsoup4 -spglib==2.7.0 - # via pymatgen-core -stack-data==0.6.3 - # via ipython -supervisor==4.3.0 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -sympy==1.14.0 - # via pymatgen-core -tabulate==0.10.0 - # via pymatgen-core -terminado==0.18.1 - # via - # jupyter-server - # jupyter-server-terminals - # notebook -tinycss2==1.4.0 - # via bleach -tornado==6.5.6 - # via - # ipykernel - # jupyter-client - # jupyter-server - # notebook - # terminado -tqdm==4.67.3 - # via pymatgen-core -traitlets==5.15.0 - # via - # ipykernel - # ipython - # jupyter-client - # jupyter-core - # jupyter-events - # jupyter-server - # matplotlib-inline - # nbclient - # nbconvert - # nbformat - # notebook -typing-extensions==4.15.0 - # via - # anyio - # beautifulsoup4 - # flexcache - # flexparser - # ipython - # opentelemetry-api - # pint - # pyopenssl - # referencing - # spglib -tzdata==2026.2 - # via arrow -tzlocal==5.3.1 - # via dateparser -uncertainties==3.2.3 - # via - # mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) - # pymatgen-core -uri-template==1.3.0 - # via jsonschema -urllib3==2.7.0 - # via - # botocore - # requests -wcwidth==0.7.0 - # via prompt-toolkit -webcolors==25.10.0 - # via jsonschema -webencodings==0.5.1 - # via - # bleach - # tinycss2 -websocket-client==1.9.0 - # via - # jupyter-server - # mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) -werkzeug==3.1.8 - # via - # flasgger-tschaume - # flask -wrapt==2.2.1 - # via ddtrace -zope-event==6.2 - # via gevent -zope-interface==8.5 - # via gevent -zstandard==0.25.0 - # via mpcontribs-api (MPContribs/mpcontribs-api/pyproject.toml) From e6a648bc9ca1d5550a0deb210415be2eefcb717e Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Fri, 17 Jul 2026 10:52:18 -0700 Subject: [PATCH 43/62] Undid non-mpcontribs-api changes --- mpcontribs-ingester/mpcontribs/ingester/cli.py | 4 +--- mpcontribs-ingester/mpcontribs/ingester/webui.py | 11 ++--------- mpcontribs-kernel-gateway/make_seed.py | 1 + mpcontribs-lux/mpcontribs/lux/autogen.py | 2 +- .../tests/projects/esoteric_ephemera/test_schemas.py | 1 + mpcontribs-portal/mpcontribs/portal/views.py | 2 ++ .../mpcontribs/users/als_beamline/scripts/__main__.py | 3 +-- .../users/als_beamline/scripts/translate_PyPt.py | 3 +++ .../users/dilute_solute_diffusion/pre_submission.py | 6 ++---- .../mpcontribs/users/qmcdb/main/views.py | 5 ++++- .../mpcontribs/users/qmcdb/records/views.py | 7 ++++++- .../users/redox_thermo_csp/pre_submission.py | 9 ++++----- .../users/redox_thermo_csp/update_energy_data.py | 3 ++- .../users/screening_inorganic_pv/pre_submission.py | 7 +++---- .../mpcontribs/users/swf/pre_submission.py | 2 ++ mpcontribs-portal/mpcontribs/users/utils.py | 3 +-- .../contribs.materialsproject.org/2dmatpedia.ipynb | 5 ++--- .../contribs.materialsproject.org/ExpXAS.ipynb | 3 ++- .../contribs.materialsproject.org/HFP2023.ipynb | 3 +++ .../MnO2_phase_selection.ipynb | 3 +-- .../carrier_transport.ipynb | 6 +++--- .../dilute_solute_diffusion.ipynb | 6 ++---- .../experimental_thermo.ipynb | 2 ++ .../experimental_thermoelectrics.ipynb | 2 +- .../ferroelectrics.ipynb | 7 ++++--- .../contribs.materialsproject.org/ion_ref_data.ipynb | 2 +- .../contribs.materialsproject.org/jarvis_dft.ipynb | 4 +--- .../contribs.materialsproject.org/matscholar.ipynb | 3 ++- .../contribs.materialsproject.org/mofexplorer.ipynb | 2 +- .../ocp/ocp-upload.ipynb | 1 + .../open_catalyst_project.ipynb | 4 +++- .../perovskites_diffusion.ipynb | 4 ++-- .../pydatarecognition.ipynb | 4 +++- .../screening_inorganic_pv.ipynb | 2 +- .../silicon_defects.ipynb | 4 ++-- .../springer_materials.ipynb | 2 +- .../transparent_conductors.ipynb | 1 + .../get_started.ipynb | 5 ++++- .../ml.materialsproject.org/get_started.ipynb | 4 +--- mpcontribs-portal/wsgi.py | 1 + mpcontribs-serverless/make_download/app.py | 1 + 41 files changed, 82 insertions(+), 68 deletions(-) diff --git a/mpcontribs-ingester/mpcontribs/ingester/cli.py b/mpcontribs-ingester/mpcontribs/ingester/cli.py index 53267ef457..53965cc507 100644 --- a/mpcontribs-ingester/mpcontribs/ingester/cli.py +++ b/mpcontribs-ingester/mpcontribs/ingester/cli.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # http://flask.pocoo.org/docs/0.10/patterns/appdispatch/ -import os -import argparse -import string +import os, argparse, string from werkzeug.serving import run_simple from werkzeug.wsgi import DispatcherMiddleware, SharedDataMiddleware from flask import Flask diff --git a/mpcontribs-ingester/mpcontribs/ingester/webui.py b/mpcontribs-ingester/mpcontribs/ingester/webui.py index c0b4debe56..5d4f711563 100644 --- a/mpcontribs-ingester/mpcontribs/ingester/webui.py +++ b/mpcontribs-ingester/mpcontribs/ingester/webui.py @@ -1,14 +1,7 @@ from __future__ import unicode_literals, print_function, absolute_import -import json -import os -import socket -import codecs -import time -import psutil -import sys -import warnings -import multiprocessing +import json, os, socket, codecs, time, psutil +import sys, warnings, multiprocessing from tempfile import gettempdir from flask import render_template, request, Response, Blueprint, current_app from flask import url_for, redirect, make_response, stream_with_context, jsonify diff --git a/mpcontribs-kernel-gateway/make_seed.py b/mpcontribs-kernel-gateway/make_seed.py index 28aa21e0d1..f92a5df7c9 100644 --- a/mpcontribs-kernel-gateway/make_seed.py +++ b/mpcontribs-kernel-gateway/make_seed.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import ddtrace.auto import nbformat as nbf nb = nbf.v4.new_notebook() diff --git a/mpcontribs-lux/mpcontribs/lux/autogen.py b/mpcontribs-lux/mpcontribs/lux/autogen.py index 95d6255663..aeee406c21 100644 --- a/mpcontribs-lux/mpcontribs/lux/autogen.py +++ b/mpcontribs-lux/mpcontribs/lux/autogen.py @@ -100,7 +100,7 @@ def pydantic_model(self) -> Type[BaseModel]: self.file_name, orient=orient, lines=self.fmt == "jsonl" ) break - except Exception: + except Exception as exc: continue else: raise ValueError( diff --git a/mpcontribs-lux/tests/projects/esoteric_ephemera/test_schemas.py b/mpcontribs-lux/tests/projects/esoteric_ephemera/test_schemas.py index 0d74c5213c..70b6ec8019 100644 --- a/mpcontribs-lux/tests/projects/esoteric_ephemera/test_schemas.py +++ b/mpcontribs-lux/tests/projects/esoteric_ephemera/test_schemas.py @@ -2,6 +2,7 @@ import gzip import json +from pathlib import Path import numpy as np import pytest diff --git a/mpcontribs-portal/mpcontribs/portal/views.py b/mpcontribs-portal/mpcontribs/portal/views.py index ad2c12c1e0..566a6b6542 100644 --- a/mpcontribs-portal/mpcontribs/portal/views.py +++ b/mpcontribs-portal/mpcontribs/portal/views.py @@ -12,6 +12,8 @@ from redis import Redis from io import BytesIO from copy import deepcopy +from pathlib import Path +from shutil import make_archive, rmtree from nbconvert import HTMLExporter from bravado.exception import HTTPNotFound from json2html import Json2Html diff --git a/mpcontribs-portal/mpcontribs/users/als_beamline/scripts/__main__.py b/mpcontribs-portal/mpcontribs/users/als_beamline/scripts/__main__.py index 4fc72ada4a..bcd0122bda 100644 --- a/mpcontribs-portal/mpcontribs/users/als_beamline/scripts/__main__.py +++ b/mpcontribs-portal/mpcontribs/users/als_beamline/scripts/__main__.py @@ -1,5 +1,4 @@ -import argparse -import os +import argparse, os from mpcontribs.io.archieml.mpfile import MPFile from pre_submission import * diff --git a/mpcontribs-portal/mpcontribs/users/als_beamline/scripts/translate_PyPt.py b/mpcontribs-portal/mpcontribs/users/als_beamline/scripts/translate_PyPt.py index 232914ab86..995f5d045d 100644 --- a/mpcontribs-portal/mpcontribs/users/als_beamline/scripts/translate_PyPt.py +++ b/mpcontribs-portal/mpcontribs/users/als_beamline/scripts/translate_PyPt.py @@ -1,3 +1,6 @@ +import pandas as pd +import os +from scipy.interpolate import interp2d def get_translate(workdir=None): diff --git a/mpcontribs-portal/mpcontribs/users/dilute_solute_diffusion/pre_submission.py b/mpcontribs-portal/mpcontribs/users/dilute_solute_diffusion/pre_submission.py index de41e67fa2..045f5bd71f 100644 --- a/mpcontribs-portal/mpcontribs/users/dilute_solute_diffusion/pre_submission.py +++ b/mpcontribs-portal/mpcontribs/users/dilute_solute_diffusion/pre_submission.py @@ -1,6 +1,4 @@ -import os -import json -import requests +import os, json, requests, sys from pandas import read_excel, isnull, ExcelWriter, Series from mpcontribs.io.core.recdict import RecursiveDict from mpcontribs.io.core.utils import clean_value, nest_dict @@ -62,7 +60,7 @@ def run(mpfile, hosts=None, download=False): if hosts is not None: if isinstance(hosts, int) and idx + 1 > hosts: break - elif isinstance(hosts, list) and host not in hosts: + elif isinstance(hosts, list) and not host in hosts: continue print("get mp-id for {}".format(host)) diff --git a/mpcontribs-portal/mpcontribs/users/qmcdb/main/views.py b/mpcontribs-portal/mpcontribs/users/qmcdb/main/views.py index cd80073db9..4e4c20ac48 100644 --- a/mpcontribs-portal/mpcontribs/users/qmcdb/main/views.py +++ b/mpcontribs-portal/mpcontribs/users/qmcdb/main/views.py @@ -1,5 +1,8 @@ from django.shortcuts import render -from records.forms import MaterialQueryForm +from django.http import HttpResponseRedirect +from django.contrib.auth.decorators import login_required +from records.forms import MaterialQueryForm, MaterialSubmissionForm +from records.tables import QMCDBSetTable from records.models import QMCDBSet from django.utils.safestring import mark_safe from django.utils.html import escape diff --git a/mpcontribs-portal/mpcontribs/users/qmcdb/records/views.py b/mpcontribs-portal/mpcontribs/users/qmcdb/records/views.py index 589ef8e3e7..c0f0cc1372 100644 --- a/mpcontribs-portal/mpcontribs/users/qmcdb/records/views.py +++ b/mpcontribs-portal/mpcontribs/users/qmcdb/records/views.py @@ -1,13 +1,18 @@ from __future__ import division from django.shortcuts import render from django.http import HttpResponseRedirect, HttpResponse +from django.contrib.auth.decorators import login_required from rest_framework import status from rest_framework.decorators import api_view from rest_framework.response import Response -from records.forms import MaterialSubmissionForm +from records.forms import MaterialQueryForm, MaterialSubmissionForm from records.models import QMCDBSet from records.serializers import QMCDBSetSerializer +from rest_framework.renderers import JSONRenderer +from rest_framework.parsers import JSONParser +from django.utils.six import BytesIO from django.utils.safestring import mark_safe +import numpy as np def manual_qmc_record_submission(request): diff --git a/mpcontribs-portal/mpcontribs/users/redox_thermo_csp/pre_submission.py b/mpcontribs-portal/mpcontribs/users/redox_thermo_csp/pre_submission.py index e95fd1f18c..15cfaf4de7 100644 --- a/mpcontribs-portal/mpcontribs/users/redox_thermo_csp/pre_submission.py +++ b/mpcontribs-portal/mpcontribs/users/redox_thermo_csp/pre_submission.py @@ -1,17 +1,16 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -import os -import json -import re -import sys +import os, json, re, sys from glob import glob from datetime import datetime from itertools import groupby import pandas as pd +from mpcontribs.io.core.utils import get_composition_from_string from mpcontribs.io.core.recdict import RecursiveDict -from mpcontribs.io.core.utils import clean_value, read_csv +from mpcontribs.io.core.utils import clean_value, read_csv, nest_dict from mpcontribs.io.core.components import Table from mpcontribs.users.utils import duplicate_check +from mpcontribs.users.redox_thermo_csp.utils import redenth_act, get_debye_temp def get_fit_pars(sample_number): diff --git a/mpcontribs-portal/mpcontribs/users/redox_thermo_csp/update_energy_data.py b/mpcontribs-portal/mpcontribs/users/redox_thermo_csp/update_energy_data.py index 3701993937..b2eed60d99 100644 --- a/mpcontribs-portal/mpcontribs/users/redox_thermo_csp/update_energy_data.py +++ b/mpcontribs-portal/mpcontribs/users/redox_thermo_csp/update_energy_data.py @@ -2,6 +2,7 @@ import datetime import os import shutil +import numpy as np from energy_analysis import EnergyAnalysis as enera from views import unstable_phases as unst @@ -20,7 +21,7 @@ new_energy_data = old_energy_data for db_id in paramlist: - if "Exp" not in db_id: + if not "Exp" in db_id: print(db_id) data_source = "Theo" # updates only theoretical data celsius = "True" # always True, parameter input in K currently disabled diff --git a/mpcontribs-portal/mpcontribs/users/screening_inorganic_pv/pre_submission.py b/mpcontribs-portal/mpcontribs/users/screening_inorganic_pv/pre_submission.py index aef7b1c136..454a5597e3 100644 --- a/mpcontribs-portal/mpcontribs/users/screening_inorganic_pv/pre_submission.py +++ b/mpcontribs-portal/mpcontribs/users/screening_inorganic_pv/pre_submission.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -import os -import json +import os, json from pandas import DataFrame from mpcontribs.io.core.recdict import RecursiveDict from mpcontribs.io.core.utils import clean_value @@ -47,11 +46,11 @@ def run(mpfile, **kwargs): rd = RecursiveDict({"formula": formula}) for k, v in config.items(): value = clean_value(d[k], v[1], max_dgts=4) - if "." not in v[0]: + if not "." in v[0]: rd[v[0]] = value else: keys = v[0].split(".") - if keys[0] not in rd: + if not keys[0] in rd: rd[keys[0]] = RecursiveDict({keys[1]: value}) else: rd[keys[0]][keys[1]] = value diff --git a/mpcontribs-portal/mpcontribs/users/swf/pre_submission.py b/mpcontribs-portal/mpcontribs/users/swf/pre_submission.py index 8c48375d97..6c46bc0c1a 100644 --- a/mpcontribs-portal/mpcontribs/users/swf/pre_submission.py +++ b/mpcontribs-portal/mpcontribs/users/swf/pre_submission.py @@ -1,4 +1,5 @@ from mpcontribs.config import mp_level01_titles +from mpcontribs.io.core.recdict import RecursiveDict from mpcontribs.io.core.utils import clean_value, get_composition_from_string from mpcontribs.users.utils import duplicate_check @@ -25,6 +26,7 @@ def round_to_100_percent(number_set, digit_after_decimal=1): def run(mpfile, **kwargs): import pymatgen import pandas as pd + from mpcontribs.users.swf.rest.rester import SwfRester # load data from google sheet google_sheet = mpfile.document[mp_level01_titles[0]].pop("google_sheet") diff --git a/mpcontribs-portal/mpcontribs/users/utils.py b/mpcontribs-portal/mpcontribs/users/utils.py index e49f04401a..4cf02217e9 100644 --- a/mpcontribs-portal/mpcontribs/users/utils.py +++ b/mpcontribs-portal/mpcontribs/users/utils.py @@ -1,5 +1,4 @@ -import inspect -import os +import inspect, os from typing import Any, Dict diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/2dmatpedia.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/2dmatpedia.ipynb index 73641ce9f3..daf3c45748 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/2dmatpedia.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/2dmatpedia.ipynb @@ -6,10 +6,9 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", - "import gzip\n", - "import json\n", + "import os, gzip, json\n", "from mpcontribs.client import Client\n", + "from pymatgen.core import Structure\n", "from pymatgen.ext.matproj import MPRester\n", "from urllib.request import urlretrieve\n", "from monty.json import MontyDecoder" diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/ExpXAS.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/ExpXAS.ipynb index 89ad71cfd9..d2fdf9ba09 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/ExpXAS.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/ExpXAS.ipynb @@ -10,7 +10,8 @@ "from mpcontribs.client import Client\n", "from pathlib import Path\n", "from pandas import read_csv\n", - "import pandas as pd" + "import pandas as pd\n", + "import numpy as np" ] }, { diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/HFP2023.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/HFP2023.ipynb index 61acc0ccc0..9b0f05ceb1 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/HFP2023.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/HFP2023.ipynb @@ -9,12 +9,15 @@ "source": [ "%env MPRESTER_MUTE_PROGRESS_BARS 1\n", "# pip install mpcontribs-client mp_api pandas flatten_dict\n", + "import os\n", "import gzip\n", "import json\n", "\n", "from pathlib import Path\n", "from mpcontribs.client import Client\n", + "from mp_api.client import MPRester\n", "from pymatgen.core import Structure\n", + "from pandas import read_csv\n", "from flatten_dict import flatten, unflatten" ] }, diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/MnO2_phase_selection.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/MnO2_phase_selection.ipynb index 328c0f2c2f..d6c63cb805 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/MnO2_phase_selection.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/MnO2_phase_selection.ipynb @@ -6,8 +6,7 @@ "metadata": {}, "outputs": [], "source": [ - "import json\n", - "import os\n", + "import json, os\n", "from mpcontribs.client import Client\n", "from pymatgen.core import Composition, Structure\n", "from pymatgen.ext.matproj import MPRester\n", diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/carrier_transport.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/carrier_transport.ipynb index e574ababd3..68e1c4f64d 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/carrier_transport.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/carrier_transport.ipynb @@ -7,13 +7,13 @@ "outputs": [], "source": [ "from mpcontribs.client import Client\n", - "import gzip\n", - "import json\n", - "import os\n", + "import gzip, json, os\n", "import numpy as np\n", "from pandas import DataFrame\n", + "from collections import defaultdict\n", "from tqdm.notebook import tqdm\n", "from unflatten import unflatten\n", + "from pathlib import Path\n", "\n", "name = 'carrier_transport'" ] diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/dilute_solute_diffusion.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/dilute_solute_diffusion.ipynb index a26d81d3bb..3ce892c236 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/dilute_solute_diffusion.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/dilute_solute_diffusion.ipynb @@ -28,9 +28,7 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", - "import json\n", - "import requests\n", + "import os, json, requests, sys\n", "from pandas import read_excel, isnull, ExcelWriter, Series\n", "from mp_api.client import MPRester\n", "from pathlib import Path\n", @@ -110,7 +108,7 @@ " if hosts is not None:\n", " if isinstance(hosts, int) and idx + 1 > hosts:\n", " break\n", - " elif isinstance(hosts, list) and host not in hosts:\n", + " elif isinstance(hosts, list) and not host in hosts:\n", " continue\n", "\n", " print(\"get mp-id for {}\".format(host))\n", diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/experimental_thermo.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/experimental_thermo.ipynb index 4ddf73bb08..a02e07d84a 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/experimental_thermo.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/experimental_thermo.ipynb @@ -48,6 +48,8 @@ "from pathlib import Path\n", "import re\n", "from tqdm import tqdm\n", + "import numpy as np\n", + "import xlrd\n", "from monty.serialization import loadfn, dumpfn" ] }, diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/experimental_thermoelectrics.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/experimental_thermoelectrics.ipynb index b3164a38e3..fc639fd76a 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/experimental_thermoelectrics.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/experimental_thermoelectrics.ipynb @@ -11,7 +11,7 @@ "from mp_api.client import MPRester\n", "import pandas as pd\n", "import os\n", - "from flatten_dict import unflatten\n", + "from flatten_dict import unflatten, flatten\n", "from math import isnan" ] }, diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/ferroelectrics.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/ferroelectrics.ipynb index 5a2a1a8876..458b629c00 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/ferroelectrics.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/ferroelectrics.ipynb @@ -11,9 +11,10 @@ "source": [ "import json\n", "import numpy as np\n", - "from mpcontribs.client import Client\n", + "from mpcontribs.client import Client, Attachment\n", "from pathlib import Path\n", - "from flatten_dict import flatten, unflatten" + "from flatten_dict import flatten, unflatten\n", + "from pymatgen.core import Structure" ] }, { @@ -223,7 +224,7 @@ " if conf and k.startswith('polarization') and isinstance(v, list):\n", " name, fields = conf[\"name\"], conf[\"fields\"]\n", " contrib[\"data\"].setdefault(name, {})\n", - " if \"unit\" not in conf:\n", + " if not \"unit\" in conf:\n", " vmax, unit = max(v), fields[\"max\"]\n", " contrib[\"data\"][name]['max'] = f\"{round(vmax, 3)} {unit}\" if unit else v\n", " contrib[\"data\"][name]['index'] = v.index(vmax)\n", diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/ion_ref_data.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/ion_ref_data.ipynb index 318359a7c4..4543e68cdb 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/ion_ref_data.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/ion_ref_data.ipynb @@ -29,7 +29,7 @@ "outputs": [], "source": [ "from pprint import pprint\n", - "from monty.serialization import loadfn\n", + "from monty.serialization import loadfn, dumpfn\n", "from pymatgen.core.ion import Ion" ] }, diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/jarvis_dft.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/jarvis_dft.ipynb index 7f08f74b59..bb08b09503 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/jarvis_dft.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/jarvis_dft.ipynb @@ -6,9 +6,7 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", - "import json\n", - "import tarfile\n", + "import os, json, tarfile\n", "from mpcontribs.client import Client\n", "from urllib.request import urlretrieve\n", "from monty.json import MontyDecoder\n", diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/matscholar.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/matscholar.ipynb index bc5766f6de..31e5a2c243 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/matscholar.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/matscholar.ipynb @@ -7,7 +7,8 @@ "metadata": {}, "outputs": [], "source": [ - "from mpcontribs.client import Client" + "from pathlib import Path\n", + "from mpcontribs.client import Client, Attachment" ] }, { diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/mofexplorer.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/mofexplorer.ipynb index c399fc6f96..a9dc872a85 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/mofexplorer.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/mofexplorer.ipynb @@ -76,7 +76,7 @@ " raw = vs[-1].replace(\"^3\", \"³\")\n", " if raw in ureg:\n", " value, unit = vs[0], raw\n", - " except Exception:\n", + " except Exception as e:\n", " value, unit = v, None\n", " else:\n", " value, unit = vs[0], None\n", diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/ocp/ocp-upload.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/ocp/ocp-upload.ipynb index 15a7de18c9..7f82031902 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/ocp/ocp-upload.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/ocp/ocp-upload.ipynb @@ -12,6 +12,7 @@ "from ujson import load\n", "from pymatgen.core.structure import Molecule, Structure\n", "from pathlib import Path\n", + "from time import time\n", "from mpcontribs.client import Client\n", "from tqdm.auto import tqdm" ] diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/open_catalyst_project.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/open_catalyst_project.ipynb index 934ac1ba4d..25f34580fc 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/open_catalyst_project.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/open_catalyst_project.ipynb @@ -9,8 +9,10 @@ "source": [ "from mpcontribs.client import Client\n", "from monty.serialization import loadfn\n", + "from json import loads\n", "from pymatgen.core.structure import Molecule, Structure\n", - "from pathlib import Path" + "from pathlib import Path\n", + "from time import time" ] }, { diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/perovskites_diffusion.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/perovskites_diffusion.ipynb index 634cb3c77c..026d09bec2 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/perovskites_diffusion.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/perovskites_diffusion.ipynb @@ -53,7 +53,7 @@ "metadata": {}, "outputs": [], "source": [ - "import tarfile\n", + "import tarfile, os\n", "from pandas import read_excel\n", "\n", "units = {\n", @@ -97,7 +97,7 @@ " key = keys[col]\n", " if isinstance(key, str):\n", " key = key.strip()\n", - " if key not in abbreviations:\n", + " if not key in abbreviations:\n", " abbreviations[key] = col\n", " else:\n", " key = col.strip().lower()\n", diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/pydatarecognition.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/pydatarecognition.ipynb index 23c100fe54..4f1d69e8b4 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/pydatarecognition.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/pydatarecognition.ipynb @@ -8,9 +8,11 @@ "outputs": [], "source": [ "%env MPRESTER_MUTE_PROGRESS_BARS 1\n", + "import os\n", "from pathlib import Path\n", "from mpcontribs.client import Client\n", - "from flatten_dict import unflatten\n", + "from mp_api.client import MPRester\n", + "from flatten_dict import unflatten, flatten\n", "from pymatgen.io.cif import CifParser\n", "from pandas import DataFrame\n", "import numpy as np" diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/screening_inorganic_pv.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/screening_inorganic_pv.ipynb index 270bc924d1..cb9e56014f 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/screening_inorganic_pv.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/screening_inorganic_pv.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "outputs": [], "source": [ - "import json\n", + "import os, json\n", "from pathlib import Path\n", "from pandas import DataFrame\n", "from mpcontribs.client import Client\n", diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/silicon_defects.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/silicon_defects.ipynb index 6c100a231c..b03018e553 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/silicon_defects.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/silicon_defects.ipynb @@ -11,7 +11,7 @@ "from mpcontribs.client import Client, Attachment\n", "from pymatgen.core import Structure\n", "from pathlib import Path\n", - "from flatten_dict import flatten" + "from flatten_dict import flatten, unflatten" ] }, { @@ -95,7 +95,7 @@ "}\n", "\n", "for k, v in list(reorg.items()):\n", - " if \"unit\" not in v:\n", + " if not \"unit\" in v:\n", " root_field = reorg.pop(k).get(\"field\")\n", " \n", " for kk, vv in excitation_reorg.items():\n", diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/springer_materials.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/springer_materials.ipynb index e7d0958f16..ce4e3b1dfd 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/springer_materials.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/springer_materials.ipynb @@ -11,7 +11,7 @@ "import re\n", "from glob import glob\n", "from mpcontribs.client import Client\n", - "from flatten_dict import unflatten" + "from flatten_dict import unflatten, flatten" ] }, { diff --git a/mpcontribs-portal/notebooks/contribs.materialsproject.org/transparent_conductors.ipynb b/mpcontribs-portal/notebooks/contribs.materialsproject.org/transparent_conductors.ipynb index f123264613..bc8a8feef7 100644 --- a/mpcontribs-portal/notebooks/contribs.materialsproject.org/transparent_conductors.ipynb +++ b/mpcontribs-portal/notebooks/contribs.materialsproject.org/transparent_conductors.ipynb @@ -7,6 +7,7 @@ "metadata": {}, "outputs": [], "source": [ + "import tarfile, os\n", "import numpy as np\n", "from pandas import read_excel\n", "from mpcontribs.client import Client" diff --git a/mpcontribs-portal/notebooks/lightsources.materialsproject.org/get_started.ipynb b/mpcontribs-portal/notebooks/lightsources.materialsproject.org/get_started.ipynb index 3b53d08f6d..c3162b1883 100644 --- a/mpcontribs-portal/notebooks/lightsources.materialsproject.org/get_started.ipynb +++ b/mpcontribs-portal/notebooks/lightsources.materialsproject.org/get_started.ipynb @@ -7,6 +7,8 @@ "outputs": [], "source": [ "import os\n", + "import json\n", + "import gzip\n", "from zipfile import ZipFile\n", "from io import StringIO, BytesIO\n", "from numpy import where\n", @@ -14,7 +16,8 @@ "from pandas import to_numeric, read_csv\n", "from mpcontribs.client import Client, Attachment\n", "from tqdm.notebook import tqdm\n", - "from decimal import Decimal" + "from decimal import Decimal\n", + "from pathlib import Path" ] }, { diff --git a/mpcontribs-portal/notebooks/ml.materialsproject.org/get_started.ipynb b/mpcontribs-portal/notebooks/ml.materialsproject.org/get_started.ipynb index 651a268de2..a13eefdc69 100644 --- a/mpcontribs-portal/notebooks/ml.materialsproject.org/get_started.ipynb +++ b/mpcontribs-portal/notebooks/ml.materialsproject.org/get_started.ipynb @@ -6,9 +6,7 @@ "metadata": {}, "outputs": [], "source": [ - "import wget\n", - "import json\n", - "import math\n", + "import wget, json, os, math\n", "from pathlib import Path\n", "from string import capwords\n", "from pybtex.database import parse_string\n", diff --git a/mpcontribs-portal/wsgi.py b/mpcontribs-portal/wsgi.py index 96ebc59a28..e314b76514 100644 --- a/mpcontribs-portal/wsgi.py +++ b/mpcontribs-portal/wsgi.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import re import os +import ddtrace.auto import django_settings_file from django.core.wsgi import get_wsgi_application from whitenoise import WhiteNoise diff --git a/mpcontribs-serverless/make_download/app.py b/mpcontribs-serverless/make_download/app.py index 9783183249..96f98401a7 100644 --- a/mpcontribs-serverless/make_download/app.py +++ b/mpcontribs-serverless/make_download/app.py @@ -1,5 +1,6 @@ # TODO ddtrace import os +import json import logging import boto3 From 6b24b3526301d7a264740ff37bba4e00a301cf23 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Fri, 17 Jul 2026 10:53:52 -0700 Subject: [PATCH 44/62] Removed .claude --- .claude/worktrees/unify-component-service | 1 - 1 file changed, 1 deletion(-) delete mode 160000 .claude/worktrees/unify-component-service diff --git a/.claude/worktrees/unify-component-service b/.claude/worktrees/unify-component-service deleted file mode 160000 index 7d84eafaec..0000000000 --- a/.claude/worktrees/unify-component-service +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7d84eafaec7223ee57b2b626f02815075a8dfc47 From 54229cdb1b62356f07df8af8d963d2a8b9899cad Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Fri, 17 Jul 2026 10:54:48 -0700 Subject: [PATCH 45/62] Synced start.sh with dev --- mpcontribs-api/scripts/start.sh | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/mpcontribs-api/scripts/start.sh b/mpcontribs-api/scripts/start.sh index 729e75932d..1452743286 100755 --- a/mpcontribs-api/scripts/start.sh +++ b/mpcontribs-api/scripts/start.sh @@ -10,19 +10,4 @@ PMGRC=$HOME/.pmgrc.yaml set -x -<<<<<< HEAD exec uvicorn mpcontribs_api.app:app --host 0.0.0.0 --port "$API_PORT" --workers "${NWORKERS:-2}" -||||||| parent of ba089932 (Infra changes for rewrite) -if [[ -n "$DD_TRACE_HOST" ]]; then - wait-for-it.sh "$DD_TRACE_HOST" -q -s -t 10 && STATS_ARG="--statsd-host $DD_AGENT_HOST:8125" || echo "WARNING: datadog agent unreachable" -fi - -[[ -n "$STATS_ARG" ]] && CMD="ddtrace-run gunicorn $STATS_ARG" || CMD="gunicorn" -exec $WAIT_FOR -- $CMD $SERVER_APP -======= -if [[ -n "$DD_TRACE_HOST" ]]; then - wait-for-it.sh "$DD_TRACE_HOST" -q -s -t 10 || echo "WARNING: datadog agent unreachable" -fi - -exec uvicorn mpcontribs_api.app:app --host 0.0.0.0 --port "$API_PORT" ->>>>>>> ba089932 (Infra changes for rewrite) From 77cc80971cf84ba6a027692188df030c615d50be Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Fri, 17 Jul 2026 11:33:55 -0700 Subject: [PATCH 46/62] Moved Slug type into types.py --- .../mpcontribs_api/domains/_shared/types.py | 17 +++++++++++++ .../domains/initiatives/models.py | 24 +++---------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/_shared/types.py b/mpcontribs-api/src/mpcontribs_api/domains/_shared/types.py index 68061141a7..0abe108bca 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/_shared/types.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/_shared/types.py @@ -197,3 +197,20 @@ def to_snake_case(name: str) -> str: # Converts strs to pretty display form (keeps unicode and most formatting) DisplayStr = Annotated[str, BeforeValidator(func=nfc_normalize)] + +# A URL-safe, human-readable slug +# carried in user.groups like ``initiative:`` +_SLUG_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") + + +def _validate_slug(v: str) -> str: + v = v.strip().lower() + if not _SLUG_RE.match(v): + raise ValidationError( + "slug must be lowercase alphanumeric words separated by single hyphens, e.g. 'battery-genome-2025'", + slug=v, + ) + return v + + +Slug = Annotated[str, Field(min_length=3, max_length=50), BeforeValidator(_validate_slug)] diff --git a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/models.py b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/models.py index 49914166fc..a140191a76 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/models.py @@ -1,36 +1,18 @@ from __future__ import annotations -import re -from typing import Annotated, Self +from typing import Self from beanie import PydanticObjectId from bson.errors import InvalidId -from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from pymongo import ASCENDING, IndexModel from mpcontribs_api.domains._shared.filters import BaseFilter from mpcontribs_api.domains._shared.models import BaseDocumentWithInput, DocumentOut -from mpcontribs_api.domains._shared.types import NFKCStr, PrefixedEmail +from mpcontribs_api.domains._shared.types import NFKCStr, PrefixedEmail, Slug from mpcontribs_api.exceptions import ValidationError from mpcontribs_api.projection import SparseFieldsModel -# A URL-safe, human-readable slug -# Also carried in user.groups like ``initiative:`` -_SLUG_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") - - -def _validate_slug(v: str) -> str: - v = v.strip().lower() - if not _SLUG_RE.match(v): - raise ValidationError( - "slug must be lowercase alphanumeric words separated by single hyphens, e.g. 'battery-genome-2025'", - slug=v, - ) - return v - - -Slug = Annotated[str, Field(min_length=3, max_length=50), BeforeValidator(_validate_slug)] - class Initiative(BaseDocumentWithInput[PydanticObjectId]): """A canonical, authoritative grouping of projects into a larger organizational effort. From 4ecedc5fb07260da160a097a687a3f754e95f3f5 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 5 Aug 2026 14:43:47 -0700 Subject: [PATCH 47/62] fix(ContributionBase): added `version` to identifier_fields --- .../src/mpcontribs_api/domains/contributions/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/contributions/models.py b/mpcontribs-api/src/mpcontribs_api/domains/contributions/models.py index 7b70d10a77..db65cc6f3f 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/contributions/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/contributions/models.py @@ -109,7 +109,7 @@ class Settings: @classmethod def identifier_fields(cls) -> frozenset[str]: """A contribution is uniquely identified (within a version) by ``project`` + ``identifier``.""" - return frozenset({"project", "identifier"}) + return frozenset({"project", "identifier", "version"}) class Contribution(ContributionBase): From e5457e6a686ca1ac1a29c32aa2c3ed85f998edd8 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 5 Aug 2026 15:53:45 -0700 Subject: [PATCH 48/62] docs(InitiativeRepository): modified documentation --- .../src/mpcontribs_api/domains/initiatives/repository.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py index 6c373a5221..18381b5111 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py @@ -79,8 +79,7 @@ async def insert_initiative(self, data: InitiativeIn) -> Initiative: limit=self._limits.max_unapproved_per_owner, ) - # ``BaseDocumentWithInput`` makes ``id`` required (no auto-default), so mint the ObjectId - # here — as ``ProjectGroup.from_input_model`` does — and force owner/flags server-side. + # ``BaseDocumentWithInput`` makes ``id`` required (no auto-default), so mint the ObjectId here initiative = self.document_model.model_validate( { "_id": PydanticObjectId(), @@ -98,8 +97,7 @@ async def insert_initiative(self, data: InitiativeIn) -> Initiative: async def patch_initiative(self, slug: str, update: InitiativePatch) -> Initiative: """Patch a scoped initiative by ``slug``, enforcing manage rights and approval rules. - - The caller must be able to *manage* the initiative (owner/collaborator/admin); mere - visibility (e.g. a public initiative) is not enough. + - The caller must be able to *manage* the initiative (owner/collaborator/admin) - Only an admin may change ``is_approved``. - The resulting state must satisfy ``is_public ⇒ is_approved`` (re-checked here because a partial ``$set`` does not run the document validator). @@ -114,7 +112,7 @@ async def patch_initiative(self, slug: str, update: InitiativePatch) -> Initiati data = update.model_dump(exclude_unset=True) if "is_approved" in data and not self._user.is_admin: - raise PermissionError(required_role="admin") + raise PermissionError("only admins can set `is_approved`", required_role="admin") resulting_approved = data.get("is_approved", existing.is_approved) resulting_public = data.get("is_public", existing.is_public) From 4f38f9d980e497735b4e0457f7c59b872474a5a0 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 5 Aug 2026 16:01:46 -0700 Subject: [PATCH 49/62] refactor(MongoDbRepository): unified patching logic and removed _resolve_one_id patching logic was duplicated. _resovle_one_id is redundant since `identifiers` uniquely specify a document --- .../domains/_shared/repository.py | 68 ++++++++----------- .../domains/initiatives/models.py | 2 - .../domains/project_groups/repository.py | 14 ++-- 3 files changed, 37 insertions(+), 47 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py index b5e276f4d2..015635fde1 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py @@ -126,22 +126,6 @@ def _identifier_query(self, identifiers: dict[str, Any]) -> dict[str, Any]: ) return {("_id" if key == "id" else key): value for key, value in identifiers.items()} - async def _resolve_one_id(self, identifiers: dict[str, Any], session: AsyncClientSession | None = None) -> Any: - """Resolve the single scoped ``_id`` matching ``identifiers``, or ``None`` if absent. - - Enforces uniqueness: the identifier fields are meant to key at most one document, so if two - are found (a duplicate under a supposedly-unique key) this raises ``ConflictError`` rather - than silently picking one. - """ - query = self._identifier_query(identifiers) - projection = self.out_model.projection(frozenset({"id"})) - docs = ( - await self.document_model.find(self._scope, query, session=session).limit(2).project(projection).to_list() - ) # pyright: ignore[reportArgumentType] - if len(docs) > 1: - raise ConflictError("identifiers matched more than one document", identifiers=identifiers) - return docs[0].id if docs else None - async def get_one( self, identifiers: dict[str, Any], @@ -149,19 +133,13 @@ async def get_one( ) -> TOut | None: """Return the single scoped document matching ``identifiers``, projected to ``fields``. - Returns ``None`` when nothing matches, but ``ConflictError`` if the identifiers match - more than one document. - Args: identifiers (dict[str, Any]): identifier field values keyed by ``identifier_fields`` fields (frozenset[str] | None): fields to project; if None the full document is returned """ query = self._identifier_query(identifiers) projection = self.out_model.projection(fields) - docs = await self.document_model.find(self._scope, query).limit(2).project(projection).to_list() # pyright: ignore[reportArgumentType] - if len(docs) > 1: - raise ConflictError("identifiers matched more than one document", identifiers=identifiers) - return docs[0] if docs else None + return await self.document_model.find_one(self._scope, query, projection_model=projection) # pyright: ignore[reportArgumentType] async def get_by_id(self, id: Any, fields: frozenset[str] | None = None) -> TDoc | TOut | None: """Return a single scoped document by id, projected to the requested fields. @@ -231,18 +209,15 @@ async def delete_one( ) -> DeleteResponse: """Delete the single scoped document matching ``identifiers``. - Uniqueness is checked before anything is deleted (see :meth:`_resolve_one_id`), so a - duplicate raises ``ConflictError`` and an absent resource raises ``NotFoundError`` — this - never deletes more than the one intended document. - Args: identifiers (dict[str, Any]): identifier field values keyed by ``identifier_fields`` session (AsyncClientSession | None): optional client session for transactions """ - oid = await self._resolve_one_id(identifiers, session=session) - if oid is None: + query = self._identifier_query(identifiers) + result = await self.document_model.find_one(self._scope, query, session=session).delete(session=session) # pyright: ignore[reportArgumentType] + if result is None or result.deleted_count == 0: raise NotFoundError(f"{self.document_model.__name__} not found", identifiers=identifiers) - return await self.delete_by_id(oid, session=session) + return DeleteResponse.from_delete_result(result) async def delete_by_id(self, id: Any, session: AsyncClientSession | None = None) -> DeleteResponse: """Delete a single scoped document by its primary key (``_id``). @@ -291,25 +266,40 @@ async def patch(self, id: Any, update: TPatch) -> TDoc: id (str): the id of the document to update update (TPatch): the partial update to apply; unset fields are dropped """ + return await self._patch_matching(self.document_model.id == id, update, NotFoundError(self._not_found(id))) + + async def _patch_matching( + self, + match: Any, + update: TPatch, + not_found: NotFoundError, + session: AsyncClientSession | None = None, + ) -> TDoc: + """Apply a partial update to the single scoped document matching ``match``. + + ``match`` is any beanie filter that keys at most one in-scope document — a primary-key + equality (:meth:`patch`) or a unique-identifier query (:meth:`patch_one`). An empty patch + is a no-op that still returns the existing document; a missing target raises ``not_found``. + """ # Only retain set fields (patch) update_data = update.model_dump(exclude_unset=True) # If update is empty, return the model anyways (consistent behavior) if not update_data: - existing = await self.document_model.find_one(self._scope, self.document_model.id == id) + existing = await self.document_model.find_one(self._scope, match, session=session) if existing is None: - raise NotFoundError(self._not_found(id)) + raise not_found return existing # Otherwise, update the fields fully (set) # Brendan TODO: Set will replace an entire field # - if we want to append to a list (ie. add a reference) we ned Push/AddToSet - query = self.document_model.find_one(self._scope, self.document_model.id == id).update( + query = self.document_model.find_one(self._scope, match, session=session).update( Set(update_data), response_type=UpdateResponse.NEW_DOCUMENT, ) updated = await query # pyright: ignore[reportGeneralTypeIssues] # beanie UpdateQuery is awaitable, but pyright doesn't see it if updated is None: - raise NotFoundError(self._not_found(id)) + raise not_found return updated async def patch_one( @@ -320,18 +310,14 @@ async def patch_one( ) -> TDoc: """Partially update the single scoped document matching ``identifiers``. - Resolves the target by its unique identifier fields (raising ``ConflictError`` on a - duplicate, ``NotFoundError`` when absent) and then applies the patch via :meth:`patch`. - Args: identifiers (dict[str, Any]): identifier field values keyed by ``identifier_fields`` update (TPatch): the partial update to apply; unset fields are dropped session (AsyncClientSession | None): optional client session for transactions """ - oid = await self._resolve_one_id(identifiers, session=session) - if oid is None: - raise NotFoundError(f"{self.document_model.__name__} not found", identifiers=identifiers) - return await self.patch(oid, update) + query = self._identifier_query(identifiers) + not_found = NotFoundError(f"{self.document_model.__name__} not found", identifiers=identifiers) + return await self._patch_matching(query, update, not_found, session=session) def _hash_payload(self, payload: dict[str, Any], *, separators: tuple[str, str] = (",", ":")) -> str: canonical = json.dumps( diff --git a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/models.py b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/models.py index a140191a76..10cf63180f 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/models.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from typing import Self from beanie import PydanticObjectId diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py index f615923a39..faef61afcd 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py @@ -79,13 +79,19 @@ async def patch_project_group( return await self.patch_one({"name": name, "owner": owner}, update) async def delete_project_group(self, name: SearchStr, owner: PrefixedEmail) -> DeleteResponse: - """Delete the single project group identified by ``name`` + ``owner``.""" - oid = await self._resolve_one_id({"name": name, "owner": owner}) - if oid is None: + """Delete the single project group identified by ``name`` + ``owner``. + + Absence (in scope) takes precedence over the ownership gate: a non-admin may only delete + their own group, so deleting another owner's visible (public) group is forbidden rather + than silently a no-op. The ``name`` + ``owner`` unique index makes the match unambiguous. + """ + doc = await self.document_model.find_one(self._scope, {"name": name, "owner": owner}) # pyright: ignore[reportArgumentType] + if doc is None: raise NotFoundError(f"{self.document_model.__name__} not found", name=name, owner=owner) if not (self._user.is_admin or owner == self._user.username): raise PermissionError(required_role="owner-or-admin") - return await self.delete_by_id(oid) + await doc.delete() + return DeleteResponse(num_deleted=1) async def delete_project_groups(self, filter: ProjectGroupFilter) -> DeleteResponse: """Bulk-delete project groups matching ``filter``, restricted to the caller's own. From 296dafc353cb729713b04769f4ed0b8a5e8990a2 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 5 Aug 2026 16:04:27 -0700 Subject: [PATCH 50/62] refactor(MongoDbProjectRepository): deduplicated patch logic previously, patching an initiative with a project used 2 db calls --- .../domains/projects/repository.py | 58 +++++++++++-------- .../domains/projects/service.py | 13 ++--- 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py index 6bd677024a..222c7988e1 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py @@ -93,7 +93,41 @@ async def patch_project_by_id(self, id: str, update: ProjectPatch) -> Project: - Resulting state must satisfy is_public <-> is_approved condition The ``initiative`` field is split out upstream in ``ProjectService.patch``, so it never - reaches this method. + reaches this method; an assignment that also edits plain fields goes through + :meth:`patch_project_with_initiative` instead. + """ + await self._enforce_patch_rules(id, update) + return await self.patch(id, update) + + async def patch_project_with_initiative(self, id: str, update: ProjectPatch, ref: DBRef | None) -> Project: + """Atomically apply a partial project update together with its canonical initiative link. + + Args: + id (str): the id of the project to update + update (ProjectPatch): the partial update to apply; unset fields are dropped. The + ``initiative`` slug must already be stripped — the resolved ``ref`` carries the link. + ref (DBRef | None): the initiative reference to assign, or None to unassign + """ + await self._enforce_patch_rules(id, update) + data = update.model_dump(exclude_unset=True) + data["initiative"] = ref + query = self.document_model.find_one(self._scope, self.document_model.id == id).update( + Set(data), + response_type=UpdateResponse.NEW_DOCUMENT, + ) + updated = await query # pyright: ignore[reportGeneralTypeIssues] # beanie UpdateQuery is awaitable + if updated is None: + raise NotFoundError(self._not_found(id)) + return updated + + async def _enforce_patch_rules(self, id: str, update: ProjectPatch) -> None: + """Enforce project patch invariants against the scoped target. + + - Only an admin may change ``is_approved``. + - The resulting state must satisfy the is_public -> is_approved condition. + + Raises ``NotFoundError`` when the project is invisible to the caller or absent, so both the + plain and initiative-bearing patch paths reject unseen documents identically. """ data = update.model_dump(exclude_unset=True) if "is_approved" in data and not self._user.is_admin: @@ -108,8 +142,6 @@ async def patch_project_by_id(self, id: str, update: ProjectPatch) -> Project: if resulting_public and not resulting_approved: raise ValidationError("a project cannot be public until it is approved", id=id) - return await self.patch(id, update) - async def delete_project_by_id(self, id: str) -> None: """Delete a scoped project by id. Restricted to the owner or an admin. @@ -174,26 +206,6 @@ async def upsert_project_by_id(self, id: str, data: ProjectIn) -> Project: raise ValidationError("a project cannot be public until it is approved", id=id) return await project.save() - async def set_initiative(self, id: str, ref: DBRef | None) -> Project: - """Set a scoped project's canonical initiative link. - - The link is written as-is (a ``DBRef`` into ``initiatives`` or ``None``); all authorization - and limit checks are the caller's (see ``ProjectInitiativeService``). Scoping ensures a - project the caller cannot see is reported as not found rather than silently missed. - - Args: - id (str): the id of the project to update - ref (DBRef | None): the initiative reference to assign, or None to unassign - """ - query = self.document_model.find_one(self._scope, self.document_model.id == id).update( - Set({"initiative": ref}), - response_type=UpdateResponse.NEW_DOCUMENT, - ) - updated = await query # pyright: ignore[reportGeneralTypeIssues] # beanie UpdateQuery is awaitable - if updated is None: - raise NotFoundError(self._not_found(id)) - return updated - async def count_initiative_members(self, initiative_id: PydanticObjectId, exclude_project_id: str | None) -> int: """Count projects assigned to an initiative, ignoring user scope. diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py index 95c8120689..2852aee49a 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py @@ -22,9 +22,11 @@ def __init__( async def patch(self, id: str, update: ProjectPatch) -> Project: """Apply a project patch, routing an ``initiative`` change through the assignment checks. - ``initiative`` carries the target initiative's ``slug`` (or ``null`` to unassign). It is - split out of the patch so it never reaches the raw ``$set`` as a bare string; any remaining - fields are applied first, then the initiative link is set. + ``initiative`` carries the target initiative's ``slug`` (or ``null`` to unassign). When + present it is split out of the patch (so it never reaches the raw ``$set`` as a bare + string), resolved to a link — running the both-rights and member-cap checks — then written + together with any co-submitted plain fields in a single atomic update, so the request never + persists a half-applied change. """ if "initiative" not in update.model_fields_set: return await self._projects.patch_project_by_id(id=id, update=update) @@ -35,10 +37,7 @@ async def patch(self, id: str, update: ProjectPatch) -> Project: # Resolve the target link (and run the both-rights + limit checks) before touching anything. ref = await self._resolve_initiative_assignment(project_id=id, slug=slug) - # Apply any co-submitted plain fields first, so a single request can rename and assign. - if data: - await self._projects.patch_project_by_id(id=id, update=ProjectPatch(**data)) - return await self._projects.set_initiative(id=id, ref=ref) + return await self._projects.patch_project_with_initiative(id=id, update=ProjectPatch(**data), ref=ref) async def _resolve_initiative_assignment(self, project_id: str, slug: str | None) -> DBRef | None: """Validate an initiative assignment and return the link to store (or None to unassign). From 3a14ca25a6e277f191ff5c37876449ed7f9bdd58 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 5 Aug 2026 16:12:08 -0700 Subject: [PATCH 51/62] test(tests): removed unnecessary tests and corrected test that checks contribution identifier_fields --- .../db/test_project_groups_repository.py | 26 ------------------- .../db/test_project_groups_service.py | 23 ---------------- .../tests/unit/domains/test_shared_models.py | 6 ++--- 3 files changed, 3 insertions(+), 52 deletions(-) diff --git a/mpcontribs-api/tests/integration/db/test_project_groups_repository.py b/mpcontribs-api/tests/integration/db/test_project_groups_repository.py index 26928cfb9a..3cb20f5b56 100644 --- a/mpcontribs-api/tests/integration/db/test_project_groups_repository.py +++ b/mpcontribs-api/tests/integration/db/test_project_groups_repository.py @@ -156,32 +156,6 @@ async def test_wrong_identifier_keys_raise_validation(self, db): with pytest.raises(ValidationError): await _repo(ADMIN).delete_one({"name": "x"}) # missing 'owner' - async def test_duplicate_identifiers_raise_conflict(self, db): - # The name_owner unique index normally makes this impossible; drop it so we can plant a - # duplicate and exercise the defensive uniqueness guard in _resolve_one_id. Tolerant of a - # prior drop within the same session. - try: - await db["project_groups"].drop_index("name_owner") - except Exception: - pass - await db["project_groups"].insert_many( - [ - {"_id": PydanticObjectId(), "name": "dup", "owner": ALICE_EMAIL, "projects": [], "description": "d"}, - {"_id": PydanticObjectId(), "name": "dup", "owner": ALICE_EMAIL, "projects": [], "description": "d"}, - ] - ) - try: - with pytest.raises(ConflictError): - await _repo(ADMIN).delete_project_group(name="dup", owner=ALICE_EMAIL) - finally: - # Restore the unique index we dropped so order-dependent tests that rely on it (e.g. the - # insert-duplicate guard) still see it. Planted duplicates must go first, or the unique - # index rebuild would fail. - await db["project_groups"].delete_many({"name": "dup"}) - await db["project_groups"].create_index( - [("name", 1), ("owner", 1)], name="name_owner", unique=True - ) - # --------------------------------------------------------------------------- # patch_one diff --git a/mpcontribs-api/tests/integration/db/test_project_groups_service.py b/mpcontribs-api/tests/integration/db/test_project_groups_service.py index 4c9d5acb45..5bbadf2fd2 100644 --- a/mpcontribs-api/tests/integration/db/test_project_groups_service.py +++ b/mpcontribs-api/tests/integration/db/test_project_groups_service.py @@ -103,29 +103,6 @@ async def test_group_not_visible_raises_not_found(self, db): with pytest.raises(NotFoundError): await _service(ANON).add_projects_by_id(str(group.id), []) - async def test_ambiguous_identifiers_raise_conflict(self, db): - # Drop the unique index so we can plant a duplicate and exercise the uniqueness guard. - try: - await db["project_groups"].drop_index("name_owner") - except Exception: - pass - await db["project_groups"].insert_many( - [ - {"_id": PydanticObjectId(), "name": "dup", "owner": ALICE_EMAIL, "projects": [], "description": "d"}, - {"_id": PydanticObjectId(), "name": "dup", "owner": ALICE_EMAIL, "projects": [], "description": "d"}, - ] - ) - try: - with pytest.raises(ConflictError): - await _service().add_projects_by_identifiers("dup", ALICE_EMAIL, []) - finally: - # Restore the unique index we dropped so order-dependent tests that rely on it still see - # it. Planted duplicates must go first, or the unique index rebuild would fail. - await db["project_groups"].delete_many({"name": "dup"}) - await db["project_groups"].create_index( - [("name", 1), ("owner", 1)], name="name_owner", unique=True - ) - # --------------------------------------------------------------------------- # delete diff --git a/mpcontribs-api/tests/unit/domains/test_shared_models.py b/mpcontribs-api/tests/unit/domains/test_shared_models.py index b7cb13ad7d..959973eea5 100644 --- a/mpcontribs-api/tests/unit/domains/test_shared_models.py +++ b/mpcontribs-api/tests/unit/domains/test_shared_models.py @@ -117,9 +117,9 @@ def test_project_uses_id(self): def test_project_group_uses_name_and_owner(self): assert ProjectGroup.identifier_fields() == frozenset({"name", "owner"}) - def test_contribution_uses_project_and_identifier(self): - assert Contribution.identifier_fields() == frozenset({"project", "identifier"}) - assert ContributionIn.identifier_fields() == frozenset({"project", "identifier"}) + def test_contribution_uses_project_identifier_version(self): + assert Contribution.identifier_fields() == frozenset({"project", "identifier", "version"}) + assert ContributionIn.identifier_fields() == frozenset({"project", "identifier", "version"}) def test_default_identifiers_reads_values_off_instance(self): oid = PydanticObjectId() From b6f0d344557d69ddafe57032baa6b3e903bb0cd8 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 5 Aug 2026 16:33:33 -0700 Subject: [PATCH 52/62] refactor(MongoDbRepository): removed useless abstraction of NotFoundError (_not_found()) --- .../src/mpcontribs_api/domains/_shared/repository.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py index 015635fde1..54a1573e22 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py @@ -72,10 +72,6 @@ def _convert_object_id(self, id: str) -> PydanticObjectId: except InvalidId: raise ValidationError("Incorrect Id format. Must be MongoDB ObjectId format.", id=id) from None - def _not_found(self, id: str) -> str: - """Build a not-found message naming this repository's resource.""" - return f"{self.document_model.__name__} with id {id} not found" - async def get_many( self, filter: TFilter, @@ -232,7 +228,7 @@ async def delete_by_id(self, id: Any, session: AsyncClientSession | None = None) """ doc = await self.document_model.find_one(self._scope, self.document_model.id == id, session=session) if doc is None: - raise NotFoundError(self._not_found(id)) + raise NotFoundError(message=f"{self.document_model.__name__} not found by id", id=id) await doc.delete(session=session) return DeleteResponse(num_deleted=1) @@ -266,7 +262,9 @@ async def patch(self, id: Any, update: TPatch) -> TDoc: id (str): the id of the document to update update (TPatch): the partial update to apply; unset fields are dropped """ - return await self._patch_matching(self.document_model.id == id, update, NotFoundError(self._not_found(id))) + return await self._patch_matching( + self.document_model.id == id, update, NotFoundError(f"{self.document_model.__name__} not found", id=id) + ) async def _patch_matching( self, From ed213357297141f65c30ad60f5d431fe32844e3a Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Tue, 18 Aug 2026 16:03:53 -0700 Subject: [PATCH 53/62] refactor(multiple): brought logic into higher level classes and simplified some method grammar moved from *_by_id and *_by_identifiers into a unified *_one grammar --- .../domains/_shared/components.py | 39 ------- .../mpcontribs_api/domains/_shared/models.py | 19 ++++ .../domains/_shared/repository.py | 94 +++++++--------- .../mpcontribs_api/domains/_shared/service.py | 58 ++++++---- .../domains/attachments/router.py | 26 +++-- .../domains/contributions/models.py | 2 +- .../domains/contributions/repository.py | 82 +++++--------- .../domains/contributions/router.py | 18 +-- .../domains/contributions/service.py | 52 ++++++++- .../domains/initiatives/repository.py | 25 +++-- .../domains/initiatives/router.py | 12 +- .../domains/project_groups/repository.py | 34 ++---- .../domains/project_groups/router.py | 36 +++--- .../domains/project_groups/service.py | 73 +++++++------ .../domains/projects/repository.py | 43 ++++---- .../mpcontribs_api/domains/projects/router.py | 28 ++--- .../domains/projects/service.py | 22 +++- .../domains/structures/router.py | 19 ++-- .../mpcontribs_api/domains/tables/router.py | 19 ++-- .../db/test_component_reachability.py | 6 +- .../db/test_components_repository.py | 46 ++++++-- .../db/test_contributions_repository.py | 103 ++++++++++++------ .../db/test_initiatives_repository.py | 48 ++++---- .../db/test_initiatives_service.py | 44 ++++---- .../db/test_project_groups_repository.py | 32 +++--- .../db/test_project_groups_service.py | 26 ++--- .../db/test_projects_repository.py | 80 +++++++------- .../integration/test_component_routes.py | 26 ++--- .../integration/test_contributions_routes.py | 40 +++---- .../tests/integration/test_initiatives.py | 8 +- .../tests/integration/test_projects.py | 88 ++++++++------- .../unit/domains/test_component_service.py | 45 ++++---- .../unit/domains/test_contribution_service.py | 40 ++++--- .../domains/test_project_group_service.py | 35 +++--- .../tests/unit/domains/test_shared_models.py | 10 +- 35 files changed, 729 insertions(+), 649 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/_shared/components.py b/mpcontribs-api/src/mpcontribs_api/domains/_shared/components.py index 1af1edb05e..fd2841eb2e 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/_shared/components.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/_shared/components.py @@ -10,7 +10,6 @@ from mpcontribs_api.domains._shared.models import Component, ComponentIn, DeleteResponse, DocumentOut from mpcontribs_api.domains._shared.repository import MongoDbRepository from mpcontribs_api.domains._shared.types import MD5Hash -from mpcontribs_api.exceptions import NotFoundError class MongoDbComponentsRepository[ @@ -94,10 +93,6 @@ async def insert_component(self, component: TIn, *, session: AsyncClientSession """ return (await self.insert_components(components=[component], session=session))[0] - async def get_component_by_id(self, id: str, fields: frozenset[str] | None) -> TDoc | TOut | None: - """Find a single component by id. See ``get_by_id``.""" - return await self.get_by_id(self._convert_object_id(id), fields) - async def delete_components( self, filter: TFilter, @@ -115,37 +110,3 @@ async def delete_components( query = filter.filter(self.document_model.find(self._scope, session=session)) result = await query.delete(session=session) return DeleteResponse(num_deleted=result.deleted_count if result else 0) - - async def delete_component_by_id( - self, - id: str, - session: AsyncClientSession | None = None, - ) -> DeleteResponse: - """Deletes a single component by Id. - - Args: - id (str): the str representation of the component's ObjectId - session (AsyncClientSession | None): the current session, used to guarantee transactions - - Returns: - DeleteResponse: A report of the deletion - """ - return await self.delete_by_id(id=self._convert_object_id(id), session=session) - - async def patch_component_by_id(self, id: str, update: TPatch) -> TDoc: - """Partially update a component by id, recomputing its content hash. - - Components are content-addressed, so a content change must update ``md5``. Unlike the base - ``patch`` (an in-place ``$set``), this loads the full document, applies the set fields, - recomputes ``md5`` from ``hash_fields``, and saves — keeping md5 consistent with content. - """ - oid = self._convert_object_id(id) - doc = await self.document_model.find_one(self._scope, self.document_model.id == oid) - if doc is None: - raise NotFoundError(self._not_found(id)) - update_data = update.model_dump(exclude_unset=True) - for field, value in update_data.items(): - setattr(doc, field, value) - doc.md5 = doc.compute_md5() - await doc.save() - return doc diff --git a/mpcontribs-api/src/mpcontribs_api/domains/_shared/models.py b/mpcontribs-api/src/mpcontribs_api/domains/_shared/models.py index 7051658f9e..0c16342e84 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/_shared/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/_shared/models.py @@ -24,6 +24,7 @@ class BaseDocumentWithInput[TId](Document): models subclass their document, so they can't be bound as a class type parameter). """ + HAS_DERIVED_FIELDS: ClassVar[bool] = False # Required, non-null, resource-specific id. Overrides Document's optional ``PydanticObjectId`` id. id: TId = Field(alias="_id") # pyright: ignore[reportGeneralTypeIssues, reportIncompatibleVariableOverride] @@ -43,6 +44,14 @@ def identifiers(self) -> dict[str, Any]: """This document's identifier field values, keyed by :meth:`identifier_fields`.""" return {field: getattr(self, field) for field in self.identifier_fields()} + def derived_field_updates(self) -> dict[str, Any]: + """Server-derived fields to persist alongside a patch. + + Called by the repository on a copy of this document that already has the patch applied + in memory; the returned mapping is merged into the same ``$set`` write. Default: none. + """ + return {} + @classmethod def from_input_model(cls, data: Any) -> Self: """Translate a validated input payload into a full stored document.""" @@ -113,12 +122,22 @@ class Component(BaseDocumentWithInput[PydanticObjectId]): never define a component's content identity. """ + HAS_DERIVED_FIELDS: ClassVar[bool] = True name: str # Server-computed; the placeholder default is overwritten by ``_recompute_md5`` on validation. md5: MD5Hash = Field(default="0" * 32) hash_fields: ClassVar[frozenset[str]] + @classmethod + def identifier_fields(cls) -> frozenset[str]: + """A component is content-addressed: its ``md5`` uniquely identifies its content.""" + return frozenset({"md5"}) + + def derived_field_updates(self) -> dict[str, Any]: + """Recompute ``md5`` from the (patched-in-memory) content so the write stays authoritative.""" + return {"md5": self.compute_md5()} + # The md5 functions look redundant but aren't, we should keep both # Used in patching to compute the hash after an update - should not return self def compute_md5(self) -> str: diff --git a/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py index 54a1573e22..1219690abd 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py @@ -72,6 +72,26 @@ def _convert_object_id(self, id: str) -> PydanticObjectId: except InvalidId: raise ValidationError("Incorrect Id format. Must be MongoDB ObjectId format.", id=id) from None + def coerce_identifiers(self, identifiers: dict[str, Any]) -> dict[str, Any]: + """Return ``identifiers`` with a string ``id`` coerced to the model's primary-key type. + + Externally supplied ids arrive as strings (path/query params). ObjectId-keyed models need + that string parsed into a ``PydanticObjectId`` before it can match ``_id``; string-keyed + models (e.g. ``Project``, whose id is its name) and every non-``id`` identifier key pass + through untouched. Idempotent: an already-parsed id is returned unchanged. + + This is the single home for identifier coercion so services that resolve documents by id + (and reuse the parsed id for cross-repository lookups) do not each reimplement it. + + Raises: + ValidationError: if ``id`` is a string that is not a valid ObjectId, for an + ObjectId-keyed model + """ + id = identifiers.get("id") + if isinstance(id, str) and self.document_model.model_fields["id"].annotation is PydanticObjectId: + return {**identifiers, "id": self._convert_object_id(id)} + return identifiers + async def get_many( self, filter: TFilter, @@ -106,17 +126,20 @@ async def get_many( def _identifier_query(self, identifiers: dict[str, Any]) -> dict[str, Any]: """Turn a ``{field: value}`` identifier dict into a scoped Mongo query fragment. - The keys must be exactly the model's :meth:`identifier_fields` - ``id`` is remapped Mongo's ``_id`` (mirroring ``BaseFilter._get_filter_conditions``) - since a raw dict query does not go through Beanie's alias resolution. + The keys must be either the model's :meth:`identifier_fields` exactly, or the bare + primary-key form ``{"id": ...}`` (which addresses any document by its ``_id`` regardless of + its semantic identifier). ``id`` is remapped to Mongo's ``_id`` (mirroring + ``BaseFilter._get_filter_conditions``) since a raw dict query does not go through Beanie's + alias resolution. Args: - identifiers (dict[str, Any]): identifier field values keyed by ``identifier_fields`` + identifiers (dict[str, Any]): identifier field values keyed by ``identifier_fields``, + or ``{"id": }`` """ expected = self.document_model.identifier_fields() - if identifiers.keys() != expected: + if identifiers.keys() != expected and identifiers.keys() != {"id"}: raise ValidationError( - "identifiers must match the model's identifier fields exactly", + "identifiers must match the model's identifier fields, or be a bare {'id': ...}", expected=sorted(expected), received=sorted(identifiers.keys()), ) @@ -137,19 +160,6 @@ async def get_one( projection = self.out_model.projection(fields) return await self.document_model.find_one(self._scope, query, projection_model=projection) # pyright: ignore[reportArgumentType] - async def get_by_id(self, id: Any, fields: frozenset[str] | None = None) -> TDoc | TOut | None: - """Return a single scoped document by id, projected to the requested fields. - - Args: - id (str): the id of the document to find - fields (frozenset[str] | None): fields to project; if None the full document is returned - """ - return await self.document_model.find_one( - self._scope, - self.document_model.id == id, - projection_model=self.out_model.projection(fields), - ) - async def list_ids(self, filter: TFilter, session: AsyncClientSession | None = None) -> list[Any]: """Return just the ids of scoped documents matching ``filter``. @@ -215,23 +225,6 @@ async def delete_one( raise NotFoundError(f"{self.document_model.__name__} not found", identifiers=identifiers) return DeleteResponse.from_delete_result(result) - async def delete_by_id(self, id: Any, session: AsyncClientSession | None = None) -> DeleteResponse: - """Delete a single scoped document by its primary key (``_id``). - - Scoping ensures callers cannot delete documents they are not permitted to see; an id that is - absent or out of scope raises ``NotFoundError``. Kept distinct from :meth:`delete_one`, whose - key is the semantic ``identifier_fields`` (which differs from ``_id`` for some resources). - - Args: - id (Any): the primary key of the document to delete - session (AsyncClientSession | None): optional client session for transactions - """ - doc = await self.document_model.find_one(self._scope, self.document_model.id == id, session=session) - if doc is None: - raise NotFoundError(message=f"{self.document_model.__name__} not found by id", id=id) - await doc.delete(session=session) - return DeleteResponse(num_deleted=1) - async def delete_by_ids(self, ids: list[Any], session: AsyncClientSession | None = None) -> DeleteResponse: """Delete multiple scoped documents by id. @@ -251,21 +244,6 @@ async def delete_by_ids(self, ids: list[Any], session: AsyncClientSession | None raise ValidationError("DeleteResult not returned internally") return DeleteResponse.from_delete_result(delete_result) - async def patch(self, id: Any, update: TPatch) -> TDoc: - """Partially update a single scoped document by id. - - Only fields explicitly set on ``update`` are applied. An empty patch is a no-op that still - returns the existing document for consistent behavior. Scoping ensures callers cannot patch - documents they are not permitted to see. - - Args: - id (str): the id of the document to update - update (TPatch): the partial update to apply; unset fields are dropped - """ - return await self._patch_matching( - self.document_model.id == id, update, NotFoundError(f"{self.document_model.__name__} not found", id=id) - ) - async def _patch_matching( self, match: Any, @@ -275,19 +253,27 @@ async def _patch_matching( ) -> TDoc: """Apply a partial update to the single scoped document matching ``match``. - ``match`` is any beanie filter that keys at most one in-scope document — a primary-key - equality (:meth:`patch`) or a unique-identifier query (:meth:`patch_one`). An empty patch + ``match`` is any beanie filter that keys at most one in-scope document. An empty patch is a no-op that still returns the existing document; a missing target raises ``not_found``. """ # Only retain set fields (patch) update_data = update.model_dump(exclude_unset=True) + existing = await self.document_model.find_one(self._scope, match, session=session) # If update is empty, return the model anyways (consistent behavior) if not update_data: - existing = await self.document_model.find_one(self._scope, match, session=session) if existing is None: raise not_found return existing + # Server-derived fields depend on the resulting document, which a bare $set never revalidates. + # Load, apply the patch in memory, and fold the recomputed values in. + if self.document_model.HAS_DERIVED_FIELDS: + if existing is None: + raise not_found + for field, value in update_data.items(): + setattr(existing, field, value) + update_data |= existing.derived_field_updates() + # Otherwise, update the fields fully (set) # Brendan TODO: Set will replace an entire field # - if we want to append to a list (ie. add a reference) we ned Push/AddToSet diff --git a/mpcontribs-api/src/mpcontribs_api/domains/_shared/service.py b/mpcontribs-api/src/mpcontribs_api/domains/_shared/service.py index d3ce92337e..a97d177aec 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/_shared/service.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/_shared/service.py @@ -1,6 +1,8 @@ from collections.abc import AsyncIterable from contextlib import AbstractAsyncContextManager +from typing import Any +from beanie import PydanticObjectId from fastapi_filter.contrib.beanie import Filter from pydantic import BaseModel from pymongo.asynchronous.client_session import AsyncClientSession @@ -70,16 +72,24 @@ async def get_many( pagination=pagination, filter=filter, fields=fields, restrict_ids=allowed ) - async def get_by_id(self, id: str, fields: frozenset[str] | None) -> TDoc | TOut | None: - """Find a single component by id, gated by contribution reachability. + async def _resolve_component_id(self, identifiers: dict[str, Any]) -> PydanticObjectId | None: + """Return the component ``_id`` after finding it via identifiers, or None if absent.""" + if "id" in identifiers: + return identifiers["id"] + existing = await self._components.get_one(identifiers, frozenset({"id"})) + return existing.id if existing is not None else None - Returns ``None`` (treated as not found) when no in-scope contribution references the id, - so callers cannot read a component belonging to a contribution they cannot see. + async def get_one(self, identifiers: dict[str, Any], fields: frozenset[str] | None) -> TDoc | TOut | None: + """Find a single component matching ``identifiers``, gated by contribution reachability. + + Returns ``None`` when no in-scope contribution references the component. + Accepts either the bare ``{"id": ...}`` form or the content-hash ``{"md5": ...}`` form. """ - oid = self._components._convert_object_id(id) - if not await self._contributions.referenced_component_ids(self._ref_field, [oid], scoped=True): + identifiers = self._components.coerce_identifiers(identifiers) + oid = await self._resolve_component_id(identifiers) + if oid is None or not await self._contributions.referenced_component_ids(self._ref_field, [oid], scoped=True): return None - return await self._components.get_component_by_id(id, fields) + return await self._components.get_one(identifiers, fields) async def insert( self, @@ -89,16 +99,19 @@ async def insert( """Bulk-insert components, deduplicated by content hash. See ``insert_components``.""" return await self._components.insert_components(components=components, session=session) - async def patch_by_id(self, id: str, update: TPatch) -> TDoc: - """Partially update a component by id, gated by contribution reachability. + async def patch_one(self, identifiers: dict[str, Any], update: TPatch) -> TDoc: + """Partially update a component matching ``identifiers``, gated by contribution reachability. + + Accepts either the bare ``{"id": ...}`` form or the content-hash ``{"md5": ...}`` form. Raises: - NotFoundError: when no in-scope contribution references the id + NotFoundError: when no in-scope contribution references the component """ - oid = self._components._convert_object_id(id) - if not await self._contributions.referenced_component_ids(self._ref_field, [oid], scoped=True): - raise NotFoundError(self._components._not_found(id)) - return await self._components.patch_component_by_id(id=id, update=update) + identifiers = self._components.coerce_identifiers(identifiers) + oid = await self._resolve_component_id(identifiers) + if oid is None or not await self._contributions.referenced_component_ids(self._ref_field, [oid], scoped=True): + raise NotFoundError(f"{self._components.document_model.__name__} not found", **identifiers) + return await self._components.patch_one(identifiers, update) async def download( self, @@ -146,11 +159,13 @@ async def delete(self, filter: TFilter) -> ComponentDeleteResponse: referenced_ids=sorted(referenced), ) - async def delete_by_id(self, id: str) -> ComponentDeleteResponse: - """Delete a single component by id, subject to the access and integrity gates. + async def delete_one(self, identifiers: dict[str, Any]) -> ComponentDeleteResponse: + """Delete a single component matching ``identifiers``, subject to the access and integrity gates. + + Accepts either the bare ``{"id": ...}`` form or the content-hash ``{"md5": ...}`` form. Args: - id (str): the str representation of the component's ObjectId + identifiers (dict[str, Any]): identifier field values, ``{"id": ...}`` or ``{"md5": ...}`` Returns: ComponentDeleteResponse: the deletion result, or a skipped result if still referenced @@ -158,10 +173,11 @@ async def delete_by_id(self, id: str) -> ComponentDeleteResponse: Raises: NotFoundError: if the component is not reachable via any in-scope contribution """ - oid = self._components._convert_object_id(id) - if not await self._contributions.referenced_component_ids(self._ref_field, [oid], scoped=True): - raise NotFoundError(self._components._not_found(id)) + identifiers = self._components.coerce_identifiers(identifiers) + oid = await self._resolve_component_id(identifiers) + if oid is None or not await self._contributions.referenced_component_ids(self._ref_field, [oid], scoped=True): + raise NotFoundError(f"{self._components.document_model.__name__} not found", **identifiers) if await self._contributions.referenced_component_ids(self._ref_field, [oid], scoped=False): return ComponentDeleteResponse(num_deleted=0, num_skipped=1, referenced_ids=[oid]) - deleted = await self._components.delete_by_id(oid) + deleted = await self._components.delete_one({"id": oid}) return ComponentDeleteResponse(num_deleted=deleted.num_deleted) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/attachments/router.py b/mpcontribs-api/src/mpcontribs_api/domains/attachments/router.py index 09214a4fa2..de913e2474 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/attachments/router.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/attachments/router.py @@ -13,7 +13,7 @@ download_filename, ) from mpcontribs_api.domains.attachments.dependencies import AttachmentServiceDep -from mpcontribs_api.domains.attachments.models import AttachmentFilter, AttachmentOut +from mpcontribs_api.domains.attachments.models import AttachmentFilter, AttachmentOut, AttachmentPatch from mpcontribs_api.pagination import CursorParams router = APIRouter() @@ -30,14 +30,15 @@ async def get_attachments( return await service.get_many(filter=filter, fields=selected, pagination=pagination) -@router.get("/{pk}") -async def get_attachment( +@router.get("/{id}") +async def get_one( service: AttachmentServiceDep, - pk: str, + id: str, fields: FieldSelector = AttachmentOut.default_fields(), ): + """Return a single attachment addressed by its ``_id``.""" selected = AttachmentOut.parse_fields(fields) - return await service.get_by_id(id=pk, fields=selected) + return await service.get_one(identifiers={"id": id}, fields=selected) @router.get("/download/{short_mime}") @@ -73,5 +74,16 @@ async def delete_attachments(service: AttachmentServiceDep, filter: AttachmentFi @router.delete("/{id}", response_model=ComponentDeleteResponse, dependencies=[Depends(require_user)]) -async def delete_attachment_by_id(service: AttachmentServiceDep, id: str): - return await service.delete_by_id(id=id) +async def delete_one(service: AttachmentServiceDep, id: str): + """Delete a single attachment addressed by its ``_id``.""" + return await service.delete_one(identifiers={"id": id}) + + +@router.patch("/{id}", dependencies=[Depends(require_user)]) +async def patch_one( + service: AttachmentServiceDep, + id: str, + update: AttachmentPatch, +): + """Patch a single attachment addressed by its ``_id`` or its content ``md5``.""" + return await service.patch_one(identifiers={"id": id}, update=update) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/contributions/models.py b/mpcontribs-api/src/mpcontribs_api/domains/contributions/models.py index db65cc6f3f..5999bfaa31 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/contributions/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/contributions/models.py @@ -168,7 +168,7 @@ def identifiers(self) -> dict[str, str]: """Returns this contribution's identifier values (see ``identifier_fields``). Overrides the base to narrow the value type to ``str`` for the callers (bulk error - reporting, ``upsert_contribution_by_identifiers``) that key on it. + reporting, ``upsert_one``) that key on it. """ return {"project": self.project, "identifier": self.identifier} diff --git a/mpcontribs-api/src/mpcontribs_api/domains/contributions/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/contributions/repository.py index 09eb3e5f31..f0ad3954c4 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/contributions/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/contributions/repository.py @@ -54,18 +54,6 @@ async def get_contributions( """Query the Contribution collection, scoped to the current user. See ``get_many``.""" return await self.get_many(pagination=pagination, filter=filter, fields=fields) - async def get_contribution_by_id(self, id: str, fields: frozenset[str] | None): - """Find a single contribution by id, scoped to the current user. See ``get_by_id``.""" - return await self.get_by_id(self._convert_object_id(id), fields) - - async def patch_contribution_by_id(self, id: str, update: ContributionPatch): - """Partially update a contribution by id, scoped to the current user. See ``patch``.""" - return await self.patch(self._convert_object_id(id), update) - - async def delete_contribution_by_id(self, id: str) -> None: - """Delete a contribution by id, scoped to the current user. See ``delete_by_id``.""" - await self.delete_by_id(self._convert_object_id(id)) - async def delete_contributions( self, filter: ContributionFilter, @@ -99,14 +87,6 @@ async def insert_contribution( await doc.insert(session=session) return doc - async def find_one_contribution(self, project: str, identifier: str) -> Contribution | None: - """Find a single contribution by (project, identifier), scoped to the current user.""" - return await self.document_model.find_one( - self._scope, - self.document_model.project == project, - self.document_model.identifier == identifier, - ) - async def max_versions(self, keys: list[tuple[str, str]]) -> dict[tuple[str, str], int]: """Return ``{(project, identifier): max_version}`` for the given keys, scoped to the user. @@ -226,62 +206,50 @@ async def update_contribution(self, doc: Contribution, update_data: dict[str, An """Apply a partial update to an existing Contribution document.""" await doc.update(Set(update_data)) - async def upsert_contribution_by_identifiers( + async def upsert_one( self, - identifiers: dict[str, str], + identifiers: dict[str, Any], contribution: ContributionIn, - version: int, + version: int | None = None, + session: AsyncClientSession | None = None, ) -> Contribution: - """Atomically upsert a Contribution by its identifying fields and resolved version. + """Atomically upsert a single Contribution addressed by ``identifiers``. - Relies on the unique index over (project, identifier, version) so that concurrent requests - targeting the same key cannot both win the insert branch. Fields the caller did not set are - not touched (partial update). On insert a fresh Contribution document is written with - ``is_public=False``. + Accepts either the bare ``{"id": ...}`` form (individual PUT) or the semantic + ``{"project", "identifier"}``, matching through the base + :meth:`MongoDbRepository._identifier_query`. When ``version`` is supplied it is stamped on + the document and folded into the semantic match, so the unique index over + (project, identifier, version) lets concurrent requests targeting the same key not both win + the insert branch. Fields the caller did not set are not touched (partial update); on insert + a fresh Contribution document is written with ``is_public=False``. Args: - identifiers: the fields ContributionIn.identifiers() returns (project, identifier) + identifiers: ``{"id": ...}`` or the ``{"project", "identifier"}`` pair contribution: the input payload to upsert - version: the version resolved by the service (1 for unique-identifier projects, or the - caller-supplied version for non-unique ones); selects which row to update + version: the version resolved by the service (selects which row to update); required for + the semantic form, ignored for the id form + session: optional client session for transactions Returns: Contribution: the document as it stands after the operation """ doc = self.document_model.from_input_model(contribution) - doc.version = version + match = identifiers + if version is not None: + doc.version = version + # The semantic (project, identifier) match must be pinned to the resolved version. + if "id" not in match: + match["version"] = version update_data = doc.model_dump(exclude={"id"}, exclude_none=True) query = self.document_model.find_one( self._scope, - self.document_model.project == identifiers["project"], - self.document_model.identifier == identifiers["identifier"], - self.document_model.version == version, + self._identifier_query(match), + session=session, ).upsert( Set(update_data), on_insert=doc, response_type=UpdateResponse.NEW_DOCUMENT, - ) - return await query # pyright: ignore[reportGeneralTypeIssues] # beanie UpdateQuery is awaitable, but pyright doesn't see it - - async def upsert_contribution_by_id(self, id: str, contribution: ContributionIn): - """Upserts a single Contribution. - - If Contributions with identical identifiers exist, update, otherwise insert - - Args: - id (str): the id of the Contribution to upsert - contribution (ContributionIn): the Contribution to be upserted - - Returns: - ContributionOut: the upserted document""" - doc = self.document_model.from_input_model(contribution) - query = self.document_model.find_one( - self._scope, - self.document_model.id == self._convert_object_id(id), - ).upsert( - Set(doc.model_dump(exclude={"id"}, exclude_none=True)), - on_insert=doc, - response_type=UpdateResponse.NEW_DOCUMENT, + session=session, ) return await query # pyright: ignore[reportGeneralTypeIssues] # beanie UpdateQuery is awaitable, but pyright doesn't see it diff --git a/mpcontribs-api/src/mpcontribs_api/domains/contributions/router.py b/mpcontribs-api/src/mpcontribs_api/domains/contributions/router.py index 73466eff28..6162aa82a2 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/contributions/router.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/contributions/router.py @@ -113,28 +113,28 @@ async def download_contributions( @router.delete("/{id}", dependencies=[Depends(require_user)]) -async def delete_contribution_by_id( +async def delete_one( service: ContributionServiceDep, id: str, ): - return await service.delete_contributions(ContributionFilter.model_validate({"id": id})) + return await service.delete_one({"id": id}) @router.get("/{id}") -async def get_contribution_by_id( - repo: ContributionDep, +async def get_one( + service: ContributionServiceDep, id: str, fields: FieldSelector = ContributionOut.default_fields(), ): selected = ContributionOut.parse_fields(fields) - return await repo.get_contribution_by_id(id=id, fields=selected) + return await service.get_one({"id": id}, fields=selected) @router.put("/{id}", dependencies=[Depends(require_user)]) -async def upsert_contribution_by_id(repo: ContributionDep, id: str, contribution: ContributionIn): - return await repo.upsert_contribution_by_id(id=id, contribution=contribution) +async def upsert_one(service: ContributionServiceDep, id: str, contribution: ContributionIn): + return await service.upsert_one({"id": id}, contribution=contribution) @router.patch("/{id}", dependencies=[Depends(require_user)]) -async def patch_contribution_by_id(repo: ContributionDep, id: str, update: ContributionPatch): - return await repo.patch_contribution_by_id(id=id, update=update) +async def patch_one(service: ContributionServiceDep, id: str, update: ContributionPatch): + return await service.patch_one({"id": id}, update=update) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/contributions/service.py b/mpcontribs-api/src/mpcontribs_api/domains/contributions/service.py index 88ba3a523f..8cd3c29e1a 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/contributions/service.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/contributions/service.py @@ -2,7 +2,7 @@ from collections import defaultdict from collections.abc import Iterable from dataclasses import dataclass -from typing import cast +from typing import Any, cast import structlog from beanie import Link, PydanticObjectId @@ -20,7 +20,13 @@ ) from mpcontribs_api.domains._shared.repository import MongoDbRepository from mpcontribs_api.domains.attachments.repository import MongoDbAttachmentRepository -from mpcontribs_api.domains.contributions.models import Contribution, ContributionFilter, ContributionIn +from mpcontribs_api.domains.contributions.models import ( + Contribution, + ContributionFilter, + ContributionIn, + ContributionOut, + ContributionPatch, +) from mpcontribs_api.domains.contributions.repository import MongoDbContributionRepository from mpcontribs_api.domains.projects.repository import MongoDbProjectRepository from mpcontribs_api.domains.structures.models import Structure @@ -75,6 +81,44 @@ def _children(self) -> dict[str, MongoDbRepository]: "tables": self._tables, } + async def get_one( + self, identifiers: dict[str, Any], fields: frozenset[str] | None + ) -> Contribution | ContributionOut | None: + """Return the single scoped contribution matching ``identifiers``. + + Accepts either the bare ``{"id": ...}`` form or the semantic + ``{"project", "identifier", "version"}`` set, resolved by the base ``_identifier_query``. + """ + return await self._contributions.get_one(self._contributions.coerce_identifiers(identifiers), fields) + + async def patch_one(self, identifiers: dict[str, Any], update: ContributionPatch) -> Contribution: + """Partially update the single scoped contribution matching ``identifiers``.""" + return await self._contributions.patch_one(self._contributions.coerce_identifiers(identifiers), update) + + async def upsert_one( + self, identifiers: dict[str, Any], contribution: ContributionIn, version: int | None = None + ) -> Contribution: + """Upsert the single scoped contribution matching ``identifiers``. See repository ``upsert_one``.""" + return await self._contributions.upsert_one(self._contributions.coerce_identifiers(identifiers), contribution, version) + + async def delete_one(self, identifiers: dict[str, Any]) -> BulkDeleteSummary: + """Delete a single contribution and its child components, matching ``identifiers``. + + Accepts either the bare ``{"id": ...}`` form or the semantic + ``{"project", "identifier", "version"}`` set. Cascades component deletion via + :meth:`delete_contributions` so children are never orphaned; a missing target is a zero-count + result (mirroring the bulk delete path, which does not 404). + """ + identifiers = self._contributions.coerce_identifiers(identifiers) + if set(identifiers) == {"id"}: + filter = ContributionFilter(id=identifiers["id"]) + else: + existing = await self._contributions.get_one(identifiers, frozenset({"id"})) + if existing is None: + return BulkDeleteSummary(num_deleted=0, num_children_deleted=0) + filter = ContributionFilter(id=existing.id) + return await self.delete_contributions(filter) + async def insert_contributions( self, contributions: list[ContributionIn], @@ -468,9 +512,7 @@ async def _bounded_upsert(item: ResolvedWrite) -> Contribution | BulkFailure: contrib = item.contribution async with sem: try: - return await self._contributions.upsert_contribution_by_identifiers( - contrib.identifiers(), contrib, item.version - ) + return await self._contributions.upsert_one(contrib.identifiers(), contrib, item.version) except Exception as exc: logger.error( "upsert_contribution_failed", index=item.index, identifier=contrib.identifiers(), exc_info=True diff --git a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py index 18381b5111..6c52d57304 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py @@ -1,6 +1,7 @@ from typing import Any from beanie import PydanticObjectId +from pymongo.asynchronous.client_session import AsyncClientSession from pymongo.errors import DuplicateKeyError from mpcontribs_api.authz import User @@ -50,10 +51,6 @@ async def get_initiatives( """Return a scoped, filtered, paginated page of initiatives. See ``get_many``.""" return await self.get_many(pagination=pagination, filter=filter, fields=fields) - async def get_initiative(self, slug: str, fields: frozenset[str] | None) -> InitiativeOut | None: - """Return the single scoped initiative identified by ``slug``. See ``get_one``.""" - return await self.get_one({"slug": slug}, fields) - async def resolve_visible(self, slug: str) -> Initiative | None: """Return the full scoped initiative document for ``slug`` (or None), for write-path checks.""" return await self.document_model.find_one(self._scope, self.document_model.slug == slug) @@ -94,14 +91,20 @@ async def insert_initiative(self, data: InitiativeIn) -> Initiative: raise ConflictError("an initiative with this slug already exists", slug=data.slug) from exc return initiative - async def patch_initiative(self, slug: str, update: InitiativePatch) -> Initiative: + async def patch_one( # pyright: ignore[reportIncompatibleMethodOverride] + self, identifiers: dict[str, Any], update: InitiativePatch, session: AsyncClientSession | None = None + ) -> Initiative: """Patch a scoped initiative by ``slug``, enforcing manage rights and approval rules. - The caller must be able to *manage* the initiative (owner/collaborator/admin) - Only an admin may change ``is_approved``. - The resulting state must satisfy ``is_public ⇒ is_approved`` (re-checked here because a partial ``$set`` does not run the document validator). + + ``identifiers`` is the ``{"slug": ...}`` form; the auth checks run against the resolved + document, and the write itself is delegated to the base :meth:`MongoDbRepository.patch_one`. """ + slug = identifiers["slug"] existing = await self.resolve_visible(slug) if existing is None: raise NotFoundError("Initiative not found", slug=slug) @@ -119,18 +122,24 @@ async def patch_initiative(self, slug: str, update: InitiativePatch) -> Initiati if resulting_public and not resulting_approved: raise ValidationError("an initiative cannot be public until it is approved", slug=slug) - return await self.patch(existing.id, update) + return await super().patch_one(identifiers, update, session=session) - async def delete_initiative(self, slug: str) -> DeleteResponse: + async def delete_one( + self, identifiers: dict[str, Any], session: AsyncClientSession | None = None + ) -> DeleteResponse: """Delete a scoped initiative by ``slug``. Restricted to the owner or an admin. Collaborators may contribute projects but may not dissolve the effort. Deleting an initiative does not touch member projects; their ``initiative`` link simply dangles until re-pointed (reads resolve a missing link to null). + + ``identifiers`` is the ``{"slug": ...}`` form; the write is delegated to the base + :meth:`MongoDbRepository.delete_one`. """ + slug = identifiers["slug"] existing = await self.resolve_visible(slug) if existing is None: raise NotFoundError("Initiative not found", slug=slug) if not (self._user.is_admin or existing.owner == self._user.username): raise PermissionError(required_role="owner-or-admin") - return await self.delete_by_id(existing.id) + return await super().delete_one(identifiers, session=session) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/router.py b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/router.py index 844818fd50..8a41815ae6 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/router.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/router.py @@ -30,14 +30,14 @@ async def get_initiatives( @router.get("/{slug}") -async def get_initiative( +async def get_one( repo: InitiativeDep, slug: str, fields: FieldSelector = InitiativeOut.default_fields(), ): """Return the single initiative identified by ``slug``, scoped to the caller.""" selected = InitiativeOut.parse_fields(fields) - return await repo.get_initiative(slug=slug, fields=selected) + return await repo.get_one({"slug": slug}, fields=selected) @router.post( @@ -56,7 +56,7 @@ async def insert_initiative( @router.patch("/{slug}", response_model=InitiativeOut, dependencies=[Depends(require_user)]) -async def patch_initiative( +async def patch_one( repo: InitiativeDep, slug: str, update: InitiativePatch, @@ -66,14 +66,14 @@ async def patch_initiative( Requires manage rights (owner/collaborator/admin). ``is_approved`` is admin-only, and an initiative cannot be made public until it is approved. """ - return await repo.patch_initiative(slug=slug, update=update) + return await repo.patch_one({"slug": slug}, update=update) @router.delete("/{slug}", status_code=status.HTTP_204_NO_CONTENT, dependencies=[Depends(require_user)]) -async def delete_initiative( +async def delete_one( repo: InitiativeDep, slug: str, ): """Delete the initiative identified by ``slug``. Restricted to its owner or an admin.""" - await repo.delete_initiative(slug=slug) + await repo.delete_one({"slug": slug}) return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py index faef61afcd..31ed1f129d 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/repository.py @@ -9,7 +9,7 @@ from mpcontribs_api.authz import User from mpcontribs_api.domains._shared.models import DeleteResponse from mpcontribs_api.domains._shared.repository import MongoDbRepository -from mpcontribs_api.domains._shared.types import PrefixedEmail, SearchStr, ShortStr +from mpcontribs_api.domains._shared.types import ShortStr from mpcontribs_api.domains.project_groups.models import ( ProjectGroup, ProjectGroupFilter, @@ -60,38 +60,26 @@ async def get_project_groups( """ return await self.get_many(pagination=pagination, filter=filter, fields=fields) - async def get_project_group( - self, - name: SearchStr, - owner: PrefixedEmail, - fields: frozenset[str] | None, - ) -> ProjectGroupOut | None: - """Return the single project group identified by ``name`` + ``owner``. See ``get_one``.""" - return await self.get_one({"name": name, "owner": owner}, fields) - async def insert_project_group(self, project_group: ProjectGroupIn) -> ProjectGroup: return await self.insert_one(in_resource=project_group) - async def patch_project_group( - self, name: SearchStr, owner: PrefixedEmail, update: ProjectGroupPatch - ) -> ProjectGroup: - """Patch the single project group identified by ``name`` + ``owner``. See ``patch_one``.""" - return await self.patch_one({"name": name, "owner": owner}, update) - - async def delete_project_group(self, name: SearchStr, owner: PrefixedEmail) -> DeleteResponse: - """Delete the single project group identified by ``name`` + ``owner``. + async def delete_one( + self, identifiers: dict[str, Any], session: AsyncClientSession | None = None + ) -> DeleteResponse: + """Delete the single project group matching ``identifiers`` (``{name, owner}`` or ``{id}``). Absence (in scope) takes precedence over the ownership gate: a non-admin may only delete their own group, so deleting another owner's visible (public) group is forbidden rather than silently a no-op. The ``name`` + ``owner`` unique index makes the match unambiguous. + The auth check runs against the resolved document; the write is delegated to the base + :meth:`MongoDbRepository.delete_one`. """ - doc = await self.document_model.find_one(self._scope, {"name": name, "owner": owner}) # pyright: ignore[reportArgumentType] + doc = await self.document_model.find_one(self._scope, self._identifier_query(identifiers), session=session) # pyright: ignore[reportArgumentType] if doc is None: - raise NotFoundError(f"{self.document_model.__name__} not found", name=name, owner=owner) - if not (self._user.is_admin or owner == self._user.username): + raise NotFoundError(f"{self.document_model.__name__} not found", **identifiers) + if not (self._user.is_admin or doc.owner == self._user.username): raise PermissionError(required_role="owner-or-admin") - await doc.delete() - return DeleteResponse(num_deleted=1) + return await super().delete_one(identifiers, session=session) async def delete_project_groups(self, filter: ProjectGroupFilter) -> DeleteResponse: """Bulk-delete project groups matching ``filter``, restricted to the caller's own. diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py index 70a783b526..5854180d92 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/router.py @@ -40,8 +40,8 @@ async def get_project_groups( @router.get("/item") -async def get_project_group( - repo: ProjectGroupDep, +async def get_one( + service: ProjectGroupServiceDep, name: SearchStr, owner: PrefixedEmail, fields: FieldSelector = ProjectGroupOut.default_fields(), @@ -49,13 +49,13 @@ async def get_project_group( """Return the single project group identified by ``name`` + ``owner``. Args: - repo (ProjectGroupDep): the project group repo we depend on + service (ProjectGroupServiceDep): the project group service we depend on name (SearchStr): the project group's name owner (PrefixedEmail): the project group's owner fields (FieldSelector): the fields to return to a user """ selected = ProjectGroupOut.parse_fields(fields) - return await repo.get_project_group(name=name, owner=owner, fields=selected) + return await service.get_one({"name": name, "owner": owner}, fields=selected) @router.post( @@ -78,8 +78,8 @@ async def insert_project_group( @router.patch("/item", response_model=ProjectGroupOut, dependencies=[Depends(require_user)]) -async def patch_project_group( - repo: ProjectGroupDep, +async def patch_one( + service: ProjectGroupServiceDep, name: SearchStr, owner: PrefixedEmail, update: ProjectGroupPatch, @@ -87,17 +87,17 @@ async def patch_project_group( """Partially update the project group identified by ``name`` + ``owner``. Args: - repo (ProjectGroupDep): the project group repo we depend on + service (ProjectGroupServiceDep): the project group service we depend on name (SearchStr): the project group's name owner (PrefixedEmail): the project group's owner update (ProjectGroupPatch): the partial update to apply - unset fields are dropped """ - return await repo.patch_project_group(name=name, owner=owner, update=update) + return await service.patch_one({"name": name, "owner": owner}, update=update) @router.delete("/item", status_code=status.HTTP_204_NO_CONTENT, dependencies=[Depends(require_user)]) -async def delete_project_group( - repo: ProjectGroupDep, +async def delete_one( + service: ProjectGroupServiceDep, name: SearchStr, owner: PrefixedEmail, ): @@ -106,11 +106,11 @@ async def delete_project_group( Raises 404 if no such group is visible to the caller, 409 if the identifiers are ambiguous. Args: - repo (ProjectGroupDep): the project group repo we depend on + service (ProjectGroupServiceDep): the project group service we depend on name (SearchStr): the project group's name owner (PrefixedEmail): the project group's owner """ - await repo.delete_project_group(name=name, owner=owner) + await service.delete_one({"name": name, "owner": owner}) return Response(status_code=status.HTTP_204_NO_CONTENT) @@ -140,7 +140,7 @@ async def add_projects_by_identifiers( Each project is verified against the projects collection (scoped to the caller); unknown or invisible projects are reported per-item in the response rather than failing the whole request. """ - return await service.add_projects_by_identifiers(name=name, owner=owner, project_ids=body.project_ids) + return await service.add_projects({"name": name, "owner": owner}, body.project_ids) @router.delete("/item/projects", response_model=BulkWriteSummary[str], dependencies=[Depends(require_user)]) @@ -154,7 +154,7 @@ async def delete_projects_by_identifiers( Ids that are not members of the group are reported per-item in the response. """ - return await service.delete_projects_by_identifiers(name=name, owner=owner, project_ids=body.project_ids) + return await service.delete_projects({"name": name, "owner": owner}, body.project_ids) @router.post("/{id}/projects", response_model=BulkWriteSummary[str], dependencies=[Depends(require_user)]) @@ -163,8 +163,8 @@ async def add_projects_by_id( id: str, body: ProjectRefs, ): - """Add projects to the group identified by ``id``. See ``add_projects_by_identifiers``.""" - return await service.add_projects_by_id(group_id=id, project_ids=body.project_ids) + """Add projects to the group identified by ``id``. See ``add_projects``.""" + return await service.add_projects({"id": id}, body.project_ids) @router.delete("/{id}/projects", response_model=BulkWriteSummary[str], dependencies=[Depends(require_user)]) @@ -173,5 +173,5 @@ async def delete_projects_by_id( id: str, body: ProjectRefs, ): - """Delete projects from the group identified by ``id``. See ``delete_projects_by_identifiers``.""" - return await service.delete_projects_by_id(group_id=id, project_ids=body.project_ids) + """Delete projects from the group identified by ``id``. See ``delete_projects``.""" + return await service.delete_projects({"id": id}, body.project_ids) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py index b882208aed..be62035cd5 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/project_groups/service.py @@ -1,8 +1,16 @@ +from typing import Any + from beanie import Link from mpcontribs_api.domains._shared.bulk import BulkFailure, BulkWriteSummary -from mpcontribs_api.domains._shared.types import PrefixedEmail, SearchStr, ShortStr -from mpcontribs_api.domains.project_groups.models import ProjectGroup, ProjectGroupIn, ProjectGroupOut +from mpcontribs_api.domains._shared.models import DeleteResponse +from mpcontribs_api.domains._shared.types import ShortStr +from mpcontribs_api.domains.project_groups.models import ( + ProjectGroup, + ProjectGroupIn, + ProjectGroupOut, + ProjectGroupPatch, +) from mpcontribs_api.domains.project_groups.repository import ProjectGroupRepository from mpcontribs_api.domains.projects.repository import MongoDbProjectRepository from mpcontribs_api.exceptions import NotFoundError @@ -25,7 +33,7 @@ def __init__( async def _project_exists(self, project_id: ShortStr) -> bool: """Whether a project with ``project_id`` exists and is visible to the caller.""" - return await self._projects.get_by_id(project_id, fields=frozenset({"id"})) is not None + return await self._projects.get_one({"id": project_id}, fields=frozenset({"id"})) is not None async def insert(self, project_group: ProjectGroupIn) -> ProjectGroup: """Insert a new group after verifying every referenced project exists and is visible. @@ -40,23 +48,30 @@ async def insert(self, project_group: ProjectGroupIn) -> ProjectGroup: raise NotFoundError("One or more projects not found or not visible", ids=missing) return await self._groups.insert_project_group(project_group) - async def _resolve_by_id(self, group_id: str) -> ProjectGroupOut: - """Resolve a visible group by its ObjectId, or raise ``NotFoundError``.""" - oid = self._groups._convert_object_id(group_id) - group = await self._groups.get_by_id(oid, fields=_GROUP_FIELDS) - if group is None: - raise NotFoundError("ProjectGroup not found", id=group_id) - return group # pyright: ignore[reportReturnType] # projected reads return the out model + async def get_one(self, identifiers: dict[str, Any], fields: frozenset[str] | None) -> ProjectGroupOut | None: + """Return the single group matching ``identifiers`` (``{"name", "owner"}`` or ``{"id"}``).""" + return await self._groups.get_one(identifiers, fields) + + async def patch_one(self, identifiers: dict[str, Any], update: ProjectGroupPatch) -> ProjectGroup: + """Patch the single group matching ``identifiers`` (``{"name", "owner"}`` or ``{"id"}``).""" + return await self._groups.patch_one(identifiers, update) - async def _resolve_by_identifiers(self, name: SearchStr, owner: PrefixedEmail) -> ProjectGroupOut: - """Resolve a visible group by its ``(name, owner)`` identifiers, or raise ``NotFoundError``. + async def delete_one(self, identifiers: dict[str, Any]) -> DeleteResponse: + """Delete the single group matching ``identifiers`` (``{"name", "owner"}`` or ``{"id"}``).""" + return await self._groups.delete_one(identifiers) - Propagates ``ConflictError`` from the repository if the identifiers are ambiguous. + async def _resolve_one(self, identifiers: dict[str, Any]) -> ProjectGroupOut: + """Resolve a visible group matching ``identifiers``, or raise ``NotFoundError``. + + ``identifiers`` is either the primary-key form ``{"id": }`` or the semantic + ``{"name": ..., "owner": ...}``. Propagates ``ConflictError`` from the repository if + ``(name, owner)`` identifiers are ambiguous. """ - group = await self._groups.get_one({"name": name, "owner": owner}, fields=_GROUP_FIELDS) + query = self._groups.coerce_identifiers(identifiers) + group = await self._groups.get_one(query, fields=_GROUP_FIELDS) if group is None: - raise NotFoundError("ProjectGroup not found", name=name, owner=owner) - return group + raise NotFoundError("ProjectGroup not found", **identifiers) + return group # pyright: ignore[reportReturnType] # projected reads return the out model async def _add(self, group: ProjectGroupOut, project_ids: list[ShortStr]) -> BulkWriteSummary[str]: """Validate each project against the projects collection, then add the valid ones. @@ -105,22 +120,10 @@ async def _delete(self, group: ProjectGroupOut, project_ids: list[ShortStr]) -> await self._groups.delete_project_refs(group.id, present) # pyright: ignore[reportArgumentType] # id is set on a resolved group return BulkWriteSummary(total=len(project_ids), succeeded=present, failed=failed) - async def add_projects_by_id(self, group_id: str, project_ids: list[ShortStr]) -> BulkWriteSummary[str]: - """Add projects to the group identified by ``group_id``.""" - return await self._add(await self._resolve_by_id(group_id), project_ids) - - async def add_projects_by_identifiers( - self, name: SearchStr, owner: PrefixedEmail, project_ids: list[ShortStr] - ) -> BulkWriteSummary[str]: - """Add projects to the group identified by ``(name, owner)``.""" - return await self._add(await self._resolve_by_identifiers(name, owner), project_ids) - - async def delete_projects_by_id(self, group_id: str, project_ids: list[ShortStr]) -> BulkWriteSummary[str]: - """Delete projects from the group identified by ``group_id``.""" - return await self._delete(await self._resolve_by_id(group_id), project_ids) - - async def delete_projects_by_identifiers( - self, name: SearchStr, owner: PrefixedEmail, project_ids: list[ShortStr] - ) -> BulkWriteSummary[str]: - """Delete projects from the group identified by ``(name, owner)``.""" - return await self._delete(await self._resolve_by_identifiers(name, owner), project_ids) + async def add_projects(self, identifiers: dict[str, Any], project_ids: list[ShortStr]) -> BulkWriteSummary[str]: + """Add projects to the group matching ``identifiers`` (``{"id": ...}`` or ``{"name", "owner"}``).""" + return await self._add(await self._resolve_one(identifiers), project_ids) + + async def delete_projects(self, identifiers: dict[str, Any], project_ids: list[ShortStr]) -> BulkWriteSummary[str]: + """Delete projects from the group matching ``identifiers`` (``{"id": ...}`` or ``{"name", "owner"}``).""" + return await self._delete(await self._resolve_one(identifiers), project_ids) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py index 222c7988e1..ec22f17d66 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py @@ -3,8 +3,10 @@ from beanie import PydanticObjectId, UpdateResponse from beanie.operators import Set from bson import DBRef +from pymongo.asynchronous.client_session import AsyncClientSession from mpcontribs_api.authz import User +from mpcontribs_api.domains._shared.models import DeleteResponse from mpcontribs_api.domains._shared.repository import MongoDbRepository from mpcontribs_api.domains.projects.models import ( Project, @@ -54,10 +56,6 @@ async def get_projects( """Query the Project collection, scoped to the current user. See ``get_many``.""" return await self.get_many(pagination=pagination, filter=filter, fields=fields) - async def get_project_by_id(self, id: str, fields: frozenset[str] | None) -> Project | ProjectOut | None: - """Find a single project by id, scoped to the current user. See ``get_by_id``.""" - return await self.get_by_id(id, fields) - async def unique_identifiers_by_id(self, ids: list[str]) -> dict[str, bool]: """Return ``{project_id: unique_identifiers}`` for the given project ids, scoped to the user. @@ -86,18 +84,20 @@ async def insert_project(self, project: ProjectIn) -> Project: """Insert a new project, rejecting a duplicate id. See ``insert_one``.""" return await self.insert_one(project) - async def patch_project_by_id(self, id: str, update: ProjectPatch) -> Project: + async def patch_one( # pyright: ignore[reportIncompatibleMethodOverride] + self, identifiers: dict[str, Any], update: ProjectPatch, session: AsyncClientSession | None = None + ) -> Project: """Partially update a scoped project by id, enforcing approval rules. - Only an admin may change ``is_approved``. - Resulting state must satisfy is_public <-> is_approved condition - The ``initiative`` field is split out upstream in ``ProjectService.patch``, so it never + The ``initiative`` field is split out upstream in ``ProjectService.patch_one``, so it never reaches this method; an assignment that also edits plain fields goes through :meth:`patch_project_with_initiative` instead. """ - await self._enforce_patch_rules(id, update) - return await self.patch(id, update) + await self._enforce_patch_rules(identifiers["id"], update) + return await super().patch_one(identifiers, update, session=session) async def patch_project_with_initiative(self, id: str, update: ProjectPatch, ref: DBRef | None) -> Project: """Atomically apply a partial project update together with its canonical initiative link. @@ -111,13 +111,13 @@ async def patch_project_with_initiative(self, id: str, update: ProjectPatch, ref await self._enforce_patch_rules(id, update) data = update.model_dump(exclude_unset=True) data["initiative"] = ref - query = self.document_model.find_one(self._scope, self.document_model.id == id).update( + query = self.document_model.find_one(self._scope, self._identifier_query({"id": id})).update( Set(data), response_type=UpdateResponse.NEW_DOCUMENT, ) updated = await query # pyright: ignore[reportGeneralTypeIssues] # beanie UpdateQuery is awaitable if updated is None: - raise NotFoundError(self._not_found(id)) + raise NotFoundError(f"{self.document_model.__name__} not found", id=id) return updated async def _enforce_patch_rules(self, id: str, update: ProjectPatch) -> None: @@ -135,28 +135,32 @@ async def _enforce_patch_rules(self, id: str, update: ProjectPatch) -> None: existing = await self.document_model.find_one(self._scope, self.document_model.id == id) if existing is None: - raise NotFoundError(self._not_found(id)) + raise NotFoundError(f"{self.document_model.__name__} not found", id=id) resulting_approved = data.get("is_approved", existing.is_approved) resulting_public = data.get("is_public", existing.is_public) if resulting_public and not resulting_approved: raise ValidationError("a project cannot be public until it is approved", id=id) - async def delete_project_by_id(self, id: str) -> None: + async def delete_one( + self, identifiers: dict[str, Any], session: AsyncClientSession | None = None + ) -> DeleteResponse: """Delete a scoped project by id. Restricted to the owner or an admin. Visibility (public/approved or group membership) is not enough to delete: a project can only be dissolved by its owner (or an admin). A caller who cannot see the project gets a - 404; a caller who can see it but does not own it gets a 403. + 404; a caller who can see it but does not own it gets a 403. The auth check runs against the + resolved document; the write is delegated to the base :meth:`MongoDbRepository.delete_one`. """ - existing = await self.document_model.find_one(self._scope, self.document_model.id == id) + id = identifiers["id"] + existing = await self.document_model.find_one(self._scope, self._identifier_query({"id": id})) if existing is None: - raise NotFoundError(self._not_found(id)) + raise NotFoundError(f"{self.document_model.__name__} not found", id=id) if not (self._user.is_admin or existing.owner == self._user.username): raise PermissionError(required_role="owner-or-admin") - await self.delete_by_id(id) + return await super().delete_one(identifiers, session=session) - async def upsert_project_by_id(self, id: str, data: ProjectIn) -> Project: + async def upsert_one(self, identifiers: dict[str, Any], data: ProjectIn) -> Project: """Upsert a project by provided id, authorized to the current user. Update the document if the id exists, otherwise insert a new one under that id. @@ -168,10 +172,10 @@ async def upsert_project_by_id(self, id: str, data: ProjectIn) -> Project: - **New project:** ``owner`` is forced to the caller; server-managed fields keep their defaults - Note: relies on the path param ``id`` for identity, not the body's id. + Note: relies on the identifier ``id`` for identity, not the body's id. Args: - id (str): the id of the project to upsert + identifiers (dict[str, Any]): the identifier of the project to upsert (``{"id": ...}``) data (ProjectIn): the data of the project to upsert Returns: @@ -184,6 +188,7 @@ async def upsert_project_by_id(self, id: str, data: ProjectIn) -> Project: if self._user.username is None: raise PermissionError(required_role="authenticated") + id = identifiers["id"] existing = await self.document_model.find_one(self.document_model.id == id) project = self.document_model.from_input_model(data) project.id = id diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/router.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/router.py index 56d453a948..82f3d9d506 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/router.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/router.py @@ -40,28 +40,28 @@ async def get_projects( @router.get("/{id}") -async def get_project_by_id( +async def get_one( id: str, - repo: ProjectDep, + service: ProjectServiceDep, fields: FieldSelector = ProjectOut.default_fields(), ): """Gets a single project by its ID. Args: id (str): the id of the project to retrieve - repo (ProjectDep): the project repo we depend on + service (ProjectServiceDep): the project service we depend on fields (str | None): optional fields to include in return. If None supplied, all fields are returned Returns: ProjectOut: the requested project, actual data returned is determined by the view the user requested """ selected = ProjectOut.parse_fields(fields) - return await repo.get_project_by_id(id=id, fields=selected) + return await service.get_one({"id": id}, fields=selected) @router.put("/{id}", response_model=ProjectOut, dependencies=[Depends(require_user)]) -async def upsert_project_by_id( - repo: ProjectDep, +async def upsert_one( + service: ProjectServiceDep, id: str, project: ProjectIn, ): @@ -71,18 +71,18 @@ async def upsert_project_by_id( Note: Relies on the path param 'id' for finding, rather than the body's id. Args: - repo (ProjectDep): the project repo we depend on + service (ProjectServiceDep): the project service we depend on id (str): the id of the project to retrieve project (ProjectIn): the data of the project to upsert Returns: ProjectOut: the full document that either replaced an old one or was inserted """ - return await repo.upsert_project_by_id(id=id, data=project) + return await service.upsert_one({"id": id}, data=project) @router.patch("/{id}", response_model=ProjectOut, dependencies=[Depends(require_user)]) -async def patch_project_by_id( +async def patch_one( service: ProjectServiceDep, id: str, update: ProjectPatch, @@ -105,21 +105,21 @@ async def patch_project_by_id( Returns: ProjectOut: the full Project with updates applied """ - return await service.patch(id=id, update=update) + return await service.patch_one({"id": id}, update=update) @router.delete("/{id}", status_code=status.HTTP_204_NO_CONTENT, dependencies=[Depends(require_user)]) -async def delete_project_by_id( - repo: ProjectDep, +async def delete_one( + service: ProjectServiceDep, id: str, ): """Deletes a project matching id. Args: - repo (ProjectDep): the project repo we depend on + service (ProjectServiceDep): the project service we depend on id (str): the id of the project to be deleted Returns: Response: a response with the 204 response code (rather than FastAPIs default 200) """ - await repo.delete_project_by_id(id=id) + await service.delete_one({"id": id}) return Response(status_code=HTTP_204_NO_CONTENT) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py index 2852aee49a..067bea3a3b 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py @@ -1,8 +1,11 @@ +from typing import Any + from bson import DBRef from mpcontribs_api.config import get_settings +from mpcontribs_api.domains._shared.models import DeleteResponse from mpcontribs_api.domains.initiatives.repository import InitiativeRepository -from mpcontribs_api.domains.projects.models import Project, ProjectPatch +from mpcontribs_api.domains.projects.models import Project, ProjectIn, ProjectOut, ProjectPatch from mpcontribs_api.domains.projects.repository import MongoDbProjectRepository from mpcontribs_api.exceptions import ConflictError, NotFoundError, PermissionError @@ -19,7 +22,7 @@ def __init__( self._initiatives = initiatives self._limits = get_settings().domain.initiatives - async def patch(self, id: str, update: ProjectPatch) -> Project: + async def patch_one(self, identifiers: dict[str, Any], update: ProjectPatch) -> Project: """Apply a project patch, routing an ``initiative`` change through the assignment checks. ``initiative`` carries the target initiative's ``slug`` (or ``null`` to unassign). When @@ -28,8 +31,9 @@ async def patch(self, id: str, update: ProjectPatch) -> Project: together with any co-submitted plain fields in a single atomic update, so the request never persists a half-applied change. """ + id = identifiers["id"] if "initiative" not in update.model_fields_set: - return await self._projects.patch_project_by_id(id=id, update=update) + return await self._projects.patch_one(identifiers, update) data = update.model_dump(exclude_unset=True) slug = data.pop("initiative", None) @@ -39,6 +43,18 @@ async def patch(self, id: str, update: ProjectPatch) -> Project: return await self._projects.patch_project_with_initiative(id=id, update=ProjectPatch(**data), ref=ref) + async def get_one(self, identifiers: dict[str, Any], fields: frozenset[str] | None) -> Project | ProjectOut | None: + """Return the single scoped project matching ``identifiers`` (``{"id": ...}``).""" + return await self._projects.get_one(identifiers, fields) + + async def upsert_one(self, identifiers: dict[str, Any], data: ProjectIn) -> Project: + """Upsert the project addressed by ``identifiers`` (``{"id": ...}``). See repository.""" + return await self._projects.upsert_one(identifiers, data) + + async def delete_one(self, identifiers: dict[str, Any]) -> DeleteResponse: + """Delete the scoped project addressed by ``identifiers`` (``{"id": ...}``). See repository.""" + return await self._projects.delete_one(identifiers) + async def _resolve_initiative_assignment(self, project_id: str, slug: str | None) -> DBRef | None: """Validate an initiative assignment and return the link to store (or None to unassign). diff --git a/mpcontribs-api/src/mpcontribs_api/domains/structures/router.py b/mpcontribs-api/src/mpcontribs_api/domains/structures/router.py index efa86a2e38..52ed017112 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/structures/router.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/structures/router.py @@ -31,14 +31,15 @@ async def get_structures( return await service.get_many(filter=filter, fields=selected, pagination=pagination) -@router.get("/{pk}") -async def get_structure( +@router.get("/{id}") +async def get_one( service: StructureServiceDep, - pk: str, + id: str, fields: FieldSelector = StructureOut.default_fields(), ): + """Return a single structure addressed by its ``_id`` or its content ``md5``.""" selected = StructureOut.parse_fields(fields) - return await service.get_by_id(id=pk, fields=selected) + return await service.get_one(identifiers={"id": id}, fields=selected) @router.get("/download/{short_mime}") @@ -82,14 +83,16 @@ async def delete_structures(service: StructureServiceDep, filter: StructureFilte @router.delete("/{id}", response_model=ComponentDeleteResponse, dependencies=[Depends(require_user)]) -async def delete_structure_by_id(service: StructureServiceDep, id: str): - return await service.delete_by_id(id=id) +async def delete_one(service: StructureServiceDep, id: str): + """Delete a single structure addressed by its ``_id`` or its content ``md5``.""" + return await service.delete_one(identifiers={"id": id}) @router.patch("/{id}", dependencies=[Depends(require_user)]) -async def patch_structure_by_id( +async def patch_one( service: StructureServiceDep, id: str, update: StructurePatch, ): - return await service.patch_by_id(id=id, update=update) + """Patch a single structure addressed by its ``_id`` or its content ``md5``.""" + return await service.patch_one(identifiers={"id": id}, update=update) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/tables/router.py b/mpcontribs-api/src/mpcontribs_api/domains/tables/router.py index 658a650575..e63ace859a 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/tables/router.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/tables/router.py @@ -31,14 +31,15 @@ async def get_tables( return await service.get_many(filter=filter, fields=selected, pagination=pagination) -@router.get("/{pk}") -async def get_table( +@router.get("/{id}") +async def get_one( service: TableServiceDep, - pk: str, + id: str, fields: FieldSelector = TableOut.default_fields(), ): + """Return a single table addressed by its ``_id`` or its content ``md5``.""" selected = TableOut.parse_fields(fields) - return await service.get_by_id(id=pk, fields=selected) + return await service.get_one(identifiers={"id": id}, fields=selected) @router.get("/download/{short_mime}") @@ -82,14 +83,16 @@ async def delete_tables(service: TableServiceDep, filter: TableFilter = FilterDe @router.delete("/{id}", response_model=ComponentDeleteResponse, dependencies=[Depends(require_user)]) -async def delete_table_by_id(service: TableServiceDep, id: str): - return await service.delete_by_id(id=id) +async def delete_one(service: TableServiceDep, id: str): + """Delete a single table addressed by its ``_id``""" + return await service.delete_one(identifiers={"id": id}) @router.patch("/{id}", dependencies=[Depends(require_user)]) -async def patch_table_by_id( +async def patch_one( service: TableServiceDep, id: str, update: TablePatch, ): - return await service.patch_by_id(id=id, update=update) + """Patch a single table addressed by its ``_id``.""" + return await service.patch_one(identifiers={"id": id}, update=update) diff --git a/mpcontribs-api/tests/integration/db/test_component_reachability.py b/mpcontribs-api/tests/integration/db/test_component_reachability.py index 0f7faf0885..c4d9ee6549 100644 --- a/mpcontribs-api/tests/integration/db/test_component_reachability.py +++ b/mpcontribs-api/tests/integration/db/test_component_reachability.py @@ -54,7 +54,7 @@ class TestComponentReadReachability: async def test_get_by_id_returns_reachable_component(self, db): att = await _attachment(1) await _contribution("mp-pub", is_public=True, attachments=[att]) - result = await _service(ANON).get_by_id(str(att.id), fields=None) + result = await _service(ANON).get_one({"id": str(att.id)}, fields=None) assert result is not None assert result.id == att.id @@ -62,13 +62,13 @@ async def test_get_by_id_hides_unreachable_component(self, db): att = await _attachment(2) # Referenced only by a private contribution -> anonymous cannot reach it. await _contribution("mp-priv", is_public=False, attachments=[att]) - result = await _service(ANON).get_by_id(str(att.id), fields=None) + result = await _service(ANON).get_one({"id": str(att.id)}, fields=None) assert result is None async def test_get_by_id_hides_orphan_component(self, db): # No contribution references this attachment at all. att = await _attachment(3) - result = await _service(ANON).get_by_id(str(att.id), fields=None) + result = await _service(ANON).get_one({"id": str(att.id)}, fields=None) assert result is None async def test_get_many_only_lists_reachable(self, db): diff --git a/mpcontribs-api/tests/integration/db/test_components_repository.py b/mpcontribs-api/tests/integration/db/test_components_repository.py index 8a3845aaf5..d48b025a0c 100644 --- a/mpcontribs-api/tests/integration/db/test_components_repository.py +++ b/mpcontribs-api/tests/integration/db/test_components_repository.py @@ -100,7 +100,7 @@ async def test_single_insert_persists(self, db): # --------------------------------------------------------------------------- -# delete_components / delete_component_by_id +# delete_components / delete_one # --------------------------------------------------------------------------- @@ -113,9 +113,9 @@ async def test_filtered_delete_removes_only_matches(self, db): assert remaining == {keep.md5} async def test_delete_by_id_removes_one(self, db): - """delete_component_by_id matches a string id by converting it to ObjectId.""" + """The inherited base delete_one removes a single component by its primary key.""" [doc] = await _repo().insert_components([_attachment(1)]) - result = await _repo().delete_component_by_id(str(doc.id)) + result = await _repo().delete_one({"id": doc.id}) assert result.num_deleted == 1 assert await _count() == 0 @@ -123,32 +123,58 @@ async def test_delete_by_unknown_id_raises(self, db): from mpcontribs_api.exceptions import NotFoundError with pytest.raises(NotFoundError): - await _repo().delete_component_by_id(str(PydanticObjectId())) + await _repo().delete_one({"id": PydanticObjectId()}) + + async def test_delete_by_md5_removes_one(self, db): + """A component is addressable by its content md5 (its declared identifier) as well as by id.""" + [doc] = await _repo().insert_components([_attachment(1)]) + result = await _repo().delete_one({"md5": doc.md5}) + assert result.num_deleted == 1 + assert await _count() == 0 + + +# --------------------------------------------------------------------------- +# get_one / patch_one address a component by id or by its content md5 +# --------------------------------------------------------------------------- + + +class TestAddressComponentByMd5: + async def test_get_one_by_md5(self, db): + [doc] = await _repo().insert_components([_attachment(1)]) + by_md5 = await _repo().get_one({"md5": doc.md5}, fields=None) + by_id = await _repo().get_one({"id": doc.id}, fields=None) + assert by_md5 is not None + assert by_md5.id == by_id.id == doc.id + + async def test_patch_one_by_md5(self, db): + [doc] = await _repo().insert_components([_attachment(1, name="data.csv")]) + updated = await _repo().patch_one({"md5": doc.md5}, AttachmentPatch(name="renamed.png")) + assert updated.name == "renamed.png" # --------------------------------------------------------------------------- -# patch_component_by_id +# patch_one recomputes the derived md5 # --------------------------------------------------------------------------- class TestPatchComponent: async def test_patch_updates_field(self, db): [doc] = await _repo().insert_components([_attachment(1, name="data.csv")]) - updated = await _repo().patch_component_by_id(str(doc.id), AttachmentPatch(name="renamed.png")) + updated = await _repo().patch_one({"id": doc.id}, AttachmentPatch(name="renamed.png")) assert updated.name == "renamed.png" async def test_empty_patch_returns_existing(self, db): [doc] = await _repo().insert_components([_attachment(1, name="data.csv")]) - updated = await _repo().patch_component_by_id(str(doc.id), AttachmentPatch()) + updated = await _repo().patch_one({"id": doc.id}, AttachmentPatch()) assert updated.id == doc.id async def test_patch_content_recomputes_md5(self, db): # name is not a hash field, so renaming must NOT change md5. [doc] = await _repo().insert_components([_attachment(1)]) - renamed = await _repo().patch_component_by_id(str(doc.id), AttachmentPatch(name="renamed.png")) + renamed = await _repo().patch_one({"id": doc.id}, AttachmentPatch(name="renamed.png")) assert renamed.md5 == doc.md5 # content IS a hash field, so changing it must recompute md5. - rehashed = await _repo().patch_component_by_id(str(doc.id), AttachmentPatch(content=999)) + rehashed = await _repo().patch_one({"id": doc.id}, AttachmentPatch(content=999)) assert rehashed.md5 != doc.md5 persisted = await Attachment.find_one(Attachment.id == doc.id) assert persisted.md5 == rehashed.md5 @@ -215,7 +241,7 @@ async def test_table_frame_round_trips_via_storage_shape(self, db): assert raw["total_data_rows"] == 2 # Read back: reassembled into the same DataFrame (index folded back as the first column). - out = await repo.get_component_by_id(str(doc.id), TableOut.parse_fields(["data"])) + out = await repo.get_one({"id": doc.id}, TableOut.parse_fields(["data"])) assert out.data.columns == ["T [K]", "1e16", "1e17"] assert out.data.equals(frame) # The raw storage keys must not leak onto the response model. diff --git a/mpcontribs-api/tests/integration/db/test_contributions_repository.py b/mpcontribs-api/tests/integration/db/test_contributions_repository.py index 4d3cd97c89..bfc60f4977 100644 --- a/mpcontribs-api/tests/integration/db/test_contributions_repository.py +++ b/mpcontribs-api/tests/integration/db/test_contributions_repository.py @@ -264,84 +264,96 @@ async def test_filter_by_needs_build(self, db): # --------------------------------------------------------------------------- -# get_contribution_by_id +# get_one (by id) # --------------------------------------------------------------------------- class TestGetContributionById: async def test_returns_doc_for_valid_id(self, db): doc = await _insert(identifier="get-id") - result = await _repo(ADMIN).get_contribution_by_id(str(doc.id), fields=None) + result = await _repo(ADMIN).get_one({"id": doc.id}, fields=None) assert result is not None assert result.identifier == "get-id" async def test_returns_none_for_missing_id(self, db): - result = await _repo(ADMIN).get_contribution_by_id(str(PydanticObjectId()), fields=None) + result = await _repo(ADMIN).get_one({"id": PydanticObjectId()}, fields=None) assert result is None async def test_admin_can_get_private_doc(self, db): doc = await _insert(identifier="get-priv", is_public=False) - result = await _repo(ADMIN).get_contribution_by_id(str(doc.id), fields=None) + result = await _repo(ADMIN).get_one({"id": doc.id}, fields=None) assert result is not None async def test_anon_cannot_get_private_doc(self, db): doc = await _insert(identifier="get-anon-priv", is_public=False) - result = await _repo(ANON).get_contribution_by_id(str(doc.id), fields=None) + result = await _repo(ANON).get_one({"id": doc.id}, fields=None) assert result is None async def test_anon_can_get_public_doc(self, db): doc = await _insert(identifier="get-anon-pub", is_public=True) - result = await _repo(ANON).get_contribution_by_id(str(doc.id), fields=None) + result = await _repo(ANON).get_one({"id": doc.id}, fields=None) assert result is not None - async def test_raises_validation_error_for_bad_id_format(self, db): - with pytest.raises(ValidationError): - await _repo(ADMIN).get_contribution_by_id("not-an-objectid", fields=None) - async def test_projection_limits_fields(self, db): doc = await _insert(identifier="get-proj", is_public=True) fields = ContributionOut.parse_fields(["formula"]) - result = await _repo(ADMIN).get_contribution_by_id(str(doc.id), fields=fields) + result = await _repo(ADMIN).get_one({"id": doc.id}, fields=fields) assert result is not None assert result.formula == "Fe2O3" assert not hasattr(result, "data") # --------------------------------------------------------------------------- -# find_one_contribution (by project + identifier) +# get_one (by the semantic {project, identifier, version} triple) # --------------------------------------------------------------------------- -class TestFindOneContribution: +class TestGetContributionBySemanticIdentifiers: async def test_finds_existing_doc(self, db): await _insert(project="find-proj", identifier="find-id") - result = await _repo(ADMIN).find_one_contribution("find-proj", "find-id") + result = await _repo(ADMIN).get_one( + {"project": "find-proj", "identifier": "find-id", "version": 1}, fields=None + ) assert result is not None assert result.project == "find-proj" assert result.identifier == "find-id" async def test_returns_none_for_missing_combination(self, db): await _insert(project="miss-proj", identifier="miss-id") - result = await _repo(ADMIN).find_one_contribution("miss-proj", "wrong-id") + result = await _repo(ADMIN).get_one( + {"project": "miss-proj", "identifier": "wrong-id", "version": 1}, fields=None + ) assert result is None async def test_scope_prevents_anon_finding_private(self, db): await _insert(project="anon-scope", identifier="priv-doc", is_public=False) - result = await _repo(ANON).find_one_contribution("anon-scope", "priv-doc") + result = await _repo(ANON).get_one( + {"project": "anon-scope", "identifier": "priv-doc", "version": 1}, fields=None + ) assert result is None async def test_scope_allows_anon_finding_public(self, db): await _insert(project="anon-scope-pub", identifier="pub-doc", is_public=True) - result = await _repo(ANON).find_one_contribution("anon-scope-pub", "pub-doc") + result = await _repo(ANON).get_one( + {"project": "anon-scope-pub", "identifier": "pub-doc", "version": 1}, fields=None + ) assert result is not None - async def test_project_identifier_combination_is_unique_lookup(self, db): + async def test_project_identifier_version_is_unique_lookup(self, db): await _insert(project="same-proj", identifier="id-a") await _insert(project="same-proj", identifier="id-b") - result = await _repo(ADMIN).find_one_contribution("same-proj", "id-a") + result = await _repo(ADMIN).get_one( + {"project": "same-proj", "identifier": "id-a", "version": 1}, fields=None + ) assert result is not None assert result.identifier == "id-a" + async def test_partial_identifier_set_is_rejected(self, db): + # The semantic set must be complete: version is part of the contribution's identity. + await _insert(project="partial-proj", identifier="partial-id") + with pytest.raises(ValidationError): + await _repo(ADMIN).get_one({"project": "partial-proj", "identifier": "partial-id"}, fields=None) + # --------------------------------------------------------------------------- # update_contribution @@ -379,65 +391,72 @@ def _naive(dt): # --------------------------------------------------------------------------- -# patch_contribution_by_id +# patch_one (by id) # --------------------------------------------------------------------------- class TestPatchContributionById: async def test_updates_formula(self, db): doc = await _insert(identifier="patch-formula") - await _repo(ADMIN).patch_contribution_by_id(str(doc.id), ContributionPatch(formula="Li2O")) + await _repo(ADMIN).patch_one({"id": doc.id}, ContributionPatch(formula="Li2O")) found = await Contribution.find_one(Contribution.id == doc.id) assert found.formula == "Li2O" async def test_unset_fields_not_overwritten(self, db): doc = await _insert(identifier="patch-preserve", formula="Fe2O3") - await _repo(ADMIN).patch_contribution_by_id(str(doc.id), ContributionPatch(needs_build=False)) + await _repo(ADMIN).patch_one({"id": doc.id}, ContributionPatch(needs_build=False)) found = await Contribution.find_one(Contribution.id == doc.id) assert found.formula == "Fe2O3" async def test_empty_patch_is_a_noop(self, db): doc = await _insert(identifier="patch-empty", formula="Fe2O3") - result = await _repo(ADMIN).patch_contribution_by_id(str(doc.id), ContributionPatch()) + result = await _repo(ADMIN).patch_one({"id": doc.id}, ContributionPatch()) assert result is not None found = await Contribution.find_one(Contribution.id == doc.id) assert found.formula == "Fe2O3" - async def test_raises_validation_error_for_bad_id(self, db): - with pytest.raises(ValidationError): - await _repo(ADMIN).patch_contribution_by_id("bad-id", ContributionPatch(formula="X")) + async def test_patch_by_semantic_identifiers(self, db): + # The same patch reachable through the full {project, identifier, version} identity. + await _insert(project="patch-sem", identifier="sem-id", formula="Fe2O3") + await _repo(ADMIN).patch_one( + {"project": "patch-sem", "identifier": "sem-id", "version": 1}, ContributionPatch(formula="Li2O") + ) + found = await Contribution.find_one(Contribution.project == "patch-sem") + assert found.formula == "Li2O" async def test_anon_cannot_patch_private_doc(self, db): from mpcontribs_api.exceptions import NotFoundError doc = await _insert(identifier="patch-anon-priv", is_public=False) with pytest.raises(NotFoundError): - await _repo(ANON).patch_contribution_by_id(str(doc.id), ContributionPatch(formula="X")) + await _repo(ANON).patch_one({"id": doc.id}, ContributionPatch(formula="X")) # --------------------------------------------------------------------------- -# delete_contribution_by_id +# delete_one (by id) # --------------------------------------------------------------------------- class TestDeleteContributionById: async def test_deleted_doc_not_found_afterwards(self, db): doc = await _insert(identifier="del-me") - await _repo(ADMIN).delete_contribution_by_id(str(doc.id)) + await _repo(ADMIN).delete_one({"id": doc.id}) found = await Contribution.find_one(Contribution.id == doc.id) assert found is None async def test_delete_nonexistent_throws_error(self, db): with pytest.raises(NotFoundError, match="not found"): - await _repo(ADMIN).delete_contribution_by_id(str(PydanticObjectId())) + await _repo(ADMIN).delete_one({"id": PydanticObjectId()}) - async def test_raises_validation_error_for_bad_id(self, db): - with pytest.raises(ValidationError): - await _repo(ADMIN).delete_contribution_by_id("not-an-id") + async def test_delete_by_semantic_identifiers(self, db): + await _insert(project="del-sem", identifier="sem-id") + await _repo(ADMIN).delete_one({"project": "del-sem", "identifier": "sem-id", "version": 1}) + found = await Contribution.find_one(Contribution.project == "del-sem") + assert found is None async def test_anon_cannot_delete_private_doc(self, db): doc = await _insert(identifier="del-anon-priv", is_public=False) with pytest.raises(NotFoundError, match="not found"): - await _repo(ANON).delete_contribution_by_id(str(doc.id)) + await _repo(ANON).delete_one({"id": doc.id}) # Scope prevents anonymous from seeing the doc, so it is never deleted. still_there = await Contribution.find_one(Contribution.id == doc.id) assert still_there is not None @@ -482,7 +501,7 @@ class TestUpsertContributionById: async def test_insert_when_id_absent_persists_document(self, db): new_id = PydanticObjectId() payload = _contrib_in(identifier="ups-new", _id=new_id) - result = await _repo(ADMIN).upsert_contribution_by_id(str(new_id), payload) + result = await _repo(ADMIN).upsert_one({"id": new_id}, payload) # Must be the resolved document, not an un-awaited query object. assert isinstance(result, Contribution) stored = await Contribution.find_one(Contribution.id == new_id) @@ -492,7 +511,19 @@ async def test_insert_when_id_absent_persists_document(self, db): async def test_update_when_id_present_applies_change(self, db): existing = await _insert(identifier="ups-existing") payload = _contrib_in(identifier="ups-existing", formula="Li2O", _id=existing.id) - result = await _repo(ADMIN).upsert_contribution_by_id(str(existing.id), payload) + result = await _repo(ADMIN).upsert_one({"id": existing.id}, payload) + assert isinstance(result, Contribution) + stored = await Contribution.find_one(Contribution.id == existing.id) + assert stored is not None + assert stored.formula == "Li2O" + + async def test_upsert_by_semantic_identifiers_with_version(self, db): + # The bulk-upsert path keys on {project, identifier} + resolved version. + existing = await _insert(project="ups-sem", identifier="sem-id", formula="Fe2O3") + payload = _contrib_in(project="ups-sem", identifier="sem-id", formula="Li2O") + result = await _repo(ADMIN).upsert_one( + {"project": "ups-sem", "identifier": "sem-id"}, payload, version=1 + ) assert isinstance(result, Contribution) stored = await Contribution.find_one(Contribution.id == existing.id) assert stored is not None diff --git a/mpcontribs-api/tests/integration/db/test_initiatives_repository.py b/mpcontribs-api/tests/integration/db/test_initiatives_repository.py index c4aec88ccd..5881002c02 100644 --- a/mpcontribs-api/tests/integration/db/test_initiatives_repository.py +++ b/mpcontribs-api/tests/integration/db/test_initiatives_repository.py @@ -43,7 +43,7 @@ async def _insert(slug: str, owner_user: User = ALICE, name: str = "An Initiativ async def _approve(slug: str) -> Initiative: - return await _repo(ADMIN).patch_initiative(slug, InitiativePatch(is_approved=True)) + return await _repo(ADMIN).patch_one({"slug": slug}, InitiativePatch(is_approved=True)) # --------------------------------------------------------------------------- @@ -103,26 +103,24 @@ class TestApprovalAndPublic: async def test_only_admin_may_approve(self, db): await _insert("approve-me", ALICE) with pytest.raises(PermissionError): - await _repo(ALICE).patch_initiative("approve-me", InitiativePatch(is_approved=True)) + await _repo(ALICE).patch_one({"slug": "approve-me"}, InitiativePatch(is_approved=True)) approved = await _approve("approve-me") assert approved.is_approved is True async def test_cannot_make_public_while_unapproved(self, db): await _insert("public-fail", ALICE) with pytest.raises(ValidationError): - await _repo(ALICE).patch_initiative("public-fail", InitiativePatch(is_public=True)) + await _repo(ALICE).patch_one({"slug": "public-fail"}, InitiativePatch(is_public=True)) async def test_public_allowed_once_approved(self, db): await _insert("public-ok", ALICE) await _approve("public-ok") - patched = await _repo(ALICE).patch_initiative("public-ok", InitiativePatch(is_public=True)) + patched = await _repo(ALICE).patch_one({"slug": "public-ok"}, InitiativePatch(is_public=True)) assert patched.is_public is True async def test_admin_can_approve_and_publish_together(self, db): await _insert("publish-both", ALICE) - patched = await _repo(ADMIN).patch_initiative( - "publish-both", InitiativePatch(is_approved=True, is_public=True) - ) + patched = await _repo(ADMIN).patch_one({"slug": "publish-both"}, InitiativePatch(is_approved=True, is_public=True)) assert patched.is_approved is True and patched.is_public is True @@ -134,38 +132,36 @@ async def test_admin_can_approve_and_publish_together(self, db): class TestManageAndScope: async def test_owner_can_rename(self, db): await _insert("rename-me", ALICE) - patched = await _repo(ALICE).patch_initiative("rename-me", InitiativePatch(name="Renamed")) + patched = await _repo(ALICE).patch_one({"slug": "rename-me"}, InitiativePatch(name="Renamed")) assert patched.name == "Renamed" async def test_collaborator_can_patch(self, db): await _insert("collab-patch", ALICE) - patched = await _repo(_collaborator("collab-patch")).patch_initiative( - "collab-patch", InitiativePatch(name="By Collaborator") - ) + patched = await _repo(_collaborator("collab-patch")).patch_one({"slug": "collab-patch"}, InitiativePatch(name="By Collaborator")) assert patched.name == "By Collaborator" async def test_visible_but_unmanaged_cannot_patch(self, db): # An approved+public initiative is visible to everyone, but a stranger still cannot manage it. await _insert("visible-public", ALICE) await _approve("visible-public") - await _repo(ALICE).patch_initiative("visible-public", InitiativePatch(is_public=True)) + await _repo(ALICE).patch_one({"slug": "visible-public"}, InitiativePatch(is_public=True)) stranger = User(username="google:carol@example.com", groups=frozenset()) with pytest.raises(PermissionError): - await _repo(stranger).patch_initiative("visible-public", InitiativePatch(name="hijack")) + await _repo(stranger).patch_one({"slug": "visible-public"}, InitiativePatch(name="hijack")) async def test_private_unapproved_scope(self, db): await _insert("scoped-priv", ALICE) - assert await _repo(ALICE).get_initiative("scoped-priv", fields=None) is not None # owner - assert await _repo(ADMIN).get_initiative("scoped-priv", fields=None) is not None # admin - assert await _repo(_collaborator("scoped-priv")).get_initiative("scoped-priv", fields=None) is not None - assert await _repo(ANON).get_initiative("scoped-priv", fields=None) is None # anon - assert await _repo(BOB).get_initiative("scoped-priv", fields=None) is None # unrelated user + assert await _repo(ALICE).get_one({"slug": "scoped-priv"}, fields=None) is not None # owner + assert await _repo(ADMIN).get_one({"slug": "scoped-priv"}, fields=None) is not None # admin + assert await _repo(_collaborator("scoped-priv")).get_one({"slug": "scoped-priv"}, fields=None) is not None + assert await _repo(ANON).get_one({"slug": "scoped-priv"}, fields=None) is None # anon + assert await _repo(BOB).get_one({"slug": "scoped-priv"}, fields=None) is None # unrelated user async def test_public_approved_visible_to_anon(self, db): await _insert("scoped-pub", ALICE) await _approve("scoped-pub") - await _repo(ALICE).patch_initiative("scoped-pub", InitiativePatch(is_public=True)) - assert await _repo(ANON).get_initiative("scoped-pub", fields=None) is not None + await _repo(ALICE).patch_one({"slug": "scoped-pub"}, InitiativePatch(is_public=True)) + assert await _repo(ANON).get_one({"slug": "scoped-pub"}, fields=None) is not None # --------------------------------------------------------------------------- @@ -176,18 +172,18 @@ async def test_public_approved_visible_to_anon(self, db): class TestDelete: async def test_owner_can_delete(self, db): await _insert("del-owner", ALICE) - result = await _repo(ALICE).delete_initiative("del-owner") + result = await _repo(ALICE).delete_one({"slug": "del-owner"}) assert result.num_deleted == 1 - assert await _repo(ADMIN).get_initiative("del-owner", fields=None) is None + assert await _repo(ADMIN).get_one({"slug": "del-owner"}, fields=None) is None async def test_collaborator_cannot_delete(self, db): await _insert("del-collab", ALICE) with pytest.raises(PermissionError): - await _repo(_collaborator("del-collab")).delete_initiative("del-collab") + await _repo(_collaborator("del-collab")).delete_one({"slug": "del-collab"}) async def test_missing_is_not_found(self, db): with pytest.raises(NotFoundError): - await _repo(ADMIN).delete_initiative("nope-missing") + await _repo(ADMIN).delete_one({"slug": "nope-missing"}) # --------------------------------------------------------------------------- @@ -228,10 +224,10 @@ async def test_filter_by_owner(self, db): class TestAdminBypass: async def test_admin_can_patch_non_owned(self, db): await _insert("admin-patch", ALICE) - patched = await _repo(ADMIN).patch_initiative("admin-patch", InitiativePatch(name="Admin Renamed")) + patched = await _repo(ADMIN).patch_one({"slug": "admin-patch"}, InitiativePatch(name="Admin Renamed")) assert patched.name == "Admin Renamed" async def test_admin_can_delete_non_owned(self, db): await _insert("admin-del", ALICE) - result = await _repo(ADMIN).delete_initiative("admin-del") + result = await _repo(ADMIN).delete_one({"slug": "admin-del"}) assert result.num_deleted == 1 diff --git a/mpcontribs-api/tests/integration/db/test_initiatives_service.py b/mpcontribs-api/tests/integration/db/test_initiatives_service.py index 061f8acad0..4e16fd0dbd 100644 --- a/mpcontribs-api/tests/integration/db/test_initiatives_service.py +++ b/mpcontribs-api/tests/integration/db/test_initiatives_service.py @@ -71,30 +71,30 @@ class TestAssign: async def test_owner_of_both_can_assign(self, db): await _insert_project("proj-a", owner=ALICE_EMAIL) init = await _insert_initiative("init-a", ALICE) - updated = await _service(ALICE).patch("proj-a", ProjectPatch(initiative="init-a")) + updated = await _service(ALICE).patch_one({"id": "proj-a"}, ProjectPatch(initiative="init-a")) assert _assigned_id(updated) == init.id async def test_collaborator_can_assign_own_project(self, db): await _insert_project("proj-b", owner=BOB_EMAIL) init = await _insert_initiative("init-collab", ALICE) bob = _collaborator("init-collab") - updated = await _service(bob).patch("proj-b", ProjectPatch(initiative="init-collab")) + updated = await _service(bob).patch_one({"id": "proj-b"}, ProjectPatch(initiative="init-collab")) assert _assigned_id(updated) == init.id async def test_plain_patch_passes_through_untouched(self, db): await _insert_project("proj-plain", owner=ALICE_EMAIL) init = await _insert_initiative("init-plain", ALICE) - await _service(ALICE).patch("proj-plain", ProjectPatch(initiative="init-plain")) + await _service(ALICE).patch_one({"id": "proj-plain"}, ProjectPatch(initiative="init-plain")) # A patch that does not mention `initiative` must not disturb the existing assignment. - updated = await _service(ALICE).patch("proj-plain", ProjectPatch(title="new-title")) + updated = await _service(ALICE).patch_one({"id": "proj-plain"}, ProjectPatch(title="new-title")) assert updated.title == "new-title" assert _assigned_id(updated) == init.id async def test_unassign_clears_link(self, db): await _insert_project("proj-un", owner=ALICE_EMAIL) await _insert_initiative("init-un", ALICE) - await _service(ALICE).patch("proj-un", ProjectPatch(initiative="init-un")) - updated = await _service(ALICE).patch("proj-un", ProjectPatch(initiative=None)) + await _service(ALICE).patch_one({"id": "proj-un"}, ProjectPatch(initiative="init-un")) + updated = await _service(ALICE).patch_one({"id": "proj-un"}, ProjectPatch(initiative=None)) assert _assigned_id(updated) is None @@ -109,30 +109,30 @@ async def test_visible_but_unmanaged_initiative_rejected(self, db): # but she neither owns nor collaborates on it, so she still cannot assign to it. await _insert_project("proj-c", owner=CAROL_EMAIL) await _insert_initiative("init-c", ALICE) - await InitiativeRepository(ADMIN).patch_initiative( - "init-c", InitiativePatch(is_approved=True, is_public=True) + await InitiativeRepository(ADMIN).patch_one( + {"slug": "init-c"}, InitiativePatch(is_approved=True, is_public=True) ) with pytest.raises(PermissionError): - await _service(CAROL).patch("proj-c", ProjectPatch(initiative="init-c")) + await _service(CAROL).patch_one({"id": "proj-c"}, ProjectPatch(initiative="init-c")) async def test_invisible_initiative_is_not_found(self, db): # Alice's private initiative is invisible to Carol, so it reads as not-found (not a 403). await _insert_project("proj-c2", owner=CAROL_EMAIL) await _insert_initiative("init-priv", ALICE) with pytest.raises(NotFoundError): - await _service(CAROL).patch("proj-c2", ProjectPatch(initiative="init-priv")) + await _service(CAROL).patch_one({"id": "proj-c2"}, ProjectPatch(initiative="init-priv")) async def test_manager_without_project_write_rejected(self, db): # Alice manages the initiative but cannot see/write Bob's private project. await _insert_project("proj-bob", owner=BOB_EMAIL) await _insert_initiative("init-d", ALICE) with pytest.raises(NotFoundError): - await _service(ALICE).patch("proj-bob", ProjectPatch(initiative="init-d")) + await _service(ALICE).patch_one({"id": "proj-bob"}, ProjectPatch(initiative="init-d")) async def test_assign_to_missing_initiative_is_not_found(self, db): await _insert_project("proj-ghost", owner=ALICE_EMAIL) with pytest.raises(NotFoundError): - await _service(ALICE).patch("proj-ghost", ProjectPatch(initiative="ghost-init")) + await _service(ALICE).patch_one({"id": "proj-ghost"}, ProjectPatch(initiative="ghost-init")) # --------------------------------------------------------------------------- @@ -146,28 +146,28 @@ async def test_unapproved_capped_at_configured_members(self, db): await _insert_initiative("init-cap", ALICE) for i in range(cap): await _insert_project(f"cap-proj-{i}", owner=ALICE_EMAIL) - await _service(ALICE).patch(f"cap-proj-{i}", ProjectPatch(initiative="init-cap")) + await _service(ALICE).patch_one({"id": f"cap-proj-{i}"}, ProjectPatch(initiative="init-cap")) await _insert_project("cap-proj-over", owner=ALICE_EMAIL) with pytest.raises(ConflictError): - await _service(ALICE).patch("cap-proj-over", ProjectPatch(initiative="init-cap")) + await _service(ALICE).patch_one({"id": "cap-proj-over"}, ProjectPatch(initiative="init-cap")) async def test_reassigning_existing_member_is_idempotent(self, db): cap = get_settings().domain.initiatives.max_projects_per_unapproved await _insert_initiative("init-idem", ALICE) for i in range(cap): await _insert_project(f"idem-proj-{i}", owner=ALICE_EMAIL) - await _service(ALICE).patch(f"idem-proj-{i}", ProjectPatch(initiative="init-idem")) + await _service(ALICE).patch_one({"id": f"idem-proj-{i}"}, ProjectPatch(initiative="init-idem")) # At the cap, re-assigning a project that is already a member must not trip the limit. - again = await _service(ALICE).patch("idem-proj-0", ProjectPatch(initiative="init-idem")) + again = await _service(ALICE).patch_one({"id": "idem-proj-0"}, ProjectPatch(initiative="init-idem")) assert again.initiative is not None async def test_approved_initiative_has_no_member_cap(self, db): cap = get_settings().domain.initiatives.max_projects_per_unapproved await _insert_initiative("init-approved", ALICE) - await InitiativeRepository(ADMIN).patch_initiative("init-approved", InitiativePatch(is_approved=True)) + await InitiativeRepository(ADMIN).patch_one({"slug": "init-approved"}, InitiativePatch(is_approved=True)) for i in range(cap + 2): # comfortably past the unapproved cap await _insert_project(f"appr-proj-{i}", owner=ALICE_EMAIL) - await _service(ALICE).patch(f"appr-proj-{i}", ProjectPatch(initiative="init-approved")) + await _service(ALICE).patch_one({"id": f"appr-proj-{i}"}, ProjectPatch(initiative="init-approved")) count = await MongoDbProjectRepository(ADMIN).count_initiative_members( initiative_id=(await InitiativeRepository(ADMIN).resolve_visible("init-approved")).id, # type: ignore[union-attr] exclude_project_id=None, @@ -185,7 +185,7 @@ async def test_admin_can_assign_to_any_initiative(self, db): # Alice's private initiative is manageable by an admin even though the admin holds no role. await _insert_project("adm-proj", owner=ALICE_EMAIL) init = await _insert_initiative("adm-init", ALICE) - updated = await _service(ADMIN).patch("adm-proj", ProjectPatch(initiative="adm-init")) + updated = await _service(ADMIN).patch_one({"id": "adm-proj"}, ProjectPatch(initiative="adm-init")) assert _assigned_id(updated) == init.id async def test_project_owner_can_unassign_without_initiative_rights(self, db): @@ -193,9 +193,9 @@ async def test_project_owner_can_unassign_without_initiative_rights(self, db): # his own project — unassignment needs only project-write access. await _insert_project("detach-proj", owner=BOB_EMAIL) await _insert_initiative("detach-init", ALICE) - await _service(_collaborator("detach-init", username=BOB_EMAIL)).patch( - "detach-proj", ProjectPatch(initiative="detach-init") + await _service(_collaborator("detach-init", username=BOB_EMAIL)).patch_one( + {"id": "detach-proj"}, ProjectPatch(initiative="detach-init") ) bob_plain = User(username=BOB_EMAIL, groups=frozenset()) - updated = await _service(bob_plain).patch("detach-proj", ProjectPatch(initiative=None)) + updated = await _service(bob_plain).patch_one({"id": "detach-proj"}, ProjectPatch(initiative=None)) assert _assigned_id(updated) is None diff --git a/mpcontribs-api/tests/integration/db/test_project_groups_repository.py b/mpcontribs-api/tests/integration/db/test_project_groups_repository.py index 3cb20f5b56..739cdaa529 100644 --- a/mpcontribs-api/tests/integration/db/test_project_groups_repository.py +++ b/mpcontribs-api/tests/integration/db/test_project_groups_repository.py @@ -56,19 +56,19 @@ async def _insert(name: str, owner: str = ALICE_EMAIL, **overrides) -> ProjectGr class TestGetOne: async def test_returns_group_by_identifiers(self, db): await _insert("group-a") - found = await _repo(ADMIN).get_project_group(name="group-a", owner=ALICE_EMAIL, fields=None) + found = await _repo(ADMIN).get_one({"name": "group-a", "owner": ALICE_EMAIL}, fields=None) assert found is not None assert found.name == "group-a" assert found.owner == ALICE_EMAIL async def test_returns_none_when_absent(self, db): - found = await _repo(ADMIN).get_project_group(name="missing", owner=ALICE_EMAIL, fields=None) + found = await _repo(ADMIN).get_one({"name": "missing", "owner": ALICE_EMAIL}, fields=None) assert found is None async def test_out_of_scope_returns_none(self, db): # Alice's private group is invisible to an anonymous caller. await _insert("group-priv") - found = await _repo(ANON).get_project_group(name="group-priv", owner=ALICE_EMAIL, fields=None) + found = await _repo(ANON).get_one({"name": "group-priv", "owner": ALICE_EMAIL}, fields=None) assert found is None @@ -85,20 +85,20 @@ def _role_user(group_id, username: str = "google:carol@example.com") -> User: class TestGroupRoleScope: async def test_role_grants_visibility(self, db): group = await _insert("role-vis") # Alice's private group - found = await _repo(_role_user(group.id)).get_project_group(name="role-vis", owner=ALICE_EMAIL, fields=None) + found = await _repo(_role_user(group.id)).get_one({"name": "role-vis", "owner": ALICE_EMAIL}, fields=None) assert found is not None assert found.id == group.id async def test_without_role_not_visible(self, db): await _insert("role-none") - found = await _repo(BOB).get_project_group(name="role-none", owner=ALICE_EMAIL, fields=None) + found = await _repo(BOB).get_one({"name": "role-none", "owner": ALICE_EMAIL}, fields=None) assert found is None async def test_malformed_role_is_ignored(self, db): await _insert("role-bad") member = User(username="google:carol@example.com", groups=frozenset({"project-group:not-an-oid"})) # A malformed role id must not raise; it simply grants nothing. - found = await _repo(member).get_project_group(name="role-bad", owner=ALICE_EMAIL, fields=None) + found = await _repo(member).get_one({"name": "role-bad", "owner": ALICE_EMAIL}, fields=None) assert found is None async def test_role_appears_in_listing(self, db): @@ -112,7 +112,7 @@ async def test_role_grants_scope_but_not_delete(self, db): # Scope makes the group visible, but deletion remains owner-or-admin (403 for a role holder). group = await _insert("role-del") with pytest.raises(PermissionError): - await _repo(_role_user(group.id)).delete_project_group(name="role-del", owner=ALICE_EMAIL) + await _repo(_role_user(group.id)).delete_one({"name": "role-del", "owner": ALICE_EMAIL}) assert await ProjectGroup.find_one(ProjectGroup.name == "role-del") is not None @@ -124,32 +124,32 @@ async def test_role_grants_scope_but_not_delete(self, db): class TestDeleteOne: async def test_deletes_matching_group(self, db): await _insert("del-a") - result = await _repo(ADMIN).delete_project_group(name="del-a", owner=ALICE_EMAIL) + result = await _repo(ADMIN).delete_one({"name": "del-a", "owner": ALICE_EMAIL}) assert result.num_deleted == 1 assert await ProjectGroup.find_one(ProjectGroup.name == "del-a") is None async def test_absent_raises_not_found(self, db): with pytest.raises(NotFoundError): - await _repo(ADMIN).delete_project_group(name="nope", owner=ALICE_EMAIL) + await _repo(ADMIN).delete_one({"name": "nope", "owner": ALICE_EMAIL}) async def test_out_of_scope_raises_not_found(self, db): # Alice's group is out of scope for anon, so it "does not exist" for them. await _insert("del-scoped") with pytest.raises(NotFoundError): - await _repo(ANON).delete_project_group(name="del-scoped", owner=ALICE_EMAIL) + await _repo(ANON).delete_one({"name": "del-scoped", "owner": ALICE_EMAIL}) # ...and it is untouched. assert await ProjectGroup.find_one(ProjectGroup.name == "del-scoped") is not None async def test_owner_can_delete_own(self, db): await _insert("del-own", owner=ALICE_EMAIL) - result = await _repo(ALICE).delete_project_group(name="del-own", owner=ALICE_EMAIL) + result = await _repo(ALICE).delete_one({"name": "del-own", "owner": ALICE_EMAIL}) assert result.num_deleted == 1 async def test_visible_public_non_owner_forbidden(self, db): # Bob can *see* Alice's public group but does not own it → 403, and it is left intact. await _insert("del-pub", owner=ALICE_EMAIL, is_public=True) with pytest.raises(PermissionError): - await _repo(BOB).delete_project_group(name="del-pub", owner=ALICE_EMAIL) + await _repo(BOB).delete_one({"name": "del-pub", "owner": ALICE_EMAIL}) assert await ProjectGroup.find_one(ProjectGroup.name == "del-pub") is not None async def test_wrong_identifier_keys_raise_validation(self, db): @@ -165,16 +165,12 @@ async def test_wrong_identifier_keys_raise_validation(self, db): class TestPatchOne: async def test_updates_field(self, db): await _insert("patch-a", description="before") - updated = await _repo(ADMIN).patch_project_group( - name="patch-a", owner=ALICE_EMAIL, update=ProjectGroupPatch(description="after") - ) + updated = await _repo(ADMIN).patch_one({"name": "patch-a", "owner": ALICE_EMAIL}, ProjectGroupPatch(description="after")) assert updated.description == "after" async def test_absent_raises_not_found(self, db): with pytest.raises(NotFoundError): - await _repo(ADMIN).patch_project_group( - name="ghost", owner=ALICE_EMAIL, update=ProjectGroupPatch(description="x") - ) + await _repo(ADMIN).patch_one({"name": "ghost", "owner": ALICE_EMAIL}, ProjectGroupPatch(description="x")) # --------------------------------------------------------------------------- diff --git a/mpcontribs-api/tests/integration/db/test_project_groups_service.py b/mpcontribs-api/tests/integration/db/test_project_groups_service.py index 5bbadf2fd2..332141eff9 100644 --- a/mpcontribs-api/tests/integration/db/test_project_groups_service.py +++ b/mpcontribs-api/tests/integration/db/test_project_groups_service.py @@ -63,7 +63,7 @@ async def test_add_by_id_links_projects(self, db): group = await _insert_group("add-id") await _insert_project("mp-1") await _insert_project("mp-2") - summary = await _service().add_projects_by_id(str(group.id), ["mp-1", "mp-2"]) + summary = await _service().add_projects({"id": str(group.id)}, ["mp-1", "mp-2"]) assert summary.succeeded == ["mp-1", "mp-2"] assert summary.failed == [] assert await _members(group.id) == ["mp-1", "mp-2"] @@ -71,20 +71,20 @@ async def test_add_by_id_links_projects(self, db): async def test_add_by_identifiers_links_projects(self, db): group = await _insert_group("add-ident") await _insert_project("mp-x") - summary = await _service().add_projects_by_identifiers("add-ident", ALICE_EMAIL, ["mp-x"]) + summary = await _service().add_projects({"name": "add-ident", "owner": ALICE_EMAIL}, ["mp-x"]) assert summary.succeeded == ["mp-x"] assert await _members(group.id) == ["mp-x"] async def test_add_is_idempotent(self, db): group = await _insert_group("add-idem") await _insert_project("mp-1") - await _service().add_projects_by_id(str(group.id), ["mp-1"]) - await _service().add_projects_by_id(str(group.id), ["mp-1"]) + await _service().add_projects({"id": str(group.id)}, ["mp-1"]) + await _service().add_projects({"id": str(group.id)}, ["mp-1"]) assert await _members(group.id) == ["mp-1"] async def test_missing_project_fails_and_leaves_group_unchanged(self, db): group = await _insert_group("add-missing") - summary = await _service().add_projects_by_id(str(group.id), ["ghost"]) + summary = await _service().add_projects({"id": str(group.id)}, ["ghost"]) assert summary.succeeded == [] assert summary.failed[0].error_code == "not_found" assert await _members(group.id) == [] @@ -93,7 +93,7 @@ async def test_out_of_scope_project_fails(self, db): # Bob's private project is invisible to Alice, so she cannot link it. group = await _insert_group("add-scope") await _insert_project("mp-bob", owner=BOB_EMAIL) - summary = await _service(ALICE).add_projects_by_id(str(group.id), ["mp-bob"]) + summary = await _service(ALICE).add_projects({"id": str(group.id)}, ["mp-bob"]) assert summary.succeeded == [] assert summary.failed[0].error_code == "not_found" assert await _members(group.id) == [] @@ -101,7 +101,7 @@ async def test_out_of_scope_project_fails(self, db): async def test_group_not_visible_raises_not_found(self, db): group = await _insert_group("add-priv") # owned by Alice, invisible to anon with pytest.raises(NotFoundError): - await _service(ANON).add_projects_by_id(str(group.id), []) + await _service(ANON).add_projects({"id": str(group.id)}, []) # --------------------------------------------------------------------------- @@ -129,24 +129,24 @@ async def test_delete_by_id_unlinks_project(self, db): group = await _insert_group("rm-id") await _insert_project("mp-1") await _insert_project("mp-2") - await _service().add_projects_by_id(str(group.id), ["mp-1", "mp-2"]) - summary = await _service().delete_projects_by_id(str(group.id), ["mp-1"]) + await _service().add_projects({"id": str(group.id)}, ["mp-1", "mp-2"]) + summary = await _service().delete_projects({"id": str(group.id)}, ["mp-1"]) assert summary.succeeded == ["mp-1"] assert await _members(group.id) == ["mp-2"] async def test_delete_by_identifiers_unlinks_project(self, db): group = await _insert_group("rm-ident") await _insert_project("mp-1") - await _service().add_projects_by_id(str(group.id), ["mp-1"]) - summary = await _service().delete_projects_by_identifiers("rm-ident", ALICE_EMAIL, ["mp-1"]) + await _service().add_projects({"id": str(group.id)}, ["mp-1"]) + summary = await _service().delete_projects({"name": "rm-ident", "owner": ALICE_EMAIL}, ["mp-1"]) assert summary.succeeded == ["mp-1"] assert await _members(group.id) == [] async def test_delete_non_member_reported_as_failure(self, db): group = await _insert_group("rm-nonmember") await _insert_project("mp-1") - await _service().add_projects_by_id(str(group.id), ["mp-1"]) - summary = await _service().delete_projects_by_id(str(group.id), ["ghost"]) + await _service().add_projects({"id": str(group.id)}, ["mp-1"]) + summary = await _service().delete_projects({"id": str(group.id)}, ["ghost"]) assert summary.succeeded == [] assert summary.failed[0].error_code == "not_found" assert await _members(group.id) == ["mp-1"] diff --git a/mpcontribs-api/tests/integration/db/test_projects_repository.py b/mpcontribs-api/tests/integration/db/test_projects_repository.py index ece6566b7e..a0286afb0b 100644 --- a/mpcontribs-api/tests/integration/db/test_projects_repository.py +++ b/mpcontribs-api/tests/integration/db/test_projects_repository.py @@ -125,29 +125,29 @@ def _noop_filter(): # --------------------------------------------------------------------------- -# get_project_by_id +# get_one # --------------------------------------------------------------------------- class TestGetProjectById: async def test_returns_project_for_valid_id(self, db): await _insert("get-by-id") - result = await _repo(ADMIN).get_project_by_id(id="get-by-id", fields=None) + result = await _repo(ADMIN).get_one({"id": "get-by-id"}, fields=None) assert result is not None assert result.id == "get-by-id" async def test_returns_none_for_missing_id(self, db): - result = await _repo(ADMIN).get_project_by_id(id="does-not-exist", fields=None) + result = await _repo(ADMIN).get_one({"id": "does-not-exist"}, fields=None) assert result is None async def test_admin_can_get_private_project(self, db): await _insert("get-priv", is_public=False) - result = await _repo(ADMIN).get_project_by_id(id="get-priv", fields=None) + result = await _repo(ADMIN).get_one({"id": "get-priv"}, fields=None) assert result is not None async def test_anon_cannot_get_private_project(self, db): await _insert("get-priv-anon", is_public=False) - result = await _repo(ANON).get_project_by_id(id="get-priv-anon", fields=None) + result = await _repo(ANON).get_one({"id": "get-priv-anon"}, fields=None) assert result is None @@ -157,7 +157,7 @@ async def test_anon_cannot_get_private_project(self, db): # Regression: Beanie stores the primary key under Mongo's ``_id`` (``id`` is an # alias), but fastapi-filter keys queries on the raw field name. Without the # ``id`` -> ``_id`` remap in BaseFilter these filters matched nothing even -# though get_project_by_id (which queries ``_id`` directly) found the document. +# though get_one (which queries ``_id`` directly) found the document. # --------------------------------------------------------------------------- @@ -325,7 +325,7 @@ async def test_all_items_covered_across_pages(self, db): # --------------------------------------------------------------------------- -# patch_project_by_id +# patch_one # --------------------------------------------------------------------------- @@ -333,7 +333,7 @@ class TestPatchProject: async def test_updates_single_field(self, db): await _insert("patch-me") patch = ProjectPatch(title="Updated Title") - await _repo(ADMIN).patch_project_by_id(id="patch-me", update=patch) + await _repo(ADMIN).patch_one({"id": "patch-me"}, patch) found = await Project.find_one(Project.id == "patch-me") assert found.title == "Updated Title" @@ -341,47 +341,47 @@ async def test_unset_fields_not_overwritten(self, db): await _insert("patch-preserve") original = await Project.find_one(Project.id == "patch-preserve") patch = ProjectPatch(title="New Title") - await _repo(ADMIN).patch_project_by_id(id="patch-preserve", update=patch) + await _repo(ADMIN).patch_one({"id": "patch-preserve"}, patch) found = await Project.find_one(Project.id == "patch-preserve") assert found.authors == original.authors async def test_not_found_raises(self, db): patch = ProjectPatch(title="Won't work") with pytest.raises(NotFoundError): - await _repo(ADMIN).patch_project_by_id(id="no-such-id", update=patch) + await _repo(ADMIN).patch_one({"id": "no-such-id"}, patch) async def test_empty_patch_returns_existing(self, db): await _insert("patch-empty") - result = await _repo(ADMIN).patch_project_by_id(id="patch-empty", update=ProjectPatch()) + result = await _repo(ADMIN).patch_one({"id": "patch-empty"}, ProjectPatch()) assert result.id == "patch-empty" # --------------------------------------------------------------------------- -# delete_project_by_id (soft-delete via DocumentWithSoftDelete) +# delete_one (soft-delete via DocumentWithSoftDelete) # --------------------------------------------------------------------------- class TestDeleteProject: async def test_deleted_project_not_in_default_query(self, db): await _insert("del-me", is_public=True, is_approved=True) - await _repo(ADMIN).delete_project_by_id(id="del-me") + await _repo(ADMIN).delete_one({"id": "del-me"}) page = await _repo(ADMIN).get_projects(filter=_noop_filter(), pagination=CursorParams(), fields=None) ids = {p.id for p in page.items} assert "del-me" not in ids async def test_delete_nonexistent_throws_error(self, db): - # delete_project_by_id does find_one().delete() — Error if not found + # delete_one does find_one().delete() — Error if not found with pytest.raises(NotFoundError, match="not found"): - await _repo(ADMIN).delete_project_by_id(id="ghost-id") + await _repo(ADMIN).delete_one({"id": "ghost-id"}) async def test_owner_can_delete_own_project(self, db): await _insert("del-own", owner="google:alice@example.com") - await _repo(ALICE).delete_project_by_id(id="del-own") + await _repo(ALICE).delete_one({"id": "del-own"}) assert await Project.find_one(Project.id == "del-own") is None async def test_admin_can_delete_any_project(self, db): await _insert("del-admin", owner="google:alice@example.com") - await _repo(ADMIN).delete_project_by_id(id="del-admin") + await _repo(ADMIN).delete_one({"id": "del-admin"}) assert await Project.find_one(Project.id == "del-admin") is None async def test_group_member_non_owner_cannot_delete(self, db): @@ -389,52 +389,52 @@ async def test_group_member_non_owner_cannot_delete(self, db): member = User(username="google:carol@example.com", groups=frozenset({"del-grp"})) await _insert("del-grp", owner="google:alice@example.com") with pytest.raises(PermissionError): - await _repo(member).delete_project_by_id(id="del-grp") + await _repo(member).delete_one({"id": "del-grp"}) assert await Project.find_one(Project.id == "del-grp") is not None async def test_visible_public_non_owner_cannot_delete(self, db): # BOB can see the public+approved project but does not own it → 403, not a silent success. await _insert("del-pub", owner="google:alice@example.com", is_public=True, is_approved=True) with pytest.raises(PermissionError): - await _repo(BOB).delete_project_by_id(id="del-pub") + await _repo(BOB).delete_one({"id": "del-pub"}) assert await Project.find_one(Project.id == "del-pub") is not None async def test_out_of_scope_delete_not_found(self, db): # BOB cannot see Alice's private project → 404 (existence is not leaked as a 403). await _insert("del-hidden", owner="google:alice@example.com", is_public=False) with pytest.raises(NotFoundError): - await _repo(BOB).delete_project_by_id(id="del-hidden") + await _repo(BOB).delete_one({"id": "del-hidden"}) assert await Project.find_one(Project.id == "del-hidden") is not None # --------------------------------------------------------------------------- -# upsert_project_by_id +# upsert_one # --------------------------------------------------------------------------- class TestUpsertProject: async def test_upsert_creates_new_project(self, db): data = _project_in("upsert-new") - await _repo(ADMIN).upsert_project_by_id(id="upsert-new", data=data) + await _repo(ADMIN).upsert_one({"id": "upsert-new"}, data=data) found = await Project.find_one(Project.id == "upsert-new") assert found is not None async def test_upsert_updates_existing_project(self, db): await _insert("upsert-existing") data = _project_in("upsert-existing", title="Replaced Title") - await _repo(ADMIN).upsert_project_by_id(id="upsert-existing", data=data) + await _repo(ADMIN).upsert_one({"id": "upsert-existing"}, data=data) found = await Project.find_one(Project.id == "upsert-existing") assert found.title == "Replaced Title" async def test_upsert_uses_path_id_not_body_id(self, db): data = _project_in("body-id") - await _repo(ADMIN).upsert_project_by_id(id="path-id", data=data) + await _repo(ADMIN).upsert_one({"id": "path-id"}, data=data) found = await Project.find_one(Project.id == "path-id") assert found is not None # --------------------------------------------------------------------------- -# upsert_project_by_id — authorization (owner-or-admin) +# upsert_one — authorization (owner-or-admin) # --------------------------------------------------------------------------- BOB = User(username="google:bob@example.com", groups=frozenset()) @@ -444,14 +444,14 @@ class TestUpsertProjectAuthorization: async def test_owner_can_overwrite_own_project(self, db): await _insert("auth-own", owner="google:alice@example.com") data = _project_in("auth-own", owner="google:alice@example.com", title="Owner Edit") - await _repo(ALICE).upsert_project_by_id(id="auth-own", data=data) + await _repo(ALICE).upsert_one({"id": "auth-own"}, data=data) found = await Project.find_one(Project.id == "auth-own") assert found.title == "Owner Edit" async def test_admin_can_overwrite_any_project(self, db): await _insert("auth-admin", owner="google:alice@example.com") data = _project_in("auth-admin", owner="google:alice@example.com", title="Admin Edit") - await _repo(ADMIN).upsert_project_by_id(id="auth-admin", data=data) + await _repo(ADMIN).upsert_one({"id": "auth-admin"}, data=data) found = await Project.find_one(Project.id == "auth-admin") assert found.title == "Admin Edit" @@ -461,14 +461,14 @@ async def test_non_owner_cannot_overwrite(self, db): from mpcontribs_api.exceptions import PermissionError as AppPermissionError with pytest.raises(AppPermissionError): - await _repo(BOB).upsert_project_by_id(id="auth-other", data=data) + await _repo(BOB).upsert_one({"id": "auth-other"}, data=data) found = await Project.find_one(Project.id == "auth-other") assert found.title == "Original" async def test_new_project_sets_owner_to_caller(self, db): # Body owner is someone else; the caller's identity must win. data = _project_in("auth-newowner", owner="google:alice@example.com") - await _repo(BOB).upsert_project_by_id(id="auth-newowner", data=data) + await _repo(BOB).upsert_one({"id": "auth-newowner"}, data=data) found = await Project.find_one(Project.id == "auth-newowner") assert found.owner == "google:bob@example.com" @@ -476,7 +476,7 @@ async def test_update_preserves_original_owner(self, db): await _insert("auth-preserve", owner="google:alice@example.com") # Alice tries to reassign ownership via the body; owner must stay hers. data = _project_in("auth-preserve", owner="google:bob@example.com", title="Edit") - await _repo(ALICE).upsert_project_by_id(id="auth-preserve", data=data) + await _repo(ALICE).upsert_one({"id": "auth-preserve"}, data=data) found = await Project.find_one(Project.id == "auth-preserve") assert found.owner == "google:alice@example.com" @@ -490,13 +490,13 @@ class TestApprovalIsAdminOnly: async def test_non_admin_cannot_patch_is_approved(self, db): await _insert("appr-patch", owner="google:alice@example.com") with pytest.raises(PermissionError): - await _repo(ALICE).patch_project_by_id(id="appr-patch", update=ProjectPatch(is_approved=True)) + await _repo(ALICE).patch_one({"id": "appr-patch"}, ProjectPatch(is_approved=True)) found = await Project.find_one(Project.id == "appr-patch") assert found.is_approved is False async def test_admin_can_patch_is_approved(self, db): await _insert("appr-patch-admin", owner="google:alice@example.com") - await _repo(ADMIN).patch_project_by_id(id="appr-patch-admin", update=ProjectPatch(is_approved=True)) + await _repo(ADMIN).patch_one({"id": "appr-patch-admin"}, ProjectPatch(is_approved=True)) found = await Project.find_one(Project.id == "appr-patch-admin") assert found.is_approved is True @@ -510,14 +510,14 @@ class TestPublicRequiresApproved: async def test_patch_public_on_unapproved_rejected(self, db): await _insert("pub-unappr", owner="google:alice@example.com", is_approved=False) with pytest.raises(ValidationError, match="approved"): - await _repo(ADMIN).patch_project_by_id(id="pub-unappr", update=ProjectPatch(is_public=True)) + await _repo(ADMIN).patch_one({"id": "pub-unappr"}, ProjectPatch(is_public=True)) found = await Project.find_one(Project.id == "pub-unappr") assert found.is_public is False async def test_patch_public_and_approved_together_succeeds(self, db): await _insert("pub-both", owner="google:alice@example.com", is_approved=False) - await _repo(ADMIN).patch_project_by_id( - id="pub-both", update=ProjectPatch(is_public=True, is_approved=True) + await _repo(ADMIN).patch_one( + {"id": "pub-both"}, ProjectPatch(is_public=True, is_approved=True) ) found = await Project.find_one(Project.id == "pub-both") assert found.is_public is True @@ -525,7 +525,7 @@ async def test_patch_public_and_approved_together_succeeds(self, db): async def test_patch_public_on_approved_succeeds(self, db): await _insert("pub-approved", owner="google:alice@example.com", is_approved=True) - await _repo(ADMIN).patch_project_by_id(id="pub-approved", update=ProjectPatch(is_public=True)) + await _repo(ADMIN).patch_one({"id": "pub-approved"}, ProjectPatch(is_public=True)) found = await Project.find_one(Project.id == "pub-approved") assert found.is_public is True @@ -538,14 +538,14 @@ async def test_patch_public_on_approved_succeeds(self, db): class TestUpsertServerManagedFields: async def test_new_project_is_private_and_unapproved(self, db): # ProjectIn has no is_public/is_approved, so a new PUT project starts safe by default. - await _repo(BOB).upsert_project_by_id(id="srv-new", data=_project_in("srv-new")) + await _repo(BOB).upsert_one({"id": "srv-new"}, data=_project_in("srv-new")) found = await Project.find_one(Project.id == "srv-new") assert found.is_public is False assert found.is_approved is False async def test_admin_upsert_cannot_approve_via_body(self, db): # Approval is PATCH-only even for an admin; a PUT can never approve a project. - await _repo(ADMIN).upsert_project_by_id(id="srv-admin-new", data=_project_in("srv-admin-new")) + await _repo(ADMIN).upsert_one({"id": "srv-admin-new"}, data=_project_in("srv-admin-new")) found = await Project.find_one(Project.id == "srv-admin-new") assert found.is_approved is False @@ -553,7 +553,7 @@ async def test_update_preserves_public_and_approved(self, db): # A full-replace PUT by the owner must not wipe server-managed publication/approval. await _insert("srv-preserve", owner="google:alice@example.com", is_public=True, is_approved=True) data = _project_in("srv-preserve", owner="google:alice@example.com", title="Renamed Title") - await _repo(ALICE).upsert_project_by_id(id="srv-preserve", data=data) + await _repo(ALICE).upsert_one({"id": "srv-preserve"}, data=data) found = await Project.find_one(Project.id == "srv-preserve") assert found.title == "Renamed Title" # content fields still update assert found.is_public is True @@ -561,6 +561,6 @@ async def test_update_preserves_public_and_approved(self, db): async def test_update_preserves_stats(self, db): await _insert("srv-stats", owner="google:alice@example.com", stats=Stats(contributions=7)) - await _repo(ALICE).upsert_project_by_id(id="srv-stats", data=_project_in("srv-stats")) + await _repo(ALICE).upsert_one({"id": "srv-stats"}, data=_project_in("srv-stats")) found = await Project.find_one(Project.id == "srv-stats") assert found.stats.contributions == 7 diff --git a/mpcontribs-api/tests/integration/test_component_routes.py b/mpcontribs-api/tests/integration/test_component_routes.py index f47fb1ae64..6452a4c1a7 100644 --- a/mpcontribs-api/tests/integration/test_component_routes.py +++ b/mpcontribs-api/tests/integration/test_component_routes.py @@ -127,15 +127,15 @@ def test_post_forwards_to_service(self, client, structure_service): class TestStructuresByIdRouting: def test_get_by_id_conventional_path(self, client, structure_service): - structure_service.get_by_id.return_value = SAMPLE_STRUCTURE + structure_service.get_one.return_value = SAMPLE_STRUCTURE assert client.get(f"/api/v1/structures/{PydanticObjectId()}").status_code == 200 def test_delete_by_id_conventional_path(self, client, structure_service): - structure_service.delete_by_id.return_value = ComponentDeleteResponse(num_deleted=1) + structure_service.delete_one.return_value = ComponentDeleteResponse(num_deleted=1) assert client.delete(f"/api/v1/structures/{PydanticObjectId()}").status_code == 200 def test_patch_by_id_conventional_path(self, client, structure_service): - structure_service.patch_by_id.return_value = SAMPLE_STRUCTURE + structure_service.patch_one.return_value = SAMPLE_STRUCTURE r = client.patch(f"/api/v1/structures/{PydanticObjectId()}", json={"name": "renamed"}) assert r.status_code == 200 @@ -186,15 +186,15 @@ def test_post_forwards_to_service(self, client, table_service): class TestTablesByIdRouting: def test_get_by_id_conventional_path(self, client, table_service): - table_service.get_by_id.return_value = SAMPLE_TABLE + table_service.get_one.return_value = SAMPLE_TABLE assert client.get(f"/api/v1/tables/{PydanticObjectId()}").status_code == 200 def test_delete_by_id_conventional_path(self, client, table_service): - table_service.delete_by_id.return_value = ComponentDeleteResponse(num_deleted=1) + table_service.delete_one.return_value = ComponentDeleteResponse(num_deleted=1) assert client.delete(f"/api/v1/tables/{PydanticObjectId()}").status_code == 200 def test_patch_by_id_conventional_path(self, client, table_service): - table_service.patch_by_id.return_value = SAMPLE_TABLE + table_service.patch_one.return_value = SAMPLE_TABLE r = client.patch(f"/api/v1/tables/{PydanticObjectId()}", json={"name": "x"}) assert r.status_code == 200 @@ -212,14 +212,14 @@ def test_list_calls_attachment_service(self, client, attachment_service): attachment_service.get_many.assert_awaited_once() def test_get_by_id_calls_attachment_service(self, client, attachment_service): - attachment_service.get_by_id.return_value = None + attachment_service.get_one.return_value = None client.get(f"/api/v1/attachments/{PydanticObjectId()}") - attachment_service.get_by_id.assert_awaited_once() + attachment_service.get_one.assert_awaited_once() def test_delete_by_id_calls_attachment_service(self, client, attachment_service): - attachment_service.delete_by_id.return_value = ComponentDeleteResponse(num_deleted=1) + attachment_service.delete_one.return_value = ComponentDeleteResponse(num_deleted=1) client.delete(f"/api/v1/attachments/{PydanticObjectId()}") - attachment_service.delete_by_id.assert_awaited_once() + attachment_service.delete_one.assert_awaited_once() def test_batch_delete_calls_attachment_service(self, client, attachment_service): attachment_service.delete.return_value = ComponentDeleteResponse(num_deleted=0) @@ -310,12 +310,12 @@ def test_structures_delete_anon_401(self, client, structure_service): def test_structure_delete_by_id_anon_401(self, client, structure_service): r = client.delete(f"/api/v1/structures/{PydanticObjectId()}", headers=FORCE_ANON_HEADERS) assert r.status_code == 401 - structure_service.delete_by_id.assert_not_called() + structure_service.delete_one.assert_not_called() def test_structure_patch_by_id_anon_401(self, client, structure_service): r = client.patch(f"/api/v1/structures/{PydanticObjectId()}", json={"name": "x"}, headers=FORCE_ANON_HEADERS) assert r.status_code == 401 - structure_service.patch_by_id.assert_not_called() + structure_service.patch_one.assert_not_called() def test_tables_delete_anon_401(self, client, table_service): r = client.delete("/api/v1/tables", headers=FORCE_ANON_HEADERS) @@ -325,7 +325,7 @@ def test_tables_delete_anon_401(self, client, table_service): def test_attachment_delete_by_id_anon_401(self, client, attachment_service): r = client.delete(f"/api/v1/attachments/{PydanticObjectId()}", headers=FORCE_ANON_HEADERS) assert r.status_code == 401 - attachment_service.delete_by_id.assert_not_called() + attachment_service.delete_one.assert_not_called() def test_structures_get_still_open_to_anon(self, client, structure_service): structure_service.get_many.return_value = Page(items=[], next_cursor=None) diff --git a/mpcontribs-api/tests/integration/test_contributions_routes.py b/mpcontribs-api/tests/integration/test_contributions_routes.py index 36adad5984..0921854f8e 100644 --- a/mpcontribs-api/tests/integration/test_contributions_routes.py +++ b/mpcontribs-api/tests/integration/test_contributions_routes.py @@ -136,24 +136,22 @@ def test_malformed_body_returns_422(self, client, contribution_service): class TestContributionByIdRouting: """RED: routes mount as /contributions{id} not /contributions/{id}.""" - def test_get_by_id_conventional_path(self, client, contribution_repo): - contribution_repo.get_contribution_by_id.return_value = SAMPLE_OUT + def test_get_by_id_conventional_path(self, client, contribution_service): + contribution_service.get_one.return_value = SAMPLE_OUT assert client.get(f"/api/v1/contributions/{PydanticObjectId()}").status_code == 200 - def test_patch_by_id_conventional_path(self, client, contribution_repo): - contribution_repo.patch_contribution_by_id.return_value = SAMPLE_OUT + def test_patch_by_id_conventional_path(self, client, contribution_service): + contribution_service.patch_one.return_value = SAMPLE_OUT r = client.patch(f"/api/v1/contributions/{PydanticObjectId()}", json={"formula": "H2O"}) assert r.status_code == 200 - def test_put_by_id_conventional_path(self, client, contribution_repo): - contribution_repo.upsert_contribution_by_id.return_value = SAMPLE_OUT + def test_put_by_id_conventional_path(self, client, contribution_service): + contribution_service.upsert_one.return_value = SAMPLE_OUT r = client.put(f"/api/v1/contributions/{PydanticObjectId()}", json=_valid_contribution_body()) assert r.status_code == 200 def test_delete_by_id_conventional_path(self, client, contribution_service): - contribution_service.delete_contributions.return_value = BulkDeleteSummary( - num_deleted=1, num_children_deleted=0 - ) + contribution_service.delete_one.return_value = BulkDeleteSummary(num_deleted=1, num_children_deleted=0) assert client.delete(f"/api/v1/contributions/{PydanticObjectId()}").status_code == 200 def test_download_route_conventional_path(self, client, contribution_repo): @@ -168,23 +166,17 @@ def test_download_route_conventional_path(self, client, contribution_repo): class TestDeleteContributionByIdWiring: def test_delete_delegates_to_service(self, client, contribution_service): - contribution_service.delete_contributions.return_value = BulkDeleteSummary( - num_deleted=1, num_children_deleted=2 - ) + contribution_service.delete_one.return_value = BulkDeleteSummary(num_deleted=1, num_children_deleted=2) oid = PydanticObjectId() - # NOTE: glued path is intentional here — see module docstring. r = client.delete(f"/api/v1/contributions/{oid}") assert r.status_code == 200 - contribution_service.delete_contributions.assert_awaited_once() + contribution_service.delete_one.assert_awaited_once() - def test_delete_builds_filter_from_path_id(self, client, contribution_service): - contribution_service.delete_contributions.return_value = BulkDeleteSummary( - num_deleted=1, num_children_deleted=0 - ) + def test_delete_passes_id_identifiers_to_service(self, client, contribution_service): + contribution_service.delete_one.return_value = BulkDeleteSummary(num_deleted=1, num_children_deleted=0) oid = PydanticObjectId() client.delete(f"/api/v1/contributions/{oid}") - passed_filter = contribution_service.delete_contributions.call_args.args[0] - assert passed_filter.id == oid + assert contribution_service.delete_one.call_args.args[0] == {"id": str(oid)} # =========================================================================== @@ -278,21 +270,21 @@ def test_delete_by_id_anon_401(self, client, contribution_service): r = client.delete(f"/api/v1/contributions/{PydanticObjectId()}", headers=FORCE_ANON_HEADERS) assert r.status_code == 401 - def test_put_by_id_anon_401(self, client, contribution_repo): + def test_put_by_id_anon_401(self, client, contribution_service): r = client.put( f"/api/v1/contributions/{PydanticObjectId()}", json=_valid_contribution_body(), headers=FORCE_ANON_HEADERS, ) assert r.status_code == 401 - contribution_repo.upsert_contribution_by_id.assert_not_called() + contribution_service.upsert_one.assert_not_called() - def test_patch_by_id_anon_401(self, client, contribution_repo): + def test_patch_by_id_anon_401(self, client, contribution_service): r = client.patch( f"/api/v1/contributions/{PydanticObjectId()}", json={"formula": "H2O"}, headers=FORCE_ANON_HEADERS ) assert r.status_code == 401 - contribution_repo.patch_contribution_by_id.assert_not_called() + contribution_service.patch_one.assert_not_called() def test_get_collection_still_open_to_anon(self, client, contribution_repo): from mpcontribs_api.pagination import Page diff --git a/mpcontribs-api/tests/integration/test_initiatives.py b/mpcontribs-api/tests/integration/test_initiatives.py index c0eba7a471..516bb5b469 100644 --- a/mpcontribs-api/tests/integration/test_initiatives.py +++ b/mpcontribs-api/tests/integration/test_initiatives.py @@ -78,7 +78,7 @@ def test_list_returns_200(self, client, initiative_repo): assert r.status_code == 200 def test_get_by_slug_returns_200(self, client, initiative_repo): - initiative_repo.get_initiative.return_value = _stored() + initiative_repo.get_one.return_value = _stored() r = client.get("/api/v1/initiatives/battery-genome", headers=AUTHED_HEADERS) assert r.status_code == 200 assert r.json()["slug"] == "battery-genome" @@ -91,7 +91,7 @@ def test_get_by_slug_returns_200(self, client, initiative_repo): class TestPatch: def test_patch_returns_200(self, client, initiative_repo): - initiative_repo.patch_initiative.return_value = _stored(name="Renamed") + initiative_repo.patch_one.return_value = _stored(name="Renamed") r = client.patch( "/api/v1/initiatives/battery-genome", json={"name": "Renamed"}, @@ -109,7 +109,7 @@ def test_anonymous_rejected_401(self, client, initiative_repo): assert r.status_code == 401 def test_not_found_propagates_404(self, client, initiative_repo): - initiative_repo.patch_initiative.side_effect = NotFoundError("nope") + initiative_repo.patch_one.side_effect = NotFoundError("nope") r = client.patch( "/api/v1/initiatives/missing", json={"name": "Renamed"}, @@ -125,7 +125,7 @@ def test_not_found_propagates_404(self, client, initiative_repo): class TestDelete: def test_delete_returns_204(self, client, initiative_repo): - initiative_repo.delete_initiative.return_value = None + initiative_repo.delete_one.return_value = None r = client.delete("/api/v1/initiatives/battery-genome", headers=AUTHED_HEADERS) assert r.status_code == 204 assert r.content == b"" diff --git a/mpcontribs-api/tests/integration/test_projects.py b/mpcontribs-api/tests/integration/test_projects.py index e5fcf39a19..7a7179c562 100644 --- a/mpcontribs-api/tests/integration/test_projects.py +++ b/mpcontribs-api/tests/integration/test_projects.py @@ -117,44 +117,43 @@ def test_valid_fields_param_forwarded(self, client, project_repo): class TestGetProjectById: - def test_found_returns_200(self, client, project_repo): - project_repo.get_project_by_id.return_value = SAMPLE_PROJECT + def test_found_returns_200(self, client, project_service): + project_service.get_one.return_value = SAMPLE_PROJECT r = client.get("/api/v1/projects/mp-sample", headers=AUTHED_HEADERS) assert r.status_code == 200 - def test_response_contains_project_data(self, client, project_repo): - project_repo.get_project_by_id.return_value = SAMPLE_PROJECT + def test_response_contains_project_data(self, client, project_service): + project_service.get_one.return_value = SAMPLE_PROJECT body = client.get("/api/v1/projects/mp-sample", headers=AUTHED_HEADERS).json() assert body["id"] == "mp-sample" assert body["title"] == "Sample Project" - def test_not_found_returns_404(self, client, project_repo): - project_repo.get_project_by_id.side_effect = NotFoundError("project not found") + def test_not_found_returns_404(self, client, project_service): + project_service.get_one.side_effect = NotFoundError("project not found") r = client.get("/api/v1/projects/nonexistent", headers=AUTHED_HEADERS) assert r.status_code == 404 - def test_not_found_error_code(self, client, project_repo): - project_repo.get_project_by_id.side_effect = NotFoundError("project not found") + def test_not_found_error_code(self, client, project_service): + project_service.get_one.side_effect = NotFoundError("project not found") body = client.get("/api/v1/projects/nonexistent", headers=AUTHED_HEADERS).json() assert body["error"]["code"] == "not_found" - def test_id_forwarded_to_repo(self, client, project_repo): - project_repo.get_project_by_id.return_value = SAMPLE_PROJECT + def test_id_forwarded_to_service(self, client, project_service): + project_service.get_one.return_value = SAMPLE_PROJECT client.get("/api/v1/projects/my-specific-id", headers=AUTHED_HEADERS) - _, kwargs = project_repo.get_project_by_id.call_args - assert kwargs["id"] == "my-specific-id" + assert project_service.get_one.call_args.args[0] == {"id": "my-specific-id"} - def test_fields_param_forwarded(self, client, project_repo): - project_repo.get_project_by_id.return_value = SAMPLE_PROJECT + def test_fields_param_forwarded(self, client, project_service): + project_service.get_one.return_value = SAMPLE_PROJECT client.get("/api/v1/projects/mp-sample", params={"_fields": "title"}, headers=AUTHED_HEADERS) - _, kwargs = project_repo.get_project_by_id.call_args + _, kwargs = project_service.get_one.call_args assert kwargs["fields"] is not None assert "title" in kwargs["fields"] - def test_no_fields_param_uses_default_fields(self, client, project_repo): - project_repo.get_project_by_id.return_value = SAMPLE_PROJECT + def test_no_fields_param_uses_default_fields(self, client, project_service): + project_service.get_one.return_value = SAMPLE_PROJECT client.get("/api/v1/projects/mp-sample", headers=AUTHED_HEADERS) - _, kwargs = project_repo.get_project_by_id.call_args + _, kwargs = project_service.get_one.call_args assert kwargs["fields"] is not None assert "title" in kwargs["fields"] @@ -166,7 +165,7 @@ def test_no_fields_param_uses_default_fields(self, client, project_repo): class TestPatchProject: def test_valid_patch_returns_200(self, client, project_service): - project_service.patch.return_value = SAMPLE_PROJECT + project_service.patch_one.return_value = SAMPLE_PROJECT r = client.patch( "/api/v1/projects/mp-sample", json={"title": "Updated Title"}, @@ -176,7 +175,7 @@ def test_valid_patch_returns_200(self, client, project_service): def test_patch_response_is_project_out(self, client, project_service): updated = ProjectOut(id="mp-sample", title="Updated Title") - project_service.patch.return_value = updated + project_service.patch_one.return_value = updated body = client.patch( "/api/v1/projects/mp-sample", json={"title": "Updated Title"}, @@ -185,7 +184,7 @@ def test_patch_response_is_project_out(self, client, project_service): assert body["title"] == "Updated Title" def test_not_found_returns_404(self, client, project_service): - project_service.patch.side_effect = NotFoundError("not found") + project_service.patch_one.side_effect = NotFoundError("not found") r = client.patch( "/api/v1/projects/missing", json={"title": "x" * 5}, @@ -202,15 +201,15 @@ def test_invalid_title_too_short_returns_422(self, client, project_service): assert r.status_code == 422 def test_id_and_update_forwarded_to_service(self, client, project_service): - project_service.patch.return_value = SAMPLE_PROJECT + project_service.patch_one.return_value = SAMPLE_PROJECT client.patch( "/api/v1/projects/mp-sample", json={"title": "New Name"}, headers=AUTHED_HEADERS, ) - _, kwargs = project_service.patch.call_args - assert kwargs["id"] == "mp-sample" - assert kwargs["update"].title == "New Name" + call = project_service.patch_one.call_args + assert call.args[0] == {"id": "mp-sample"} + assert call.kwargs["update"].title == "New Name" # --------------------------------------------------------------------------- @@ -219,21 +218,20 @@ def test_id_and_update_forwarded_to_service(self, client, project_service): class TestDeleteProject: - def test_delete_returns_204(self, client, project_repo): - project_repo.delete_project_by_id.return_value = None + def test_delete_returns_204(self, client, project_service): + project_service.delete_one.return_value = None r = client.delete("/api/v1/projects/mp-sample", headers=AUTHED_HEADERS) assert r.status_code == 204 - def test_delete_response_has_no_body(self, client, project_repo): - project_repo.delete_project_by_id.return_value = None + def test_delete_response_has_no_body(self, client, project_service): + project_service.delete_one.return_value = None r = client.delete("/api/v1/projects/mp-sample", headers=AUTHED_HEADERS) assert r.content == b"" - def test_id_forwarded_to_repo(self, client, project_repo): - project_repo.delete_project_by_id.return_value = None + def test_id_forwarded_to_service(self, client, project_service): + project_service.delete_one.return_value = None client.delete("/api/v1/projects/mp-sample", headers=AUTHED_HEADERS) - _, kwargs = project_repo.delete_project_by_id.call_args - assert kwargs["id"] == "mp-sample" + assert project_service.delete_one.call_args.args[0] == {"id": "mp-sample"} # --------------------------------------------------------------------------- @@ -254,17 +252,17 @@ def _valid_body(self, **overrides): body.update(overrides) return body - def test_valid_upsert_returns_200(self, client, project_repo): - project_repo.upsert_project_by_id.return_value = SAMPLE_PROJECT + def test_valid_upsert_returns_200(self, client, project_service): + project_service.upsert_one.return_value = SAMPLE_PROJECT r = client.put("/api/v1/projects/mp-sample", json=self._valid_body(), headers=AUTHED_HEADERS) assert r.status_code == 200 - def test_conflict_returns_409(self, client, project_repo): - project_repo.upsert_project_by_id.side_effect = ConflictError("already exists") + def test_conflict_returns_409(self, client, project_service): + project_service.upsert_one.side_effect = ConflictError("already exists") r = client.put("/api/v1/projects/mp-sample", json=self._valid_body(), headers=AUTHED_HEADERS) assert r.status_code == 409 - def test_missing_required_field_returns_422(self, client, project_repo): + def test_missing_required_field_returns_422(self, client, project_service): body = self._valid_body() del body["title"] r = client.put("/api/v1/projects/mp-sample", json=body, headers=AUTHED_HEADERS) @@ -287,19 +285,19 @@ def _body(self): "unique_identifiers": True, } - def test_anonymous_put_returns_401(self, client, project_repo): - project_repo.upsert_project_by_id.return_value = SAMPLE_PROJECT + def test_anonymous_put_returns_401(self, client, project_service): + project_service.upsert_one.return_value = SAMPLE_PROJECT r = client.put("/api/v1/projects/mp-sample", json=self._body(), headers=ANON_HEADERS) assert r.status_code == 401 assert r.json()["error"]["code"] == "authentication_error" - project_repo.upsert_project_by_id.assert_not_called() + project_service.upsert_one.assert_not_called() - def test_anonymous_patch_returns_401(self, client, project_repo): + def test_anonymous_patch_returns_401(self, client, project_service): r = client.patch("/api/v1/projects/mp-sample", json={"title": "Updated Title"}, headers=ANON_HEADERS) assert r.status_code == 401 - project_repo.patch_project_by_id.assert_not_called() + project_service.patch_one.assert_not_called() - def test_anonymous_delete_returns_401(self, client, project_repo): + def test_anonymous_delete_returns_401(self, client, project_service): r = client.delete("/api/v1/projects/mp-sample", headers=ANON_HEADERS) assert r.status_code == 401 - project_repo.delete_project_by_id.assert_not_called() + project_service.delete_one.assert_not_called() diff --git a/mpcontribs-api/tests/unit/domains/test_component_service.py b/mpcontribs-api/tests/unit/domains/test_component_service.py index 53f86ce17d..e297f81a4d 100644 --- a/mpcontribs-api/tests/unit/domains/test_component_service.py +++ b/mpcontribs-api/tests/unit/domains/test_component_service.py @@ -15,6 +15,13 @@ def _oid() -> PydanticObjectId: return PydanticObjectId() +def _coerce_identifiers(identifiers: dict) -> dict: + """Stub for the repo's ObjectId-keyed ``coerce_identifiers`` (string id -> ObjectId).""" + if isinstance(identifiers.get("id"), str): + return {**identifiers, "id": PydanticObjectId(identifiers["id"])} + return identifiers + + def _make_service( *, candidate_ids: list[PydanticObjectId], @@ -29,9 +36,8 @@ def _make_service( components = AsyncMock(name="components") components.list_ids = AsyncMock(return_value=candidate_ids) components.delete_by_ids = AsyncMock(side_effect=lambda ids: DeleteResponse(num_deleted=len(ids))) - components.delete_by_id = AsyncMock(return_value=DeleteResponse(num_deleted=1)) - components._convert_object_id = MagicMock(side_effect=lambda s: PydanticObjectId(s)) - components._not_found = MagicMock(return_value="not found") + components.delete_one = AsyncMock(return_value=DeleteResponse(num_deleted=1)) + components.coerce_identifiers = MagicMock(side_effect=_coerce_identifiers) contributions = AsyncMock(name="contributions") @@ -121,30 +127,30 @@ async def test_delete_by_id_not_reachable_raises_not_found(): svc, _, _ = _make_service(candidate_ids=[], reachable=set(), referenced=set()) with pytest.raises(NotFoundError): - await svc.delete_by_id(str(oid)) + await svc.delete_one({"id": str(oid)}) async def test_delete_by_id_referenced_is_skipped(): oid = _oid() svc, components, _ = _make_service(candidate_ids=[], reachable={oid}, referenced={oid}) - result = await svc.delete_by_id(str(oid)) + result = await svc.delete_one({"id": str(oid)}) assert result.num_deleted == 0 assert result.num_skipped == 1 assert result.referenced_ids == [oid] - components.delete_by_id.assert_not_awaited() + components.delete_one.assert_not_awaited() async def test_delete_by_id_reachable_and_unreferenced_deletes(): oid = _oid() svc, components, _ = _make_service(candidate_ids=[], reachable={oid}, referenced=set()) - result = await svc.delete_by_id(str(oid)) + result = await svc.delete_one({"id": str(oid)}) assert result.num_deleted == 1 assert result.num_skipped == 0 - components.delete_by_id.assert_awaited_once_with(oid) + components.delete_one.assert_awaited_once_with({"id": oid}) # --------------------------------------------------------------------------- @@ -155,8 +161,7 @@ async def test_delete_by_id_reachable_and_unreferenced_deletes(): def _make_read_service(*, reachable: set[PydanticObjectId]) -> tuple[ComponentService, AsyncMock, AsyncMock]: """ComponentService whose contribution repo reports `reachable` ids as in-scope.""" components = AsyncMock(name="components") - components._convert_object_id = MagicMock(side_effect=lambda s: PydanticObjectId(s)) - components._not_found = MagicMock(return_value="not found") + components.coerce_identifiers = MagicMock(side_effect=_coerce_identifiers) contributions = AsyncMock(name="contributions") @@ -174,21 +179,21 @@ async def test_get_by_id_unreachable_returns_none_without_fetch(): oid = _oid() svc, components, _ = _make_read_service(reachable=set()) - result = await svc.get_by_id(str(oid), fields=None) + result = await svc.get_one({"id": str(oid)}, fields=None) assert result is None - components.get_component_by_id.assert_not_awaited() + components.get_one.assert_not_awaited() async def test_get_by_id_reachable_fetches_component(): oid = _oid() svc, components, _ = _make_read_service(reachable={oid}) - components.get_component_by_id = AsyncMock(return_value="the-component") + components.get_one = AsyncMock(return_value="the-component") - result = await svc.get_by_id(str(oid), fields=None) + result = await svc.get_one({"id": str(oid)}, fields=None) assert result == "the-component" - components.get_component_by_id.assert_awaited_once() + components.get_one.assert_awaited_once() async def test_get_many_restricts_to_reachable_ids(): @@ -209,16 +214,16 @@ async def test_patch_by_id_unreachable_raises_not_found(): svc, components, _ = _make_read_service(reachable=set()) with pytest.raises(NotFoundError): - await svc.patch_by_id(str(oid), update=MagicMock()) - components.patch_component_by_id.assert_not_awaited() + await svc.patch_one({"id": str(oid)}, update=MagicMock()) + components.patch_one.assert_not_awaited() async def test_patch_by_id_reachable_patches(): oid = _oid() svc, components, _ = _make_read_service(reachable={oid}) - components.patch_component_by_id = AsyncMock(return_value="patched") + components.patch_one = AsyncMock(return_value="patched") - result = await svc.patch_by_id(str(oid), update=MagicMock()) + result = await svc.patch_one({"id": str(oid)}, update=MagicMock()) assert result == "patched" - components.patch_component_by_id.assert_awaited_once() + components.patch_one.assert_awaited_once() diff --git a/mpcontribs-api/tests/unit/domains/test_contribution_service.py b/mpcontribs-api/tests/unit/domains/test_contribution_service.py index 02451f5033..cfdc4e6695 100644 --- a/mpcontribs-api/tests/unit/domains/test_contribution_service.py +++ b/mpcontribs-api/tests/unit/domains/test_contribution_service.py @@ -519,12 +519,12 @@ async def test_insert_project_not_found_is_validation_failure(self): async def test_upsert_unique_forces_version_one(self): svc, contrib_repo, *_ = _make_service() # unique by default - contrib_repo.upsert_contribution_by_identifiers.return_value = MagicMock(spec=Contribution) + contrib_repo.upsert_one.return_value = MagicMock(spec=Contribution) # A supplied version is ignored for unique-identifier projects (inferred as 1). await svc.upsert_contributions([_contrib_in(identifier="mp-1", version=9)]) - assert contrib_repo.upsert_contribution_by_identifiers.call_args.args[2] == 1 + assert contrib_repo.upsert_one.call_args.args[2] == 1 contrib_repo.max_versions.assert_not_called() async def test_upsert_non_unique_requires_version(self): @@ -534,15 +534,15 @@ async def test_upsert_non_unique_requires_version(self): assert summary.succeeded == [] assert [f.error_code for f in summary.failed] == ["validation_error"] - contrib_repo.upsert_contribution_by_identifiers.assert_not_called() + contrib_repo.upsert_one.assert_not_called() async def test_upsert_non_unique_passes_supplied_version(self): svc, contrib_repo, *_ = _make_service(unique_identifiers=False) - contrib_repo.upsert_contribution_by_identifiers.return_value = MagicMock(spec=Contribution) + contrib_repo.upsert_one.return_value = MagicMock(spec=Contribution) await svc.upsert_contributions([_contrib_in(identifier="mp-1", version=7)]) - assert contrib_repo.upsert_contribution_by_identifiers.call_args.args[2] == 7 + assert contrib_repo.upsert_one.call_args.args[2] == 7 # --------------------------------------------------------------------------- @@ -593,8 +593,7 @@ async def test_raises_before_any_db_write(self): dirty = _contrib_in(structures=[_structure_in()]) with pytest.raises(ValidationError): await svc.upsert_contributions([dirty]) - contrib_repo.upsert_contribution_by_identifiers.assert_not_called() - contrib_repo.find_one_contribution.assert_not_called() + contrib_repo.upsert_one.assert_not_called() contrib_repo.insert_contribution.assert_not_called() contrib_repo.update_contribution.assert_not_called() @@ -607,7 +606,7 @@ async def test_raises_before_any_db_write(self): class TestUpsertContributionsAtomic: async def test_calls_atomic_repo_method_once_per_item(self): svc, contrib_repo, *_ = _make_service() - contrib_repo.upsert_contribution_by_identifiers.return_value = MagicMock(spec=Contribution) + contrib_repo.upsert_one.return_value = MagicMock(spec=Contribution) contribs = [_contrib_in(identifier=f"mp-{i}") for i in range(3)] summary = await svc.upsert_contributions(contribs) @@ -615,20 +614,19 @@ async def test_calls_atomic_repo_method_once_per_item(self): assert summary.total == 3 assert len(summary.succeeded) == 3 assert summary.failed == [] - assert contrib_repo.upsert_contribution_by_identifiers.call_count == 3 + assert contrib_repo.upsert_one.call_count == 3 # The legacy read-then-write path must not be used - contrib_repo.find_one_contribution.assert_not_called() contrib_repo.update_contribution.assert_not_called() contrib_repo.insert_contribution.assert_not_called() async def test_passes_identifiers_dict_and_input_to_repo(self): svc, contrib_repo, *_ = _make_service() - contrib_repo.upsert_contribution_by_identifiers.return_value = MagicMock(spec=Contribution) + contrib_repo.upsert_one.return_value = MagicMock(spec=Contribution) contrib = _contrib_in(project="my-proj", identifier="mp-99") await svc.upsert_contributions([contrib]) - call = contrib_repo.upsert_contribution_by_identifiers.call_args + call = contrib_repo.upsert_one.call_args assert call.args[0] == {"project": "my-proj", "identifier": "mp-99"} assert call.args[1] is contrib @@ -642,7 +640,7 @@ async def _upsert(identifiers, contrib, version): returned[contrib.identifier] = doc return doc - contrib_repo.upsert_contribution_by_identifiers.side_effect = _upsert + contrib_repo.upsert_one.side_effect = _upsert contribs = [_contrib_in(identifier=f"mp-{i}") for i in range(3)] summary = await svc.upsert_contributions(contribs) @@ -655,7 +653,7 @@ async def test_empty_batch_returns_empty_summary(self): assert summary.total == 0 assert summary.succeeded == [] assert summary.failed == [] - contrib_repo.upsert_contribution_by_identifiers.assert_not_called() + contrib_repo.upsert_one.assert_not_called() async def test_same_key_concurrent_upserts_both_go_through_atomic_call(self): """Race-safety regression: two items with the same (project, identifier) in one batch @@ -663,7 +661,7 @@ async def test_same_key_concurrent_upserts_both_go_through_atomic_call(self): tiebreaker — the service must not pre-deduplicate or otherwise swallow one. """ svc, contrib_repo, *_ = _make_service() - contrib_repo.upsert_contribution_by_identifiers.return_value = MagicMock(spec=Contribution) + contrib_repo.upsert_one.return_value = MagicMock(spec=Contribution) contribs = [ _contrib_in(project="p", identifier="same"), @@ -672,7 +670,7 @@ async def test_same_key_concurrent_upserts_both_go_through_atomic_call(self): summary = await svc.upsert_contributions(contribs) assert len(summary.succeeded) == 2 - assert contrib_repo.upsert_contribution_by_identifiers.call_count == 2 + assert contrib_repo.upsert_one.call_count == 2 async def test_one_failure_is_reported_not_raised(self): svc, contrib_repo, *_ = _make_service() @@ -682,7 +680,7 @@ async def _upsert(identifiers, contrib, version): raise ConflictError("boom") return MagicMock(spec=Contribution) - contrib_repo.upsert_contribution_by_identifiers.side_effect = _upsert + contrib_repo.upsert_one.side_effect = _upsert contribs = [_contrib_in(identifier=f"mp-{i}") for i in range(3)] summary = await svc.upsert_contributions(contribs) @@ -750,7 +748,7 @@ async def test_insert_unauthorized_and_oversize_yield_single_failure(self): async def test_upsert_rejects_unauthorized_project_per_item(self): svc, contrib_repo, *_ = _make_service(user=_member_user("allowed")) - contrib_repo.upsert_contribution_by_identifiers.return_value = MagicMock(spec=Contribution) + contrib_repo.upsert_one.return_value = MagicMock(spec=Contribution) contribs = [ _contrib_in(project="allowed", identifier="ok"), @@ -764,7 +762,7 @@ async def test_upsert_rejects_unauthorized_project_per_item(self): assert summary.failed[0].error_code == "permission_denied" assert "forbidden" in summary.failed[0].message # Only the authorized item reached the atomic repo method - contrib_repo.upsert_contribution_by_identifiers.assert_called_once() + contrib_repo.upsert_one.assert_called_once() async def test_upsert_anonymous_authorized_for_nothing(self): svc, contrib_repo, *_ = _make_service(user=User()) # anonymous: no username, no groups @@ -774,7 +772,7 @@ async def test_upsert_anonymous_authorized_for_nothing(self): assert summary.total == 1 assert summary.succeeded == [] assert [f.error_code for f in summary.failed] == ["permission_denied"] - contrib_repo.upsert_contribution_by_identifiers.assert_not_called() + contrib_repo.upsert_one.assert_not_called() # --------------------------------------------------------------------------- @@ -798,7 +796,7 @@ async def test_upsert_anonymous_authorized_for_nothing(self): # in_flight -= 1 # return MagicMock(spec=Contribution) -# contrib_repo.upsert_contribution_by_identifiers.side_effect = _upsert +# contrib_repo.upsert_one.side_effect = _upsert # contribs = [_contrib_in(identifier=f"mp-{i}") for i in range(5)] # await svc.upsert_contributions(contribs) diff --git a/mpcontribs-api/tests/unit/domains/test_project_group_service.py b/mpcontribs-api/tests/unit/domains/test_project_group_service.py index d5ab9ef0de..52a90d68b9 100644 --- a/mpcontribs-api/tests/unit/domains/test_project_group_service.py +++ b/mpcontribs-api/tests/unit/domains/test_project_group_service.py @@ -37,16 +37,21 @@ def _make_service(group: ProjectGroupOut | None, *, visible_projects: set[str] | groups.get_one.side_effect = ConflictError("ambiguous") else: groups.get_one.return_value = group - groups.get_by_id.return_value = group - # _convert_object_id is a sync repo method; keep it sync so the service gets a real id, not a coroutine. - groups._convert_object_id = MagicMock(side_effect=lambda s: PydanticObjectId(s)) + # coerce_identifiers is a sync repo method; keep it sync so the service gets a real dict, not a coroutine. + def _coerce_identifiers(identifiers): + if isinstance(identifiers.get("id"), str): + return {**identifiers, "id": PydanticObjectId(identifiers["id"])} + return identifiers + + groups.coerce_identifiers = MagicMock(side_effect=_coerce_identifiers) groups.add_project_refs.return_value = group groups.delete_project_refs.return_value = group - async def _get_project(pid, fields=None): + async def _get_project(identifiers, fields=None): + pid = identifiers["id"] return {"_id": pid} if pid in visible else None - projects.get_by_id.side_effect = _get_project + projects.get_one.side_effect = _get_project return ProjectGroupService(groups=groups, projects=projects), groups, projects @@ -88,7 +93,7 @@ async def test_empty_projects_inserts_without_validation(self): service, groups, projects = _make_service(None) payload = self._payload([]) await service.insert(payload) - projects.get_by_id.assert_not_awaited() + projects.get_one.assert_not_awaited() groups.insert_project_group.assert_awaited_once_with(payload) @@ -101,17 +106,17 @@ class TestGroupResolution: async def test_add_by_id_missing_group_raises_not_found(self): service, _, _ = _make_service(None) with pytest.raises(NotFoundError): - await service.add_projects_by_id("0" * 24, ["mp-1"]) + await service.add_projects({"id": "0" * 24}, ["mp-1"]) async def test_add_by_identifiers_missing_group_raises_not_found(self): service, _, _ = _make_service(None) with pytest.raises(NotFoundError): - await service.add_projects_by_identifiers("g", "google:a@b.com", ["mp-1"]) + await service.add_projects({"name": "g", "owner": "google:a@b.com"}, ["mp-1"]) async def test_ambiguous_identifiers_propagate_conflict(self): service, _, _ = _make_service(_group(), ambiguous=True) with pytest.raises(ConflictError): - await service.add_projects_by_identifiers("g", "google:a@b.com", ["mp-1"]) + await service.add_projects({"name": "g", "owner": "google:a@b.com"}, ["mp-1"]) # --------------------------------------------------------------------------- @@ -123,7 +128,7 @@ class TestAddProjects: async def test_valid_projects_are_added(self): group = _group() service, groups, _ = _make_service(group, visible_projects={"mp-1", "mp-2"}) - summary = await service.add_projects_by_id(str(group.id), ["mp-1", "mp-2"]) + summary = await service.add_projects({"id": str(group.id)}, ["mp-1", "mp-2"]) assert summary.total == 2 assert summary.succeeded == ["mp-1", "mp-2"] assert summary.failed == [] @@ -132,7 +137,7 @@ async def test_valid_projects_are_added(self): async def test_missing_project_reported_as_failure(self): group = _group() service, groups, _ = _make_service(group, visible_projects={"mp-1"}) - summary = await service.add_projects_by_id(str(group.id), ["mp-1", "ghost"]) + summary = await service.add_projects({"id": str(group.id)}, ["mp-1", "ghost"]) assert summary.succeeded == ["mp-1"] assert len(summary.failed) == 1 assert summary.failed[0].index == 1 @@ -143,7 +148,7 @@ async def test_missing_project_reported_as_failure(self): async def test_no_valid_projects_skips_update(self): group = _group() service, groups, _ = _make_service(group, visible_projects=set()) - summary = await service.add_projects_by_id(str(group.id), ["ghost"]) + summary = await service.add_projects({"id": str(group.id)}, ["ghost"]) assert summary.succeeded == [] assert len(summary.failed) == 1 groups.add_project_refs.assert_not_awaited() @@ -151,7 +156,7 @@ async def test_no_valid_projects_skips_update(self): async def test_duplicate_input_added_once(self): group = _group() service, groups, _ = _make_service(group, visible_projects={"mp-1"}) - summary = await service.add_projects_by_id(str(group.id), ["mp-1", "mp-1"]) + summary = await service.add_projects({"id": str(group.id)}, ["mp-1", "mp-1"]) assert summary.succeeded == ["mp-1"] groups.add_project_refs.assert_awaited_once_with(group.id, ["mp-1"]) @@ -165,7 +170,7 @@ class TestDeleteProjects: async def test_members_deleted_non_members_reported(self): group = _group(["mp-1", "mp-2"]) service, groups, _ = _make_service(group) - summary = await service.delete_projects_by_id(str(group.id), ["mp-1", "ghost"]) + summary = await service.delete_projects({"id": str(group.id)}, ["mp-1", "ghost"]) assert summary.succeeded == ["mp-1"] assert len(summary.failed) == 1 assert summary.failed[0].index == 1 @@ -175,7 +180,7 @@ async def test_members_deleted_non_members_reported(self): async def test_no_members_skips_update(self): group = _group(["mp-1"]) service, groups, _ = _make_service(group) - summary = await service.delete_projects_by_id(str(group.id), ["ghost"]) + summary = await service.delete_projects({"id": str(group.id)}, ["ghost"]) assert summary.succeeded == [] assert len(summary.failed) == 1 groups.delete_project_refs.assert_not_awaited() diff --git a/mpcontribs-api/tests/unit/domains/test_shared_models.py b/mpcontribs-api/tests/unit/domains/test_shared_models.py index 959973eea5..b883731733 100644 --- a/mpcontribs-api/tests/unit/domains/test_shared_models.py +++ b/mpcontribs-api/tests/unit/domains/test_shared_models.py @@ -107,11 +107,12 @@ def test_serializes_under_id_not_underscore_id(self): class TestIdentifierContract: - def test_default_identifier_fields_is_primary_key(self): - # Content-addressed components fall back to the base default. - assert Attachment.identifier_fields() == frozenset({"id"}) + def test_component_uses_md5(self): + # Content-addressed components identify by their content hash. + assert Attachment.identifier_fields() == frozenset({"md5"}) def test_project_uses_id(self): + # Project inherits the base default (primary key) unchanged. assert Project.identifier_fields() == frozenset({"id"}) def test_project_group_uses_name_and_owner(self): @@ -125,7 +126,8 @@ def test_default_identifiers_reads_values_off_instance(self): oid = PydanticObjectId() doc = Attachment.from_input(_attachment_in()) doc.id = oid - assert doc.identifiers() == {"id": oid} + # identifiers() reads the declared identifier fields off the instance — md5 for components. + assert doc.identifiers() == {"md5": doc.md5} def test_contribution_identifiers_returns_natural_key_values(self): contrib = ContributionIn( From 4f28a5ebcd9f422115ec3644f40a7fe2c29927cc Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 19 Aug 2026 10:22:55 -0700 Subject: [PATCH 54/62] build(pyproject.toml): removed commented out packages --- mpcontribs-api/pyproject.toml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/mpcontribs-api/pyproject.toml b/mpcontribs-api/pyproject.toml index 6b89a5c763..e66fd3e176 100644 --- a/mpcontribs-api/pyproject.toml +++ b/mpcontribs-api/pyproject.toml @@ -32,13 +32,7 @@ authors = [ {name="The Materials Project", email="feedback@materialsproject.org"}, ] dependencies = [ - # "pint>=0.24", - # "psycopg2-binary", # "rq<=2.3.2", # see https://github.com/rq/Flask-RQ2/issues/620 - # "setproctitle", - # "uncertainties", - # "websocket_client", - # "zstandard", "aioboto3>=15.5.0", "beanie>=2.1.0", "fastapi[standard]>=0.136.3", From c72dda42693fe11739a26e7d0df106523cd266c7 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 19 Aug 2026 10:49:52 -0700 Subject: [PATCH 55/62] docs(MongoDbRepository): removed some long docstrings --- .../domains/_shared/repository.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py index 1219690abd..cda16a5e3b 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py @@ -75,14 +75,6 @@ def _convert_object_id(self, id: str) -> PydanticObjectId: def coerce_identifiers(self, identifiers: dict[str, Any]) -> dict[str, Any]: """Return ``identifiers`` with a string ``id`` coerced to the model's primary-key type. - Externally supplied ids arrive as strings (path/query params). ObjectId-keyed models need - that string parsed into a ``PydanticObjectId`` before it can match ``_id``; string-keyed - models (e.g. ``Project``, whose id is its name) and every non-``id`` identifier key pass - through untouched. Idempotent: an already-parsed id is returned unchanged. - - This is the single home for identifier coercion so services that resolve documents by id - (and reuse the parsed id for cross-repository lookups) do not each reimplement it. - Raises: ValidationError: if ``id`` is a string that is not a valid ObjectId, for an ObjectId-keyed model @@ -244,6 +236,15 @@ async def delete_by_ids(self, ids: list[Any], session: AsyncClientSession | None raise ValidationError("DeleteResult not returned internally") return DeleteResponse.from_delete_result(delete_result) + def _patch_update_fields(self, update: TPatch) -> dict[str, Any]: + """Map a patch model to the MongoDB ``$set`` field dict. + + Defaults to the patch's set fields (``exclude_unset``), which replaces each named field + wholesale. Subclasses whose patch targets a nested sub-document override this to emit dotted + ``parent.child`` keys so only the named leaves change and their siblings are left intact. + """ + return update.model_dump(exclude_unset=True) + async def _patch_matching( self, match: Any, @@ -257,7 +258,7 @@ async def _patch_matching( is a no-op that still returns the existing document; a missing target raises ``not_found``. """ # Only retain set fields (patch) - update_data = update.model_dump(exclude_unset=True) + update_data = self._patch_update_fields(update) existing = await self.document_model.find_one(self._scope, match, session=session) # If update is empty, return the model anyways (consistent behavior) if not update_data: From 22ca3d20081208283689dc40eab303ff8942e45e Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 19 Aug 2026 10:51:18 -0700 Subject: [PATCH 56/62] refactor(MongoDbContributionRepository): removed redundant subclass-specific methods in favor of parent class methods --- .../domains/consumers/dependencies.py | 4 +- .../domains/consumers/models.py | 5 ++ .../domains/consumers/repository.py | 67 +++---------------- .../domains/consumers/router.py | 2 +- .../db/test_consumers_repository.py | 11 +-- 5 files changed, 23 insertions(+), 66 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/consumers/dependencies.py b/mpcontribs-api/src/mpcontribs_api/domains/consumers/dependencies.py index ce53f76513..3e69d7bbb3 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/consumers/dependencies.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/consumers/dependencies.py @@ -24,8 +24,8 @@ async def get_effective_limits(user: UserDep) -> ConsumerSettings: if user.consumer_id is None: return ConsumerSettings() - override = await MongoDbConsumerRepository(user).get_by_consumer_id(user.consumer_id) - return override.settings if override is not None else ConsumerSettings() + override = await MongoDbConsumerRepository(user).get_one({"consumer_id": user.consumer_id}) + return override.settings if override and override.settings else ConsumerSettings() ConsumerLimitsDep = Annotated[ConsumerSettings, Depends(get_effective_limits)] diff --git a/mpcontribs-api/src/mpcontribs_api/domains/consumers/models.py b/mpcontribs-api/src/mpcontribs_api/domains/consumers/models.py index a4f8a543c0..6e6d218ae2 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/consumers/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/consumers/models.py @@ -52,6 +52,11 @@ class Consumer(BaseDocumentWithInput[PydanticObjectId]): consumer_id: str settings: ConsumerSettings = Field(default_factory=ConsumerSettings) + @classmethod + def identifier_fields(cls) -> frozenset[str]: + """A consumer override is keyed by Kong's ``consumer_id`` (its unique natural key).""" + return frozenset({"consumer_id"}) + @classmethod def with_defaults(cls, consumer_id: str = "") -> Consumer: """In-memory Consumer whose ``settings`` carry the env-backed default limits. diff --git a/mpcontribs-api/src/mpcontribs_api/domains/consumers/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/consumers/repository.py index 7d4f644d95..a1db8e2429 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/consumers/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/consumers/repository.py @@ -1,9 +1,5 @@ from typing import Any -from beanie import UpdateResponse -from beanie.operators import Set -from pymongo.asynchronous.client_session import AsyncClientSession - from mpcontribs_api.authz import User from mpcontribs_api.domains._shared.repository import MongoDbRepository from mpcontribs_api.domains.consumers.models import ( @@ -13,8 +9,6 @@ ConsumerOut, ConsumerPatch, ) -from mpcontribs_api.exceptions import ConflictError, NotFoundError -from mpcontribs_api.pagination import CursorParams class MongoDbConsumerRepository(MongoDbRepository[Consumer, ConsumerIn, ConsumerOut, ConsumerFilter, ConsumerPatch]): @@ -22,7 +16,9 @@ class MongoDbConsumerRepository(MongoDbRepository[Consumer, ConsumerIn, Consumer Consumer overrides are an admin-only resource: every route that reaches this repository is gated by ``require_admin``, so no per-user read scope is needed and ``_build_scope`` returns an - empty filter (admins see all overrides). + empty filter (admins see all overrides). Reads and deletes use the base repository directly + (keyed on ``consumer_id`` via ``Consumer.identifier_fields``); only the nested-``settings`` patch + shape is resource-specific. """ document_model = Consumer @@ -33,57 +29,12 @@ def _build_scope(user: User) -> dict[str, Any]: # Admin-only resource (routes enforce ``require_admin``); no visibility filter required. return {} - async def get_consumers( - self, - filter: ConsumerFilter, - pagination: CursorParams, - fields: frozenset[str] | None, - ): - """List consumer overrides. See ``get_many``.""" - return await self.get_many(pagination=pagination, filter=filter, fields=fields) - - async def get_by_consumer_id(self, consumer_id: str) -> Consumer | None: - """Return the override document for a Kong ``consumer_id``, or ``None`` if none exists.""" - return await Consumer.find_one(Consumer.consumer_id == consumer_id) + def _patch_update_fields(self, update: ConsumerPatch) -> dict[str, Any]: + """Flatten the patch to dotted ``settings.`` keys. - async def insert_one(self, in_resource: ConsumerIn) -> Consumer: - """Insert a new override, rejecting a duplicate ``consumer_id`` with a clean 409. - - The unique index on ``consumer_id`` is the hard guarantee; this pre-check turns the common - case into a readable conflict instead of a raw driver error. + The limits live under a nested ``settings`` sub-document; dotting the update makes a partial + patch change only the named limits and leave the siblings intact (a plain ``$set`` of + ``settings`` would replace the whole sub-document). """ - existing = await self.get_by_consumer_id(in_resource.consumer_id) - if existing is not None: - raise ConflictError( - "An override for this consumer already exists", - consumer_id=in_resource.consumer_id, - ) - return await super().insert_one(in_resource) - - async def patch_one( # pyright: ignore[reportIncompatibleMethodOverride] - self, identifiers: dict[str, Any], update: ConsumerPatch, session: AsyncClientSession | None = None - ) -> Consumer: - """Partially update an override's limits by identifiers. - - The limits live under a nested ``settings`` sub-document; the update is flattened to dotted - ``settings.`` keys so a partial patch changes only the named limits and leaves the - siblings intact (a plain ``$set`` of ``settings`` would replace the whole sub-document). - """ - match = self._identifier_query(identifiers) - not_found = NotFoundError(f"{self.document_model.__name__} not found", identifiers=identifiers) overrides = update.settings.model_dump(exclude_unset=True) if update.settings else {} - dotted = {f"settings.{field}": value for field, value in overrides.items()} - if not dotted: - # Empty patch is a no-op that still returns the existing document (consistent behavior). - existing = await self.document_model.find_one(self._scope, match, session=session) - if existing is None: - raise not_found - return existing - - updated = await self.document_model.find_one(self._scope, match, session=session).update( - Set(dotted), - response_type=UpdateResponse.NEW_DOCUMENT, - ) # pyright: ignore[reportGeneralTypeIssues] # beanie UpdateQuery is awaitable, but pyright doesn't see it - if updated is None: - raise not_found - return updated + return {f"settings.{field}": value for field, value in overrides.items()} diff --git a/mpcontribs-api/src/mpcontribs_api/domains/consumers/router.py b/mpcontribs-api/src/mpcontribs_api/domains/consumers/router.py index a6ebff8d76..b515422e16 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/consumers/router.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/consumers/router.py @@ -31,7 +31,7 @@ async def get_consumers( if fields is None: fields = list(ConsumerOut.default_fields()) selected = ConsumerOut.parse_fields(fields) - return await repo.get_consumers(filter=filter, pagination=pagination, fields=selected) + return await repo.get_many(filter=filter, pagination=pagination, fields=selected) @router.get("/{id}") diff --git a/mpcontribs-api/tests/integration/db/test_consumers_repository.py b/mpcontribs-api/tests/integration/db/test_consumers_repository.py index 99b7b14b6d..9abecdf0ba 100644 --- a/mpcontribs-api/tests/integration/db/test_consumers_repository.py +++ b/mpcontribs-api/tests/integration/db/test_consumers_repository.py @@ -25,14 +25,14 @@ def _repo() -> MongoDbConsumerRepository: # --------------------------------------------------------------------------- -# insert_one / get_by_consumer_id +# insert_one / get_one (by consumer_id) # --------------------------------------------------------------------------- class TestInsertAndLookup: async def test_insert_then_lookup_by_consumer_id(self, db): await _repo().insert_one(ConsumerIn(consumer_id="kong-1")) - found = await _repo().get_by_consumer_id("kong-1") + found = await _repo().get_one({"consumer_id": "kong-1"}) assert found is not None assert found.consumer_id == "kong-1" @@ -42,7 +42,7 @@ async def test_duplicate_consumer_id_raises_conflict(self, db): await _repo().insert_one(ConsumerIn(consumer_id="kong-dup")) async def test_lookup_missing_returns_none(self, db): - assert await _repo().get_by_consumer_id("kong-absent") is None + assert await _repo().get_one({"consumer_id": "kong-absent"}) is None async def test_partial_override_snapshots_defaults_for_siblings(self, db): # Admin overrides only max_projects; the stored document must carry a fully-resolved @@ -50,8 +50,9 @@ async def test_partial_override_snapshots_defaults_for_siblings(self, db): await _repo().insert_one( ConsumerIn(consumer_id="kong-partial", settings=ConsumerSettings(max_projects=1)) ) - stored = await _repo().get_by_consumer_id("kong-partial") + stored = await _repo().get_one({"consumer_id": "kong-partial"}) assert stored is not None + assert stored.settings is not None assert stored.settings.max_projects == 1 assert stored.settings.max_columns == get_settings().consumer.max_columns @@ -118,7 +119,7 @@ class TestDeleteConsumer: async def test_delete_removes_override(self, db): created = await _repo().insert_one(ConsumerIn(consumer_id="kong-del")) await _repo().delete_one({"id": created.id}) - assert await _repo().get_by_consumer_id("kong-del") is None + assert await _repo().get_one({"consumer_id": "kong-del"}) is None async def test_delete_missing_raises_not_found(self, db): from beanie import PydanticObjectId From 977672d2462e5be1adddff3281c4128f9775d99e Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 19 Aug 2026 11:13:01 -0700 Subject: [PATCH 57/62] refactor(Stats): removed empty() classmethod in favor of idiomatic pydantic field default declarations --- .../src/mpcontribs_api/domains/projects/models.py | 7 +------ .../tests/integration/db/test_projects_repository.py | 2 +- mpcontribs-api/tests/unit/domains/test_projects_models.py | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py index 6b6def1fc1..df10bb5702 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py @@ -51,11 +51,6 @@ class Stats(BaseModel): attachments: int = 0 size: float = 0 - @classmethod - def empty(cls) -> Stats: - """A zeroed rollup for a project with no contributions yet.""" - return cls(columns=0, contributions=0, tables=0, structures=0, attachments=0, size=0.0) - class Reference(BaseModel): # TODO: Labels have some restrictions, not sure exactly what yet @@ -103,7 +98,7 @@ class Project(ProjectBase, BaseDocumentWithInput[ShortStr]): """Document model of what is actually stored.""" # Server-owned: derived from the project's contributions - stats: Stats = Field(default_factory=Stats.empty) + stats: Stats = Field(default_factory=Stats) columns: list[Column] = Field(default_factory=list) @classmethod diff --git a/mpcontribs-api/tests/integration/db/test_projects_repository.py b/mpcontribs-api/tests/integration/db/test_projects_repository.py index ed945c6eec..f4761f4878 100644 --- a/mpcontribs-api/tests/integration/db/test_projects_repository.py +++ b/mpcontribs-api/tests/integration/db/test_projects_repository.py @@ -584,7 +584,7 @@ async def test_upsert_update_preserves_stats_and_columns(self, db): async def test_upsert_new_starts_with_empty_stats(self, db): await _repo(ALICE).upsert_one({"id": "srv-new-empty"}, _project_in("srv-new-empty")) found = await Project.find_one(Project.id == "srv-new-empty") - assert found.stats == Stats.empty() + assert found.stats == Stats() assert found.columns == [] async def test_non_admin_cannot_approve_new_project_via_upsert(self, db): diff --git a/mpcontribs-api/tests/unit/domains/test_projects_models.py b/mpcontribs-api/tests/unit/domains/test_projects_models.py index e17fb12fdc..5a35cbdac8 100644 --- a/mpcontribs-api/tests/unit/domains/test_projects_models.py +++ b/mpcontribs-api/tests/unit/domains/test_projects_models.py @@ -288,7 +288,7 @@ def test_from_input_model_defaults(self): def test_from_input_model_starts_with_empty_server_owned_fields(self): # stats/columns aren't on the input model and default empty on the document. project = Project.from_input_model(self._make_input(), id="test-proj") - assert project.stats == Stats.empty() + assert project.stats == Stats() assert project.columns == [] From d784e0a1da26d419ccaf2b0d4654841d3aafc750 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 19 Aug 2026 11:26:41 -0700 Subject: [PATCH 58/62] refactor(Project): removed repeated class Settings that improperly overrode the ProjectBase declaration --- mpcontribs-api/src/mpcontribs_api/domains/projects/models.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py index df10bb5702..5e4a510c33 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/models.py @@ -118,10 +118,6 @@ def decode_cursor(cursor: str) -> str: def server_managed_fields(cls) -> tuple: return ("is_public", "is_approved", "stats", "mp_category") - class Settings: - name = "projects" - keep_nulls = False - class ProjectOut(DocumentOut[ShortStr]): """Full response of all public-facing fields.""" From 3ed1bc1accc02aa86afb164e36327d0da85e2903 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 19 Aug 2026 11:29:03 -0700 Subject: [PATCH 59/62] refactor(MongoDbProjectRepository): removed patch_project_with_initiative in favor of a unified, general patch implementation --- .../domains/contributions/service.py | 4 +- .../domains/projects/dependencies.py | 2 +- .../domains/projects/repository.py | 38 +++++-------------- .../domains/projects/service.py | 3 +- 4 files changed, 14 insertions(+), 33 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/contributions/service.py b/mpcontribs-api/src/mpcontribs_api/domains/contributions/service.py index c2ab6df4de..f68773a7fc 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/contributions/service.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/contributions/service.py @@ -117,9 +117,7 @@ async def patch_one(self, identifiers: dict[str, Any], update: ContributionPatch async def upsert_one(self, identifiers: dict[str, Any], contribution: ContributionIn) -> Contribution: """Upsert the single scoped contribution matching ``identifiers``. See repository ``upsert_one``.""" - return await self._contributions.upsert_one( - self._contributions.coerce_identifiers(identifiers), contribution - ) + return await self._contributions.upsert_one(self._contributions.coerce_identifiers(identifiers), contribution) async def delete_one(self, identifiers: dict[str, Any]) -> BulkDeleteSummary: """Delete a single contribution and its child components, matching ``identifiers``. diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/dependencies.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/dependencies.py index 24ba2c0093..393e4759e7 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/dependencies.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/dependencies.py @@ -3,8 +3,8 @@ from fastapi import Depends from mpcontribs_api.dependencies import UserDep -from mpcontribs_api.domains.initiatives.repository import InitiativeRepository from mpcontribs_api.domains.consumers.dependencies import ConsumerLimitsDep +from mpcontribs_api.domains.initiatives.repository import InitiativeRepository from mpcontribs_api.domains.projects.repository import ( MongoDbProjectRepository, ) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py index 3566674e25..073d05cd58 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/repository.py @@ -1,8 +1,6 @@ from typing import Any -from beanie import PydanticObjectId, UpdateResponse -from beanie.operators import Set -from bson import DBRef +from beanie import PydanticObjectId from pymongo import UpdateOne from pymongo.asynchronous.client_session import AsyncClientSession @@ -146,7 +144,11 @@ async def insert_project(self, id: str, project: ProjectIn) -> Project: return document async def patch_one( # pyright: ignore[reportIncompatibleMethodOverride] - self, identifiers: dict[str, Any], update: ProjectPatch, session: AsyncClientSession | None = None + self, + identifiers: dict[str, Any], + update: ProjectPatch, + session: AsyncClientSession | None = None, + extra_set: dict[str, Any] | None = None, ) -> Project: """Partially update a scoped project by id, enforcing approval rules. @@ -154,32 +156,12 @@ async def patch_one( # pyright: ignore[reportIncompatibleMethodOverride] - Resulting state must satisfy is_public <-> is_approved condition The ``initiative`` field is split out upstream in ``ProjectService.patch_one``, so it never - reaches this method; an assignment that also edits plain fields goes through - :meth:`patch_project_with_initiative` instead. + reaches this method as a bare slug; an assignment that also edits plain fields arrives with + the resolved link passed through ``extra_set`` (``{"initiative": }``) and is + written together with the plain fields in the single ``$set``. """ await self._enforce_patch_rules(identifiers["id"], update) - return await super().patch_one(identifiers, update, session=session) - - async def patch_project_with_initiative(self, id: str, update: ProjectPatch, ref: DBRef | None) -> Project: - """Atomically apply a partial project update together with its canonical initiative link. - - Args: - id (str): the id of the project to update - update (ProjectPatch): the partial update to apply; unset fields are dropped. The - ``initiative`` slug must already be stripped — the resolved ``ref`` carries the link. - ref (DBRef | None): the initiative reference to assign, or None to unassign - """ - await self._enforce_patch_rules(id, update) - data = update.model_dump(exclude_unset=True) - data["initiative"] = ref - query = self.document_model.find_one(self._scope, self._identifier_query({"id": id})).update( - Set(data), - response_type=UpdateResponse.NEW_DOCUMENT, - ) - updated = await query # pyright: ignore[reportGeneralTypeIssues] # beanie UpdateQuery is awaitable - if updated is None: - raise NotFoundError(f"{self.document_model.__name__} not found", id=id) - return updated + return await super().patch_one(identifiers, update, session=session, extra_set=extra_set) async def _enforce_patch_rules(self, id: str, update: ProjectPatch) -> None: """Enforce project patch invariants against the scoped target. diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py index 067bea3a3b..adc4fe0ffd 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py @@ -41,7 +41,8 @@ async def patch_one(self, identifiers: dict[str, Any], update: ProjectPatch) -> # Resolve the target link (and run the both-rights + limit checks) before touching anything. ref = await self._resolve_initiative_assignment(project_id=id, slug=slug) - return await self._projects.patch_project_with_initiative(id=id, update=ProjectPatch(**data), ref=ref) + # `initiative` is server derived, so ProjectPatch can't handle it (expects str), so hand it in extra_set + return await self._projects.patch_one(identifiers, ProjectPatch(**data), extra_set={"initiative": ref}) async def get_one(self, identifiers: dict[str, Any], fields: frozenset[str] | None) -> Project | ProjectOut | None: """Return the single scoped project matching ``identifiers`` (``{"id": ...}``).""" From 5afc70da21349084e8629eee84d3e52077ca7f4e Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 19 Aug 2026 11:31:27 -0700 Subject: [PATCH 60/62] refactor(MongoDbRepository): added `extra_sets` as an argument in `patch_one` to handle server calculated fields that don't fit within *Patch (ie. Links) --- .../src/mpcontribs_api/domains/_shared/repository.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py index cda16a5e3b..baacda8e2f 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/_shared/repository.py @@ -251,14 +251,21 @@ async def _patch_matching( update: TPatch, not_found: NotFoundError, session: AsyncClientSession | None = None, + extra_set: dict[str, Any] | None = None, ) -> TDoc: """Apply a partial update to the single scoped document matching ``match``. ``match`` is any beanie filter that keys at most one in-scope document. An empty patch is a no-op that still returns the existing document; a missing target raises ``not_found``. + + ``extra_set`` carries server-resolved fields that the patch model cannot express — e.g. a + slug that a service has already resolved to a ``DBRef`` link. Its keys are merged into the + ``$set`` after the patch dump, so a non-empty ``extra_set`` also makes the update non-empty. """ # Only retain set fields (patch) update_data = self._patch_update_fields(update) + if extra_set: + update_data |= extra_set existing = await self.document_model.find_one(self._scope, match, session=session) # If update is empty, return the model anyways (consistent behavior) if not update_data: @@ -292,6 +299,7 @@ async def patch_one( identifiers: dict[str, Any], update: TPatch, session: AsyncClientSession | None = None, + extra_set: dict[str, Any] | None = None, ) -> TDoc: """Partially update the single scoped document matching ``identifiers``. @@ -299,10 +307,12 @@ async def patch_one( identifiers (dict[str, Any]): identifier field values keyed by ``identifier_fields`` update (TPatch): the partial update to apply; unset fields are dropped session (AsyncClientSession | None): optional client session for transactions + extra_set (dict[str, Any] | None): server-resolved fields to merge into the ``$set`` + alongside the patch — for values the patch model cannot carry (e.g. a resolved link) """ query = self._identifier_query(identifiers) not_found = NotFoundError(f"{self.document_model.__name__} not found", identifiers=identifiers) - return await self._patch_matching(query, update, not_found, session=session) + return await self._patch_matching(query, update, not_found, session=session, extra_set=extra_set) def _hash_payload(self, payload: dict[str, Any], *, separators: tuple[str, str] = (",", ":")) -> str: canonical = json.dumps( From 71b04b1fd4018acb5f5608676f16ac3e273d96e9 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 19 Aug 2026 12:03:14 -0700 Subject: [PATCH 61/62] fix(Initiative): added from_input_model and fixed return type of default_fields to tuple --- .../src/mpcontribs_api/domains/initiatives/models.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/models.py b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/models.py index 10cf63180f..3f5a3db923 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/models.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/models.py @@ -43,6 +43,10 @@ class Settings: ] validate_on_save = True + @classmethod + def from_input_model(cls, data: InitiativeIn, owner: PrefixedEmail) -> Self: # pyright: ignore[reportIncompatibleMethodOverride] + return cls(_id=PydanticObjectId(), **data.model_dump(), owner=owner) + @classmethod def identifier_fields(cls) -> frozenset[str]: """An ``Initiative`` is uniquely identified by its globally-unique ``slug``.""" @@ -77,8 +81,8 @@ class InitiativeOut(DocumentOut[PydanticObjectId]): is_approved: bool | None = None @staticmethod - def default_fields() -> list[str]: - return ["slug", "name", "owner", "is_public", "is_approved"] + def default_fields() -> tuple[str, ...]: + return ("slug", "name", "owner", "is_public", "is_approved") class InitiativePatch(SparseFieldsModel): From 12e5f727fe947579088cf86deb33aff4ac24a1c3 Mon Sep 17 00:00:00 2001 From: Brendan Foley Date: Wed, 19 Aug 2026 12:04:38 -0700 Subject: [PATCH 62/62] refactor(MongoDbInitiativeRepository): removed `is_visible` asa useless one-line abstraction and implemente from_input_model --- .../domains/initiatives/repository.py | 19 +++---------------- .../domains/projects/service.py | 8 ++++---- .../db/test_initiatives_service.py | 2 +- 3 files changed, 8 insertions(+), 21 deletions(-) diff --git a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py index 6c52d57304..8c3d8933bd 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/initiatives/repository.py @@ -1,6 +1,5 @@ from typing import Any -from beanie import PydanticObjectId from pymongo.asynchronous.client_session import AsyncClientSession from pymongo.errors import DuplicateKeyError @@ -51,10 +50,6 @@ async def get_initiatives( """Return a scoped, filtered, paginated page of initiatives. See ``get_many``.""" return await self.get_many(pagination=pagination, filter=filter, fields=fields) - async def resolve_visible(self, slug: str) -> Initiative | None: - """Return the full scoped initiative document for ``slug`` (or None), for write-path checks.""" - return await self.document_model.find_one(self._scope, self.document_model.slug == slug) - async def insert_initiative(self, data: InitiativeIn) -> Initiative: """Create an initiative owned by the caller, enforcing the per-owner unapproved quota. @@ -76,15 +71,7 @@ async def insert_initiative(self, data: InitiativeIn) -> Initiative: limit=self._limits.max_unapproved_per_owner, ) - # ``BaseDocumentWithInput`` makes ``id`` required (no auto-default), so mint the ObjectId here - initiative = self.document_model.model_validate( - { - "_id": PydanticObjectId(), - "slug": data.slug, - "name": data.name, - "owner": self._user.username, - } - ) + initiative = self.document_model.from_input_model(data=data, owner=self._user.username) try: await initiative.insert() except DuplicateKeyError as exc: # unique slug index @@ -105,7 +92,7 @@ async def patch_one( # pyright: ignore[reportIncompatibleMethodOverride] document, and the write itself is delegated to the base :meth:`MongoDbRepository.patch_one`. """ slug = identifiers["slug"] - existing = await self.resolve_visible(slug) + existing = await self.document_model.find_one(self._scope, self.document_model.slug == slug) if existing is None: raise NotFoundError("Initiative not found", slug=slug) if not ( @@ -137,7 +124,7 @@ async def delete_one( :meth:`MongoDbRepository.delete_one`. """ slug = identifiers["slug"] - existing = await self.resolve_visible(slug) + existing = await self.document_model.find_one(self._scope, self.document_model.slug == slug) if existing is None: raise NotFoundError("Initiative not found", slug=slug) if not (self._user.is_admin or existing.owner == self._user.username): diff --git a/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py b/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py index adc4fe0ffd..7ae294d3ad 100644 --- a/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py +++ b/mpcontribs-api/src/mpcontribs_api/domains/projects/service.py @@ -66,16 +66,16 @@ async def _resolve_initiative_assignment(self, project_id: str, slug: str | None if slug is None: return None - initiative = await self._initiatives.resolve_visible(slug) - if initiative is None: + initiative = await self._initiatives.get_one({"slug": slug}) + if initiative is None or initiative.id is None: raise NotFoundError("Initiative not found or not visible", slug=slug) user = self._initiatives._user - if not (user.can_manage(id=initiative.slug, resource="initiative") or initiative.owner == user.username): + if not (user.can_manage(id=slug, resource="initiative") or initiative.owner == user.username): raise PermissionError( message="user does not have adequate acceess to this resource", required_role="initiative-owner-collaborator-or-admin", - resource_id=initiative.slug, + resource_id=slug, ) if not initiative.is_approved: diff --git a/mpcontribs-api/tests/integration/db/test_initiatives_service.py b/mpcontribs-api/tests/integration/db/test_initiatives_service.py index 785516bd7d..2a59037d42 100644 --- a/mpcontribs-api/tests/integration/db/test_initiatives_service.py +++ b/mpcontribs-api/tests/integration/db/test_initiatives_service.py @@ -171,7 +171,7 @@ async def test_approved_initiative_has_no_member_cap(self, db, monkeypatch): await _insert_project(f"appr-proj-{i}", owner=ALICE_EMAIL) await _service(ALICE).patch_one({"id": f"appr-proj-{i}"}, ProjectPatch(initiative="init-approved")) count = await MongoDbProjectRepository(ADMIN).count_initiative_members( - initiative_id=(await InitiativeRepository(ADMIN).resolve_visible("init-approved")).id, # type: ignore[union-attr] + initiative_id=(await InitiativeRepository(ADMIN).get_one({"slug": "init-approved"})).id, # type: ignore[union-attr] exclude_project_id=None, ) assert count == cap + 2