From b1f40ded44751c42930699a86c5520ef9832a78f Mon Sep 17 00:00:00 2001 From: Mitchell Kotler Date: Wed, 8 Jul 2026 15:37:58 -0400 Subject: [PATCH 1/9] support multiple entitlements --- .../serverless/tests/test_error_handling.py | 2 +- documentcloud/core/tests.py | 3 +- .../documents/models/saved_search.py | 1 - documentcloud/documents/modifications.py | 1 - .../processing/tests/pipeline_tests/mocks.py | 4 +- documentcloud/documents/solr.py | 2 +- .../documents/tests/test_saved_searches.py | 1 - documentcloud/entities/tests/factories.py | 1 - documentcloud/organizations/models.py | 26 +-- .../organizations/tests/test_models.py | 165 +++++++++++++++++- requirements/base.txt | 2 +- requirements/local.txt | 2 +- requirements/production.txt | 2 +- tasks.py | 6 +- 14 files changed, 191 insertions(+), 27 deletions(-) diff --git a/documentcloud/common/serverless/tests/test_error_handling.py b/documentcloud/common/serverless/tests/test_error_handling.py index db26b324..23c0a8eb 100644 --- a/documentcloud/common/serverless/tests/test_error_handling.py +++ b/documentcloud/common/serverless/tests/test_error_handling.py @@ -11,6 +11,7 @@ # Third Party import pytest +from sharedmock.mock import SharedMock # DocumentCloud from documentcloud.common import redis_fields @@ -26,7 +27,6 @@ StorageHandler, Workspace, ) -from sharedmock.mock import SharedMock # Since redis is used in the SharedMock calls, it needs to be pickle-able # in order to be sent across the process boundary. FakeRedis and Mock's both diff --git a/documentcloud/core/tests.py b/documentcloud/core/tests.py index d38420b9..e6379117 100644 --- a/documentcloud/core/tests.py +++ b/documentcloud/core/tests.py @@ -1,4 +1,5 @@ # Django +from django.conf import settings from django.contrib.flatpages.models import FlatPage from django.contrib.sites.models import Site from django.db import transaction @@ -58,7 +59,7 @@ def sign(self, data): token = "testtoken" timestamp = int(time.time()) signature = hmac.new( - key="".encode("utf8"), + key=settings.MAILGUN_API_KEY.encode("utf8"), msg=f"{timestamp}{token}".encode("utf8"), digestmod=hashlib.sha256, ).hexdigest() diff --git a/documentcloud/documents/models/saved_search.py b/documentcloud/documents/models/saved_search.py index f071a2e2..d0777417 100644 --- a/documentcloud/documents/models/saved_search.py +++ b/documentcloud/documents/models/saved_search.py @@ -1,4 +1,3 @@ -# Standard Library # Django from django.db import models from django.utils.translation import gettext_lazy as _ diff --git a/documentcloud/documents/modifications.py b/documentcloud/documents/modifications.py index 1682dc58..cb183fbb 100644 --- a/documentcloud/documents/modifications.py +++ b/documentcloud/documents/modifications.py @@ -1,4 +1,3 @@ -# Standard Library # Django from django.db import transaction diff --git a/documentcloud/documents/processing/tests/pipeline_tests/mocks.py b/documentcloud/documents/processing/tests/pipeline_tests/mocks.py index ef933c6d..422d232d 100644 --- a/documentcloud/documents/processing/tests/pipeline_tests/mocks.py +++ b/documentcloud/documents/processing/tests/pipeline_tests/mocks.py @@ -8,8 +8,10 @@ from contextlib import ExitStack from unittest.mock import patch -# DocumentCloud +# Third Party from config import celery_app + +# DocumentCloud from documentcloud.common import path from documentcloud.common.serverless.utils import get_redis, initialize from documentcloud.documents.processing.tests.pipeline_tests.fake_pdf import FakePage diff --git a/documentcloud/documents/solr.py b/documentcloud/documents/solr.py index 58fdab09..2adb8936 100644 --- a/documentcloud/documents/solr.py +++ b/documentcloud/documents/solr.py @@ -33,10 +33,10 @@ # Third Party import pysolr import requests +from config import celery_app from dateutil.parser import parse # DocumentCloud -from config import celery_app from documentcloud.common import path from documentcloud.common.environment import storage from documentcloud.core.utils import grouper diff --git a/documentcloud/documents/tests/test_saved_searches.py b/documentcloud/documents/tests/test_saved_searches.py index 74889c1a..5e7100c3 100644 --- a/documentcloud/documents/tests/test_saved_searches.py +++ b/documentcloud/documents/tests/test_saved_searches.py @@ -2,7 +2,6 @@ # Django from rest_framework import status -# Third Party import pytest # DocumentCloud diff --git a/documentcloud/entities/tests/factories.py b/documentcloud/entities/tests/factories.py index febdbde2..b3c1c2de 100644 --- a/documentcloud/entities/tests/factories.py +++ b/documentcloud/entities/tests/factories.py @@ -1,6 +1,5 @@ # Standard Library -# Standard Library import datetime # Third Party diff --git a/documentcloud/organizations/models.py b/documentcloud/organizations/models.py index 131902b6..0af6ac62 100644 --- a/documentcloud/organizations/models.py +++ b/documentcloud/organizations/models.py @@ -91,8 +91,12 @@ def has_admin(self, user): return self.users.filter(pk=user.pk, memberships__admin=True).exists() def _update_resources(self, data, date_update): - # calc reqs/month in case it has changed - self.ai_credits_per_month = self.calc_ai_credits_per_month(data["max_users"]) + # sum AI credits across all entitlements + users = data["max_users"] + self.ai_credits_per_month = sum( + self._calc_ent_ai_credits(ent_data, users) + for ent_data in data["entitlements"] + ) # if date update has changed, then this is a monthly restore of the # subscription, and we should restore monthly AI credits. If not, AI credits @@ -109,8 +113,13 @@ def _update_resources(self, data, date_update): self.monthly_ai_credits = self.ai_credits_per_month self.date_update = date_update - def _choose_entitlement(self, entitlements): - return max(entitlements, key=lambda e: e["resources"].get("base_ai_credits", 0)) + @staticmethod + def _calc_ent_ai_credits(ent_data, users): + r = ent_data["resources"] + base = r.get("base_ai_credits", 0) + per_user = r.get("ai_credits_per_user", 0) + minimum = r.get("minimum_users", 1) + return base + max(0, users - minimum) * per_user @transaction.atomic def merge(self, uuid): @@ -145,15 +154,6 @@ def merge(self, uuid): self.merged = other - def calc_ai_credits_per_month(self, users): - """Calculate how many AI credits an organization gets per month on this plan - for a given number of users""" - return ( - self.entitlement.base_ai_credits - + (users - self.entitlement.minimum_users) - * self.entitlement.ai_credits_per_user - ) - @transaction.atomic def use_ai_credits(self, amount, user_id, note): """Try to deduct AI credits from the organization's balance diff --git a/documentcloud/organizations/tests/test_models.py b/documentcloud/organizations/tests/test_models.py index d61a4090..7a5ad0d9 100644 --- a/documentcloud/organizations/tests/test_models.py +++ b/documentcloud/organizations/tests/test_models.py @@ -1,14 +1,33 @@ +# Standard Library +from datetime import date + # Third Party import pytest # DocumentCloud from documentcloud.organizations.exceptions import InsufficientAICreditsError from documentcloud.organizations.models import Organization -from documentcloud.organizations.tests.factories import OrganizationFactory +from documentcloud.organizations.tests.factories import ( + EntitlementFactory, + OrganizationEntitlementFactory, + OrganizationFactory, + ProfessionalEntitlementFactory, +) from documentcloud.users.models import User from documentcloud.users.tests.factories import UserFactory +def ent_json(entitlement, date_update): + """Helper function for serializing entitlement data""" + return { + "name": entitlement.name, + "slug": entitlement.slug, + "description": entitlement.description, + "resources": entitlement.resources, + "date_update": date_update, + } + + class TestOrganization: @pytest.mark.django_db() @@ -66,6 +85,150 @@ def test_merge_fks(self): ) +class TestSquareletUpdateDataMultiEntitlement: + """Test cases for update_data with multiple entitlements""" + + def _org_data(self, organization, entitlements, max_users=5): + return { + "name": organization.name, + "slug": organization.slug, + "individual": False, + "private": False, + "entitlements": entitlements, + "max_users": max_users, + "card": "", + } + + @pytest.mark.django_db() + def test_two_paid_entitlements_sums_ai_credits(self): + """Two paid entitlements: ai_credits_per_month = sum of both""" + ent1 = ProfessionalEntitlementFactory() # base_ai_credits=2000, min=1 + ent2 = OrganizationEntitlementFactory() # base_ai_credits=5000, min=5 + organization = OrganizationFactory() + date_update = date(2024, 3, 1) + + organization.update_data( + self._org_data( + organization, + [ent_json(ent1, date_update), ent_json(ent2, date_update)], + max_users=5, + ) + ) + organization.refresh_from_db() + # Professional: 2000 + max(0, 5-1)*0 = 2000 + # Organization: 5000 + max(0, 5-5)*500 = 5000 + assert organization.ai_credits_per_month == 7000 + assert organization.monthly_ai_credits == 7000 + + @pytest.mark.django_db() + def test_paid_and_grant_entitlement_sums_ai_credits(self): + """Paid entitlement + grant entitlement: both contribute to total""" + paid = OrganizationEntitlementFactory() + grant = EntitlementFactory( + name="Grant", + resources={ + "minimum_users": 1, + "base_ai_credits": 500, + "ai_credits_per_user": 0, + "feature_level": 0, + }, + ) + organization = OrganizationFactory() + date_update = date(2024, 3, 1) + + organization.update_data( + self._org_data( + organization, + [ent_json(paid, date_update), ent_json(grant, date_update)], + max_users=5, + ) + ) + organization.refresh_from_db() + # Organization: 5000, Grant: 500 + assert organization.ai_credits_per_month == 5500 + assert organization.monthly_ai_credits == 5500 + + @pytest.mark.django_db() + def test_primary_entitlement_is_highest_feature_level(self): + """org.entitlement FK points to entitlement with highest feature_level""" + low = ProfessionalEntitlementFactory() # feature_level=1 + high = OrganizationEntitlementFactory() # feature_level=2 + organization = OrganizationFactory() + date_update = date(2024, 3, 1) + + organization.update_data( + self._org_data( + organization, + [ent_json(low, date_update), ent_json(high, date_update)], + ) + ) + organization.refresh_from_db() + assert organization.entitlement.slug == high.slug + + @pytest.mark.django_db() + def test_equal_feature_level_tie_breaks_to_first(self): + """Equal feature_level: first entitlement in list wins the FK""" + ent1 = EntitlementFactory( + name="GrantA", + resources={"base_ai_credits": 100, "feature_level": 1}, + ) + ent2 = EntitlementFactory( + name="GrantB", + resources={"base_ai_credits": 200, "feature_level": 1}, + ) + organization = OrganizationFactory() + date_update = date(2024, 3, 1) + + organization.update_data( + self._org_data( + organization, + [ent_json(ent1, date_update), ent_json(ent2, date_update)], + ) + ) + organization.refresh_from_db() + assert organization.entitlement.slug == ent1.slug + assert organization.ai_credits_per_month == 300 + + @pytest.mark.django_db() + def test_users_below_minimum_does_not_reduce_base(self): + """users < minimum_users: base AI credits are not reduced""" + ent = OrganizationEntitlementFactory() # min=5, base=5000, per_user=500 + organization = OrganizationFactory() + + organization.update_data( + self._org_data(organization, [ent_json(ent, date(2024, 3, 1))], max_users=2) + ) + organization.refresh_from_db() + # max(0, 2-5) = 0, so just base=5000 + assert organization.ai_credits_per_month == 5000 + + @pytest.mark.django_db() + def test_multi_entitlement_monthly_restore(self): + """Monthly restore resets monthly_ai_credits to sum of all entitlements""" + ent1 = ProfessionalEntitlementFactory() + ent2 = OrganizationEntitlementFactory() + organization = OrganizationFactory( + entitlement=ent2, + date_update=date(2024, 2, 1), + ai_credits_per_month=7000, + monthly_ai_credits=1000, + ) + + organization.update_data( + self._org_data( + organization, + [ + ent_json(ent1, date(2024, 3, 1)), + ent_json(ent2, date(2024, 3, 1)), + ], + max_users=5, + ) + ) + organization.refresh_from_db() + assert organization.ai_credits_per_month == 7000 + assert organization.monthly_ai_credits == 7000 + + class TestOrganizationCollective: """Tests for Organization collective resource sharing""" diff --git a/requirements/base.txt b/requirements/base.txt index 2660a3cb..bec5853c 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -451,7 +451,7 @@ sqlparse==0.5.4 # via # django # django-debug-toolbar -squarelet-auth==0.1.14 +squarelet-auth==0.1.15 # via -r requirements/base.in stack-data==0.3.0 # via ipython diff --git a/requirements/local.txt b/requirements/local.txt index 0e77b19f..e68d3b86 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -786,7 +786,7 @@ sqlparse==0.5.4 # -r requirements/base.txt # django # django-debug-toolbar -squarelet-auth==0.1.14 +squarelet-auth==0.1.15 # via -r requirements/base.txt stack-data==0.3.0 # via diff --git a/requirements/production.txt b/requirements/production.txt index 5c5da4a4..030a497d 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -622,7 +622,7 @@ sqlparse==0.5.4 # -r requirements/base.txt # django # django-debug-toolbar -squarelet-auth==0.1.14 +squarelet-auth==0.1.15 # via -r requirements/base.txt stack-data==0.3.0 # via diff --git a/tasks.py b/tasks.py index c9aa4e3b..bfdff7dc 100755 --- a/tasks.py +++ b/tasks.py @@ -120,9 +120,9 @@ def format(c): cmd="black documentcloud --exclude migrations && " "black config/urls.py && " "black config/settings && " - "isort documentcloud && " + "isort -rc documentcloud && " "isort config/urls.py && " - "isort config/settings" + "isort -rc config/settings" ) ) @@ -233,11 +233,13 @@ def download_tesseract_data(c): """Download Tesseract data files. Needed to be able to do OCR locally.""" c.run("cd config/aws/lambda; ./build.sh") + @task def initialize_minio(c): """Initialize Minio bucket and policies for local development""" c.run(DJANGO_RUN.format(cmd="python manage.py initialize_minio")) + @task def deploy_lambdas(c, staging=False): """Deploy lambda functions on AWS""" From 7b8011749be3143db518293cd2e2c175d7a839f9 Mon Sep 17 00:00:00 2001 From: Mitchell Kotler Date: Wed, 8 Jul 2026 15:56:31 -0400 Subject: [PATCH 2/9] isort --- .../common/serverless/tests/test_error_handling.py | 2 +- .../documents/processing/tests/pipeline_tests/mocks.py | 4 +--- documentcloud/documents/search.py | 2 +- documentcloud/documents/solr.py | 4 ++-- documentcloud/documents/tasks.py | 2 +- documentcloud/documents/tests/test_saved_searches.py | 1 + documentcloud/documents/tests/test_search.py | 2 +- documentcloud/documents/views.py | 2 +- documentcloud/entities/tests/factories.py | 1 + tasks.py | 6 +++--- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/documentcloud/common/serverless/tests/test_error_handling.py b/documentcloud/common/serverless/tests/test_error_handling.py index 23c0a8eb..db26b324 100644 --- a/documentcloud/common/serverless/tests/test_error_handling.py +++ b/documentcloud/common/serverless/tests/test_error_handling.py @@ -11,7 +11,6 @@ # Third Party import pytest -from sharedmock.mock import SharedMock # DocumentCloud from documentcloud.common import redis_fields @@ -27,6 +26,7 @@ StorageHandler, Workspace, ) +from sharedmock.mock import SharedMock # Since redis is used in the SharedMock calls, it needs to be pickle-able # in order to be sent across the process boundary. FakeRedis and Mock's both diff --git a/documentcloud/documents/processing/tests/pipeline_tests/mocks.py b/documentcloud/documents/processing/tests/pipeline_tests/mocks.py index 422d232d..ef933c6d 100644 --- a/documentcloud/documents/processing/tests/pipeline_tests/mocks.py +++ b/documentcloud/documents/processing/tests/pipeline_tests/mocks.py @@ -8,10 +8,8 @@ from contextlib import ExitStack from unittest.mock import patch -# Third Party -from config import celery_app - # DocumentCloud +from config import celery_app from documentcloud.common import path from documentcloud.common.serverless.utils import get_redis, initialize from documentcloud.documents.processing.tests.pipeline_tests.fake_pdf import FakePage diff --git a/documentcloud/documents/search.py b/documentcloud/documents/search.py index e8faf3a2..4583c3cf 100644 --- a/documentcloud/documents/search.py +++ b/documentcloud/documents/search.py @@ -9,12 +9,12 @@ from datetime import datetime # Third Party -import pysolr from luqum.parser import ParseError, parser from luqum.tree import BaseOperation, Boost, Group, Not, Prohibit, Range, Unary, Word from luqum.utils import LuceneTreeTransformer, LuceneTreeVisitor # DocumentCloud +import pysolr from documentcloud.core.pagination import CursorPagination, PageNumberPagination from documentcloud.documents.constants import DATA_KEY_REGEX from documentcloud.documents.models import Document diff --git a/documentcloud/documents/solr.py b/documentcloud/documents/solr.py index 2adb8936..a4faf172 100644 --- a/documentcloud/documents/solr.py +++ b/documentcloud/documents/solr.py @@ -31,12 +31,12 @@ import time # Third Party -import pysolr import requests -from config import celery_app from dateutil.parser import parse # DocumentCloud +import pysolr +from config import celery_app from documentcloud.common import path from documentcloud.common.environment import storage from documentcloud.core.utils import grouper diff --git a/documentcloud/documents/tasks.py b/documentcloud/documents/tasks.py index d3ac75de..efe201dd 100644 --- a/documentcloud/documents/tasks.py +++ b/documentcloud/documents/tasks.py @@ -12,11 +12,11 @@ from datetime import date # Third Party -import pysolr import redis from requests.exceptions import HTTPError, RequestException # DocumentCloud +import pysolr from documentcloud.common.environment import httpsub, storage from documentcloud.core.choices import Language from documentcloud.documents import entity_extraction, modifications, solr diff --git a/documentcloud/documents/tests/test_saved_searches.py b/documentcloud/documents/tests/test_saved_searches.py index 5e7100c3..74889c1a 100644 --- a/documentcloud/documents/tests/test_saved_searches.py +++ b/documentcloud/documents/tests/test_saved_searches.py @@ -2,6 +2,7 @@ # Django from rest_framework import status +# Third Party import pytest # DocumentCloud diff --git a/documentcloud/documents/tests/test_search.py b/documentcloud/documents/tests/test_search.py index 166bbadf..54b4b63b 100644 --- a/documentcloud/documents/tests/test_search.py +++ b/documentcloud/documents/tests/test_search.py @@ -8,13 +8,13 @@ from unittest.mock import Mock # Third Party -import pysolr import pytest import pytz from furl import furl from luqum.parser import parser # DocumentCloud +import pysolr from documentcloud.documents.choices import Access, Status from documentcloud.documents.models import Document from documentcloud.documents.search import ( diff --git a/documentcloud/documents/views.py b/documentcloud/documents/views.py index dd8c9254..b3df1d15 100644 --- a/documentcloud/documents/views.py +++ b/documentcloud/documents/views.py @@ -18,7 +18,6 @@ # Third Party import environ -import pysolr from django_filters import rest_framework as django_filters from drf_spectacular.openapi import OpenApiParameter from drf_spectacular.utils import OpenApiExample, OpenApiResponse, extend_schema @@ -27,6 +26,7 @@ from rest_flex_fields.utils import split_levels # DocumentCloud +import pysolr from documentcloud.addons.choices import Event from documentcloud.addons.models import AddOnEvent from documentcloud.common.environment import httpsub diff --git a/documentcloud/entities/tests/factories.py b/documentcloud/entities/tests/factories.py index b3c1c2de..febdbde2 100644 --- a/documentcloud/entities/tests/factories.py +++ b/documentcloud/entities/tests/factories.py @@ -1,5 +1,6 @@ # Standard Library +# Standard Library import datetime # Third Party diff --git a/tasks.py b/tasks.py index bfdff7dc..8f999993 100755 --- a/tasks.py +++ b/tasks.py @@ -117,12 +117,12 @@ def format(c): """Format your code""" c.run( DJANGO_RUN_USER.format( - cmd="black documentcloud --exclude migrations && " + cmd="sh -c 'black documentcloud --exclude migrations && " "black config/urls.py && " "black config/settings && " - "isort -rc documentcloud && " + "isort documentcloud && " "isort config/urls.py && " - "isort -rc config/settings" + "isort config/settings'" ) ) From 7b6d2d4814885b602484253777fe069fdcaaef76 Mon Sep 17 00:00:00 2001 From: Mitchell Kotler Date: Wed, 8 Jul 2026 16:26:10 -0400 Subject: [PATCH 3/9] isort --- .isort.cfg | 1 + documentcloud/documents/search.py | 2 +- documentcloud/documents/solr.py | 2 +- documentcloud/documents/tasks.py | 2 +- documentcloud/documents/tests/test_search.py | 2 +- documentcloud/documents/views.py | 2 +- 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index ee97e390..50935845 100755 --- a/.isort.cfg +++ b/.isort.cfg @@ -5,6 +5,7 @@ skip= skip_glob=*/migrations/* known_future_library=future known_django=django,celery,rest_framework +known_third_party=pysolr known_first_party=documentcloud indent=' ' multi_line_output=3 diff --git a/documentcloud/documents/search.py b/documentcloud/documents/search.py index 4583c3cf..e8faf3a2 100644 --- a/documentcloud/documents/search.py +++ b/documentcloud/documents/search.py @@ -9,12 +9,12 @@ from datetime import datetime # Third Party +import pysolr from luqum.parser import ParseError, parser from luqum.tree import BaseOperation, Boost, Group, Not, Prohibit, Range, Unary, Word from luqum.utils import LuceneTreeTransformer, LuceneTreeVisitor # DocumentCloud -import pysolr from documentcloud.core.pagination import CursorPagination, PageNumberPagination from documentcloud.documents.constants import DATA_KEY_REGEX from documentcloud.documents.models import Document diff --git a/documentcloud/documents/solr.py b/documentcloud/documents/solr.py index a4faf172..58fdab09 100644 --- a/documentcloud/documents/solr.py +++ b/documentcloud/documents/solr.py @@ -31,11 +31,11 @@ import time # Third Party +import pysolr import requests from dateutil.parser import parse # DocumentCloud -import pysolr from config import celery_app from documentcloud.common import path from documentcloud.common.environment import storage diff --git a/documentcloud/documents/tasks.py b/documentcloud/documents/tasks.py index efe201dd..d3ac75de 100644 --- a/documentcloud/documents/tasks.py +++ b/documentcloud/documents/tasks.py @@ -12,11 +12,11 @@ from datetime import date # Third Party +import pysolr import redis from requests.exceptions import HTTPError, RequestException # DocumentCloud -import pysolr from documentcloud.common.environment import httpsub, storage from documentcloud.core.choices import Language from documentcloud.documents import entity_extraction, modifications, solr diff --git a/documentcloud/documents/tests/test_search.py b/documentcloud/documents/tests/test_search.py index 54b4b63b..166bbadf 100644 --- a/documentcloud/documents/tests/test_search.py +++ b/documentcloud/documents/tests/test_search.py @@ -8,13 +8,13 @@ from unittest.mock import Mock # Third Party +import pysolr import pytest import pytz from furl import furl from luqum.parser import parser # DocumentCloud -import pysolr from documentcloud.documents.choices import Access, Status from documentcloud.documents.models import Document from documentcloud.documents.search import ( diff --git a/documentcloud/documents/views.py b/documentcloud/documents/views.py index b3df1d15..dd8c9254 100644 --- a/documentcloud/documents/views.py +++ b/documentcloud/documents/views.py @@ -18,6 +18,7 @@ # Third Party import environ +import pysolr from django_filters import rest_framework as django_filters from drf_spectacular.openapi import OpenApiParameter from drf_spectacular.utils import OpenApiExample, OpenApiResponse, extend_schema @@ -26,7 +27,6 @@ from rest_flex_fields.utils import split_levels # DocumentCloud -import pysolr from documentcloud.addons.choices import Event from documentcloud.addons.models import AddOnEvent from documentcloud.common.environment import httpsub From 169d9167e77a1544e3b7f328f0853a1f88225399 Mon Sep 17 00:00:00 2001 From: Mitchell Kotler Date: Thu, 9 Jul 2026 11:29:30 -0400 Subject: [PATCH 4/9] use quantity from entitlement instead of max_users --- documentcloud/organizations/models.py | 13 ++-- .../organizations/tests/test_models.py | 60 +++++++++++++------ requirements/base.txt | 2 +- requirements/local.txt | 2 +- requirements/production.txt | 2 +- 5 files changed, 55 insertions(+), 24 deletions(-) diff --git a/documentcloud/organizations/models.py b/documentcloud/organizations/models.py index 0af6ac62..1f0f9966 100644 --- a/documentcloud/organizations/models.py +++ b/documentcloud/organizations/models.py @@ -92,9 +92,8 @@ def has_admin(self, user): def _update_resources(self, data, date_update): # sum AI credits across all entitlements - users = data["max_users"] self.ai_credits_per_month = sum( - self._calc_ent_ai_credits(ent_data, users) + self._calc_ent_ai_credits(ent_data) for ent_data in data["entitlements"] ) @@ -114,12 +113,18 @@ def _update_resources(self, data, date_update): self.date_update = date_update @staticmethod - def _calc_ent_ai_credits(ent_data, users): + def _calc_ent_ai_credits(ent_data): + """Compute the AI credit quota for one entitlement entry. + + Uses ent_data["quantity"] (the per-subscription billing quantity sent + by Squarelet) rather than the stale org-level max_users field. + """ r = ent_data["resources"] base = r.get("base_ai_credits", 0) per_user = r.get("ai_credits_per_user", 0) minimum = r.get("minimum_users", 1) - return base + max(0, users - minimum) * per_user + quantity = ent_data.get("quantity", 1) + return base + max(0, quantity - minimum) * per_user @transaction.atomic def merge(self, uuid): diff --git a/documentcloud/organizations/tests/test_models.py b/documentcloud/organizations/tests/test_models.py index 7a5ad0d9..2b5ddcdf 100644 --- a/documentcloud/organizations/tests/test_models.py +++ b/documentcloud/organizations/tests/test_models.py @@ -17,7 +17,7 @@ from documentcloud.users.tests.factories import UserFactory -def ent_json(entitlement, date_update): +def ent_json(entitlement, date_update, quantity=1): """Helper function for serializing entitlement data""" return { "name": entitlement.name, @@ -25,6 +25,7 @@ def ent_json(entitlement, date_update): "description": entitlement.description, "resources": entitlement.resources, "date_update": date_update, + "quantity": quantity, } @@ -88,14 +89,13 @@ def test_merge_fks(self): class TestSquareletUpdateDataMultiEntitlement: """Test cases for update_data with multiple entitlements""" - def _org_data(self, organization, entitlements, max_users=5): + def _org_data(self, organization, entitlements): return { "name": organization.name, "slug": organization.slug, "individual": False, "private": False, "entitlements": entitlements, - "max_users": max_users, "card": "", } @@ -110,12 +110,14 @@ def test_two_paid_entitlements_sums_ai_credits(self): organization.update_data( self._org_data( organization, - [ent_json(ent1, date_update), ent_json(ent2, date_update)], - max_users=5, + [ + ent_json(ent1, date_update, quantity=1), + ent_json(ent2, date_update, quantity=5), + ], ) ) organization.refresh_from_db() - # Professional: 2000 + max(0, 5-1)*0 = 2000 + # Professional: 2000 + max(0, 1-1)*0 = 2000 # Organization: 5000 + max(0, 5-5)*500 = 5000 assert organization.ai_credits_per_month == 7000 assert organization.monthly_ai_credits == 7000 @@ -139,12 +141,14 @@ def test_paid_and_grant_entitlement_sums_ai_credits(self): organization.update_data( self._org_data( organization, - [ent_json(paid, date_update), ent_json(grant, date_update)], - max_users=5, + [ + ent_json(paid, date_update, quantity=5), + ent_json(grant, date_update, quantity=1), + ], ) ) organization.refresh_from_db() - # Organization: 5000, Grant: 500 + # Organization: 5000 + max(0, 5-5)*500 = 5000, Grant: 500 assert organization.ai_credits_per_month == 5500 assert organization.monthly_ai_credits == 5500 @@ -159,7 +163,10 @@ def test_primary_entitlement_is_highest_feature_level(self): organization.update_data( self._org_data( organization, - [ent_json(low, date_update), ent_json(high, date_update)], + [ + ent_json(low, date_update, quantity=1), + ent_json(high, date_update, quantity=5), + ], ) ) organization.refresh_from_db() @@ -182,7 +189,10 @@ def test_equal_feature_level_tie_breaks_to_first(self): organization.update_data( self._org_data( organization, - [ent_json(ent1, date_update), ent_json(ent2, date_update)], + [ + ent_json(ent1, date_update, quantity=1), + ent_json(ent2, date_update, quantity=1), + ], ) ) organization.refresh_from_db() @@ -190,18 +200,35 @@ def test_equal_feature_level_tie_breaks_to_first(self): assert organization.ai_credits_per_month == 300 @pytest.mark.django_db() - def test_users_below_minimum_does_not_reduce_base(self): - """users < minimum_users: base AI credits are not reduced""" + def test_quantity_below_minimum_does_not_reduce_base(self): + """quantity < minimum_users: base AI credits are not reduced""" ent = OrganizationEntitlementFactory() # min=5, base=5000, per_user=500 organization = OrganizationFactory() organization.update_data( - self._org_data(organization, [ent_json(ent, date(2024, 3, 1))], max_users=2) + self._org_data( + organization, [ent_json(ent, date(2024, 3, 1), quantity=2)] + ) ) organization.refresh_from_db() # max(0, 2-5) = 0, so just base=5000 assert organization.ai_credits_per_month == 5000 + @pytest.mark.django_db() + def test_quantity_above_minimum_adds_per_user_credits(self): + """quantity > minimum_users: extra quantity adds per-user AI credits""" + ent = OrganizationEntitlementFactory() # min=5, base=5000, per_user=500 + organization = OrganizationFactory() + + organization.update_data( + self._org_data( + organization, [ent_json(ent, date(2024, 3, 1), quantity=8)] + ) + ) + organization.refresh_from_db() + # 5000 + max(0, 8-5)*500 = 5000 + 1500 = 6500 + assert organization.ai_credits_per_month == 6500 + @pytest.mark.django_db() def test_multi_entitlement_monthly_restore(self): """Monthly restore resets monthly_ai_credits to sum of all entitlements""" @@ -218,10 +245,9 @@ def test_multi_entitlement_monthly_restore(self): self._org_data( organization, [ - ent_json(ent1, date(2024, 3, 1)), - ent_json(ent2, date(2024, 3, 1)), + ent_json(ent1, date(2024, 3, 1), quantity=1), + ent_json(ent2, date(2024, 3, 1), quantity=5), ], - max_users=5, ) ) organization.refresh_from_db() diff --git a/requirements/base.txt b/requirements/base.txt index bec5853c..9c09ccc4 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -451,7 +451,7 @@ sqlparse==0.5.4 # via # django # django-debug-toolbar -squarelet-auth==0.1.15 +squarelet-auth==0.1.16 # via -r requirements/base.in stack-data==0.3.0 # via ipython diff --git a/requirements/local.txt b/requirements/local.txt index e68d3b86..3482d582 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -786,7 +786,7 @@ sqlparse==0.5.4 # -r requirements/base.txt # django # django-debug-toolbar -squarelet-auth==0.1.15 +squarelet-auth==0.1.16 # via -r requirements/base.txt stack-data==0.3.0 # via diff --git a/requirements/production.txt b/requirements/production.txt index 030a497d..e9dd130b 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -622,7 +622,7 @@ sqlparse==0.5.4 # -r requirements/base.txt # django # django-debug-toolbar -squarelet-auth==0.1.15 +squarelet-auth==0.1.16 # via -r requirements/base.txt stack-data==0.3.0 # via From 607ef80c775aae9434f83e499fb27d24fe30ceb0 Mon Sep 17 00:00:00 2001 From: Mitchell Kotler Date: Thu, 9 Jul 2026 13:16:35 -0400 Subject: [PATCH 5/9] format --- documentcloud/organizations/models.py | 3 +-- documentcloud/organizations/tests/test_models.py | 8 ++------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/documentcloud/organizations/models.py b/documentcloud/organizations/models.py index 1f0f9966..a4261b42 100644 --- a/documentcloud/organizations/models.py +++ b/documentcloud/organizations/models.py @@ -93,8 +93,7 @@ def has_admin(self, user): def _update_resources(self, data, date_update): # sum AI credits across all entitlements self.ai_credits_per_month = sum( - self._calc_ent_ai_credits(ent_data) - for ent_data in data["entitlements"] + self._calc_ent_ai_credits(ent_data) for ent_data in data["entitlements"] ) # if date update has changed, then this is a monthly restore of the diff --git a/documentcloud/organizations/tests/test_models.py b/documentcloud/organizations/tests/test_models.py index 2b5ddcdf..9916230c 100644 --- a/documentcloud/organizations/tests/test_models.py +++ b/documentcloud/organizations/tests/test_models.py @@ -206,9 +206,7 @@ def test_quantity_below_minimum_does_not_reduce_base(self): organization = OrganizationFactory() organization.update_data( - self._org_data( - organization, [ent_json(ent, date(2024, 3, 1), quantity=2)] - ) + self._org_data(organization, [ent_json(ent, date(2024, 3, 1), quantity=2)]) ) organization.refresh_from_db() # max(0, 2-5) = 0, so just base=5000 @@ -221,9 +219,7 @@ def test_quantity_above_minimum_adds_per_user_credits(self): organization = OrganizationFactory() organization.update_data( - self._org_data( - organization, [ent_json(ent, date(2024, 3, 1), quantity=8)] - ) + self._org_data(organization, [ent_json(ent, date(2024, 3, 1), quantity=8)]) ) organization.refresh_from_db() # 5000 + max(0, 8-5)*500 = 5000 + 1500 = 6500 From eac85a4a83faeb6b75da594124ff3883f6baaa48 Mon Sep 17 00:00:00 2001 From: duckduckgrayduck <102841251+duckduckgrayduck@users.noreply.github.com> Date: Mon, 13 Jul 2026 13:00:40 -0500 Subject: [PATCH 6/9] Add documentation about setting up local OCR data --- README.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7b8f77de..e22d6ac1 100644 --- a/README.md +++ b/README.md @@ -57,13 +57,23 @@ You must first have these set up and ready to go: 12. Go to [Django admin for DocumentCloud](https://api.dev.documentcloud.org/admin) and add the required static [flat page](https://api.dev.documentcloud.org/admin/flatpages/flatpage/) called `/tipofday/`. It can be blank. Do not prefix the URL with `/pages/`. Specifying the `Site` as `example.com` is alright. 13. Create an initial Minio bucket to simulate AWS S3 locally: - Run `inv initialize-minio` -14. Upload a document: +14. Download and setup Tesseract language data locally so that OCR works. + From within documentcloud/documents/processing/ocr/tesseract/tessdata/ run the following to download the trained data for the English language: + ```curl -L -o eng.traineddata https://github.com/tesseract-ocr/tessdata_fast/raw/main/eng.traineddata``` + You'll also need to install the orientation and script detection data: + ```curl -L -o osd.traineddata https://github.com/tesseract-ocr/tessdata_fast/raw/main/osd.traineddata``` + Lastly, you need to use the invoke command `inv manage upload_languages` to take the trained data from that directory and + place it in the correct location in the Minio bucket to test OCR in a local development environment. You may add additional languages as needed by modifying the + curl command for the language data above. +15. Upload a document: - **Check your memory allocation on Docker is at least 7gb.** A sign that you do not have enough memory allocated is if containers are randomly failing or if your system is swapping heavily, especially when uploading documents. - The "upload" button should not be grayed out (if it is, check your user organization Verified Journalist status above) - If you get an error on your console about signatures, fix minio as above. - If you get an error on your console about tipofday not found, add the static page as above. -15. Develop DocumentCloud and its frontend! -16. You can run the tests with `inv test`. +16. Develop DocumentCloud and its frontend! +17. Tests can be run using `inv test`. If a test doesn't exist for new functionality, please create tests for the new functionality to ensure test coverage. +Before submitting a pull request, ensure all code passes quality and formatting checks by running `inv format` and `inv pylint` and resolving any issues. + - If you want to run a subset of the tests, you can specify the directory containing the test you want with the `path` switch like so: `inv test --path documentcloud/documents`. - You can specify a single file in `--path` if you only want to run the tests in that file. From 616975c584a61ba562f7614ac451c5c3a6ef0eaf Mon Sep 17 00:00:00 2001 From: duckduckgrayduck <102841251+duckduckgrayduck@users.noreply.github.com> Date: Mon, 13 Jul 2026 13:03:16 -0500 Subject: [PATCH 7/9] Formatting attempt --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e22d6ac1..75383701 100644 --- a/README.md +++ b/README.md @@ -58,11 +58,11 @@ You must first have these set up and ready to go: 13. Create an initial Minio bucket to simulate AWS S3 locally: - Run `inv initialize-minio` 14. Download and setup Tesseract language data locally so that OCR works. - From within documentcloud/documents/processing/ocr/tesseract/tessdata/ run the following to download the trained data for the English language: + - From within documentcloud/documents/processing/ocr/tesseract/tessdata/ run the following to download the trained data for the English language: ```curl -L -o eng.traineddata https://github.com/tesseract-ocr/tessdata_fast/raw/main/eng.traineddata``` - You'll also need to install the orientation and script detection data: + - You'll also need to install the orientation and script detection data: ```curl -L -o osd.traineddata https://github.com/tesseract-ocr/tessdata_fast/raw/main/osd.traineddata``` - Lastly, you need to use the invoke command `inv manage upload_languages` to take the trained data from that directory and + - Lastly, you need to use the invoke command `inv manage upload_languages` to take the trained data from that directory and place it in the correct location in the Minio bucket to test OCR in a local development environment. You may add additional languages as needed by modifying the curl command for the language data above. 15. Upload a document: From 8d3f835709c6b90b3b830a49573f1f407842e43f Mon Sep 17 00:00:00 2001 From: duckduckgrayduck <102841251+duckduckgrayduck@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:45:34 -0500 Subject: [PATCH 8/9] Add dismissed field to AddOnEvents --- .../migrations/0031_addonevent_dismissed.py | 22 +++++++++++++++++++ documentcloud/addons/models.py | 9 ++++++++ documentcloud/addons/serializers.py | 1 + documentcloud/addons/views.py | 4 ++++ tasks.py | 10 +++++++++ 5 files changed, 46 insertions(+) create mode 100644 documentcloud/addons/migrations/0031_addonevent_dismissed.py diff --git a/documentcloud/addons/migrations/0031_addonevent_dismissed.py b/documentcloud/addons/migrations/0031_addonevent_dismissed.py new file mode 100644 index 00000000..5b6f3de2 --- /dev/null +++ b/documentcloud/addons/migrations/0031_addonevent_dismissed.py @@ -0,0 +1,22 @@ +# Generated by Django 5.2.13 on 2026-07-20 21:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("addons", "0030_addonevent_site_indexes"), + ] + + operations = [ + migrations.AddField( + model_name="addonevent", + name="dismissed", + field=models.BooleanField( + default=False, + help_text="If this event has been dismissed from view and should no longer be shown to the user", + verbose_name="dismissed", + ), + ), + ] diff --git a/documentcloud/addons/models.py b/documentcloud/addons/models.py index cfd1a2a6..8f67f312 100644 --- a/documentcloud/addons/models.py +++ b/documentcloud/addons/models.py @@ -601,6 +601,15 @@ class AddOnEvent(models.Model): help_text=_("Timestamp of when the add-on event was last updated"), ) + dismissed = models.BooleanField( + _("dismissed"), + default=False, + help_text=_( + "If this event has been dismissed from view and should no longer be " + "shown to the user" + ), + ) + class Meta: indexes = [ models.Index( diff --git a/documentcloud/addons/serializers.py b/documentcloud/addons/serializers.py index 2a355946..7b02ebb9 100644 --- a/documentcloud/addons/serializers.py +++ b/documentcloud/addons/serializers.py @@ -245,6 +245,7 @@ class Meta: "scratch", "created_at", "updated_at", + "dismissed", ] extra_kwargs = { "addon": {"queryset": AddOn.objects.none()}, diff --git a/documentcloud/addons/views.py b/documentcloud/addons/views.py index 4c2ef374..eb76e8dc 100644 --- a/documentcloud/addons/views.py +++ b/documentcloud/addons/views.py @@ -1043,6 +1043,10 @@ class Filter(django_filters.FilterSet): "`https://www.nifc.gov`." ), ) + dismissed = django_filters.BooleanFilter( + field_name="dismissed", + help_text="Filter events by whether they have been dismissed.", + ) def site_filter(self, queryset, name, value): # pylint: disable=unused-argument diff --git a/tasks.py b/tasks.py index 8f999993..c5d28f44 100755 --- a/tasks.py +++ b/tasks.py @@ -14,6 +14,16 @@ ) WEB_OPEN = "xdg-open {} > /dev/null 2>&1" +@task +def up(c): + """Start the docker images""" + c.run("docker compose up -d") + + +@task +def down(c): + """Shut down the docker images""" + c.run(f"docker compose down") @task def test( From 104fe192de0b909954eb983cdfdca4086c2ecf45 Mon Sep 17 00:00:00 2001 From: duckduckgrayduck <102841251+duckduckgrayduck@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:55:02 -0500 Subject: [PATCH 9/9] Add some tests --- documentcloud/addons/tests/test_views.py | 80 ++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/documentcloud/addons/tests/test_views.py b/documentcloud/addons/tests/test_views.py index f52f514c..232ee91d 100644 --- a/documentcloud/addons/tests/test_views.py +++ b/documentcloud/addons/tests/test_views.py @@ -438,3 +438,83 @@ def test_filter_message_absent_is_noop(self, client): response = client.get("/api/addon_runs/") assert response.status_code == status.HTTP_200_OK assert len(response.json()["results"]) == 3 + + def test_default_not_dismissed(self): + """A newly created event is not dismissed""" + event = AddOnEventFactory() + assert event.dismissed is False + + def test_dismissed_in_serializer(self, client): + """The dismissed field is exposed on the event""" + event = AddOnEventFactory() + client.force_authenticate(user=event.user) + response = client.get(f"/api/addon_events/{event.pk}/") + assert response.status_code == status.HTTP_200_OK + assert response.json()["dismissed"] is False + + def test_dismiss(self, client): + """The owner can dismiss their event""" + event = AddOnEventFactory(dismissed=False) + client.force_authenticate(user=event.user) + response = client.patch( + f"/api/addon_events/{event.pk}/", {"dismissed": True}, format="json" + ) + assert response.status_code == status.HTTP_200_OK + assert response.json()["dismissed"] is True + event.refresh_from_db() + assert event.dismissed is True + + def test_undismiss(self, client): + """The owner can un-dismiss their event""" + event = AddOnEventFactory(dismissed=True) + client.force_authenticate(user=event.user) + response = client.patch( + f"/api/addon_events/{event.pk}/", {"dismissed": False}, format="json" + ) + assert response.status_code == status.HTTP_200_OK + event.refresh_from_db() + assert event.dismissed is False + + def test_dismiss_non_owner(self, client): + """A non-owner can't see the event, so the patch 404s and nothing changes""" + event = AddOnEventFactory(dismissed=False) + other = UserFactory() + client.force_authenticate(user=other) + response = client.patch( + f"/api/addon_events/{event.pk}/", {"dismissed": True}, format="json" + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + event.refresh_from_db() + assert event.dismissed is False + + def test_filter_dismissed_false(self, client): + """?dismissed=false returns only non-dismissed events""" + user = UserFactory() + visible = AddOnEventFactory(user=user, dismissed=False) + AddOnEventFactory(user=user, dismissed=True) + client.force_authenticate(user=user) + response = client.get("/api/addon_events/", {"dismissed": "false"}) + assert response.status_code == status.HTTP_200_OK + ids = [r["id"] for r in response.json()["results"]] + assert ids == [visible.pk] + + def test_filter_dismissed_true(self, client): + """?dismissed=true returns only dismissed events""" + user = UserFactory() + AddOnEventFactory(user=user, dismissed=False) + dismissed = AddOnEventFactory(user=user, dismissed=True) + client.force_authenticate(user=user) + response = client.get("/api/addon_events/", {"dismissed": "true"}) + assert response.status_code == status.HTTP_200_OK + ids = [r["id"] for r in response.json()["results"]] + assert ids == [dismissed.pk] + + def test_filter_dismissed_absent_is_noop(self, client): + """Omitting the dismissed filter returns all viewable events""" + user = UserFactory() + AddOnEventFactory(user=user, dismissed=False) + AddOnEventFactory(user=user, dismissed=True) + client.force_authenticate(user=user) + response = client.get("/api/addon_events/") + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["results"]) == 2