diff --git a/.isort.cfg b/.isort.cfg index ee97e390..50935845 100755 --- a/.isort.cfg +++ b/.isort.cfg @@ -5,6 +5,7 @@ skip= skip_glob=*/migrations/* known_future_library=future known_django=django,celery,rest_framework +known_third_party=pysolr known_first_party=documentcloud indent=' ' multi_line_output=3 diff --git a/README.md b/README.md index 7b8f77de..75383701 100644 --- a/README.md +++ b/README.md @@ -57,13 +57,23 @@ You must first have these set up and ready to go: 12. Go to [Django admin for DocumentCloud](https://api.dev.documentcloud.org/admin) and add the required static [flat page](https://api.dev.documentcloud.org/admin/flatpages/flatpage/) called `/tipofday/`. It can be blank. Do not prefix the URL with `/pages/`. Specifying the `Site` as `example.com` is alright. 13. Create an initial Minio bucket to simulate AWS S3 locally: - Run `inv initialize-minio` -14. Upload a document: +14. Download and setup Tesseract language data locally so that OCR works. + - From within documentcloud/documents/processing/ocr/tesseract/tessdata/ run the following to download the trained data for the English language: + ```curl -L -o eng.traineddata https://github.com/tesseract-ocr/tessdata_fast/raw/main/eng.traineddata``` + - You'll also need to install the orientation and script detection data: + ```curl -L -o osd.traineddata https://github.com/tesseract-ocr/tessdata_fast/raw/main/osd.traineddata``` + - Lastly, you need to use the invoke command `inv manage upload_languages` to take the trained data from that directory and + place it in the correct location in the Minio bucket to test OCR in a local development environment. You may add additional languages as needed by modifying the + curl command for the language data above. +15. Upload a document: - **Check your memory allocation on Docker is at least 7gb.** A sign that you do not have enough memory allocated is if containers are randomly failing or if your system is swapping heavily, especially when uploading documents. - The "upload" button should not be grayed out (if it is, check your user organization Verified Journalist status above) - If you get an error on your console about signatures, fix minio as above. - If you get an error on your console about tipofday not found, add the static page as above. -15. Develop DocumentCloud and its frontend! -16. You can run the tests with `inv test`. +16. Develop DocumentCloud and its frontend! +17. Tests can be run using `inv test`. If a test doesn't exist for new functionality, please create tests for the new functionality to ensure test coverage. +Before submitting a pull request, ensure all code passes quality and formatting checks by running `inv format` and `inv pylint` and resolving any issues. + - If you want to run a subset of the tests, you can specify the directory containing the test you want with the `path` switch like so: `inv test --path documentcloud/documents`. - You can specify a single file in `--path` if you only want to run the tests in that file. diff --git a/documentcloud/addons/migrations/0031_addonevent_dismissed.py b/documentcloud/addons/migrations/0031_addonevent_dismissed.py new file mode 100644 index 00000000..5b6f3de2 --- /dev/null +++ b/documentcloud/addons/migrations/0031_addonevent_dismissed.py @@ -0,0 +1,22 @@ +# Generated by Django 5.2.13 on 2026-07-20 21:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("addons", "0030_addonevent_site_indexes"), + ] + + operations = [ + migrations.AddField( + model_name="addonevent", + name="dismissed", + field=models.BooleanField( + default=False, + help_text="If this event has been dismissed from view and should no longer be shown to the user", + verbose_name="dismissed", + ), + ), + ] diff --git a/documentcloud/addons/models.py b/documentcloud/addons/models.py index cfd1a2a6..8f67f312 100644 --- a/documentcloud/addons/models.py +++ b/documentcloud/addons/models.py @@ -601,6 +601,15 @@ class AddOnEvent(models.Model): help_text=_("Timestamp of when the add-on event was last updated"), ) + dismissed = models.BooleanField( + _("dismissed"), + default=False, + help_text=_( + "If this event has been dismissed from view and should no longer be " + "shown to the user" + ), + ) + class Meta: indexes = [ models.Index( diff --git a/documentcloud/addons/serializers.py b/documentcloud/addons/serializers.py index 2a355946..7b02ebb9 100644 --- a/documentcloud/addons/serializers.py +++ b/documentcloud/addons/serializers.py @@ -245,6 +245,7 @@ class Meta: "scratch", "created_at", "updated_at", + "dismissed", ] extra_kwargs = { "addon": {"queryset": AddOn.objects.none()}, diff --git a/documentcloud/addons/tests/test_views.py b/documentcloud/addons/tests/test_views.py index f52f514c..232ee91d 100644 --- a/documentcloud/addons/tests/test_views.py +++ b/documentcloud/addons/tests/test_views.py @@ -438,3 +438,83 @@ def test_filter_message_absent_is_noop(self, client): response = client.get("/api/addon_runs/") assert response.status_code == status.HTTP_200_OK assert len(response.json()["results"]) == 3 + + def test_default_not_dismissed(self): + """A newly created event is not dismissed""" + event = AddOnEventFactory() + assert event.dismissed is False + + def test_dismissed_in_serializer(self, client): + """The dismissed field is exposed on the event""" + event = AddOnEventFactory() + client.force_authenticate(user=event.user) + response = client.get(f"/api/addon_events/{event.pk}/") + assert response.status_code == status.HTTP_200_OK + assert response.json()["dismissed"] is False + + def test_dismiss(self, client): + """The owner can dismiss their event""" + event = AddOnEventFactory(dismissed=False) + client.force_authenticate(user=event.user) + response = client.patch( + f"/api/addon_events/{event.pk}/", {"dismissed": True}, format="json" + ) + assert response.status_code == status.HTTP_200_OK + assert response.json()["dismissed"] is True + event.refresh_from_db() + assert event.dismissed is True + + def test_undismiss(self, client): + """The owner can un-dismiss their event""" + event = AddOnEventFactory(dismissed=True) + client.force_authenticate(user=event.user) + response = client.patch( + f"/api/addon_events/{event.pk}/", {"dismissed": False}, format="json" + ) + assert response.status_code == status.HTTP_200_OK + event.refresh_from_db() + assert event.dismissed is False + + def test_dismiss_non_owner(self, client): + """A non-owner can't see the event, so the patch 404s and nothing changes""" + event = AddOnEventFactory(dismissed=False) + other = UserFactory() + client.force_authenticate(user=other) + response = client.patch( + f"/api/addon_events/{event.pk}/", {"dismissed": True}, format="json" + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + event.refresh_from_db() + assert event.dismissed is False + + def test_filter_dismissed_false(self, client): + """?dismissed=false returns only non-dismissed events""" + user = UserFactory() + visible = AddOnEventFactory(user=user, dismissed=False) + AddOnEventFactory(user=user, dismissed=True) + client.force_authenticate(user=user) + response = client.get("/api/addon_events/", {"dismissed": "false"}) + assert response.status_code == status.HTTP_200_OK + ids = [r["id"] for r in response.json()["results"]] + assert ids == [visible.pk] + + def test_filter_dismissed_true(self, client): + """?dismissed=true returns only dismissed events""" + user = UserFactory() + AddOnEventFactory(user=user, dismissed=False) + dismissed = AddOnEventFactory(user=user, dismissed=True) + client.force_authenticate(user=user) + response = client.get("/api/addon_events/", {"dismissed": "true"}) + assert response.status_code == status.HTTP_200_OK + ids = [r["id"] for r in response.json()["results"]] + assert ids == [dismissed.pk] + + def test_filter_dismissed_absent_is_noop(self, client): + """Omitting the dismissed filter returns all viewable events""" + user = UserFactory() + AddOnEventFactory(user=user, dismissed=False) + AddOnEventFactory(user=user, dismissed=True) + client.force_authenticate(user=user) + response = client.get("/api/addon_events/") + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["results"]) == 2 diff --git a/documentcloud/addons/views.py b/documentcloud/addons/views.py index 4c2ef374..eb76e8dc 100644 --- a/documentcloud/addons/views.py +++ b/documentcloud/addons/views.py @@ -1043,6 +1043,10 @@ class Filter(django_filters.FilterSet): "`https://www.nifc.gov`." ), ) + dismissed = django_filters.BooleanFilter( + field_name="dismissed", + help_text="Filter events by whether they have been dismissed.", + ) def site_filter(self, queryset, name, value): # pylint: disable=unused-argument diff --git a/documentcloud/core/tests.py b/documentcloud/core/tests.py index d38420b9..e6379117 100644 --- a/documentcloud/core/tests.py +++ b/documentcloud/core/tests.py @@ -1,4 +1,5 @@ # Django +from django.conf import settings from django.contrib.flatpages.models import FlatPage from django.contrib.sites.models import Site from django.db import transaction @@ -58,7 +59,7 @@ def sign(self, data): token = "testtoken" timestamp = int(time.time()) signature = hmac.new( - key="".encode("utf8"), + key=settings.MAILGUN_API_KEY.encode("utf8"), msg=f"{timestamp}{token}".encode("utf8"), digestmod=hashlib.sha256, ).hexdigest() diff --git a/documentcloud/documents/models/saved_search.py b/documentcloud/documents/models/saved_search.py index f071a2e2..d0777417 100644 --- a/documentcloud/documents/models/saved_search.py +++ b/documentcloud/documents/models/saved_search.py @@ -1,4 +1,3 @@ -# Standard Library # Django from django.db import models from django.utils.translation import gettext_lazy as _ diff --git a/documentcloud/documents/modifications.py b/documentcloud/documents/modifications.py index 1682dc58..cb183fbb 100644 --- a/documentcloud/documents/modifications.py +++ b/documentcloud/documents/modifications.py @@ -1,4 +1,3 @@ -# Standard Library # Django from django.db import transaction diff --git a/documentcloud/organizations/models.py b/documentcloud/organizations/models.py index 131902b6..a4261b42 100644 --- a/documentcloud/organizations/models.py +++ b/documentcloud/organizations/models.py @@ -91,8 +91,10 @@ def has_admin(self, user): return self.users.filter(pk=user.pk, memberships__admin=True).exists() def _update_resources(self, data, date_update): - # calc reqs/month in case it has changed - self.ai_credits_per_month = self.calc_ai_credits_per_month(data["max_users"]) + # sum AI credits across all entitlements + self.ai_credits_per_month = sum( + self._calc_ent_ai_credits(ent_data) for ent_data in data["entitlements"] + ) # if date update has changed, then this is a monthly restore of the # subscription, and we should restore monthly AI credits. If not, AI credits @@ -109,8 +111,19 @@ def _update_resources(self, data, date_update): self.monthly_ai_credits = self.ai_credits_per_month self.date_update = date_update - def _choose_entitlement(self, entitlements): - return max(entitlements, key=lambda e: e["resources"].get("base_ai_credits", 0)) + @staticmethod + def _calc_ent_ai_credits(ent_data): + """Compute the AI credit quota for one entitlement entry. + + Uses ent_data["quantity"] (the per-subscription billing quantity sent + by Squarelet) rather than the stale org-level max_users field. + """ + r = ent_data["resources"] + base = r.get("base_ai_credits", 0) + per_user = r.get("ai_credits_per_user", 0) + minimum = r.get("minimum_users", 1) + quantity = ent_data.get("quantity", 1) + return base + max(0, quantity - minimum) * per_user @transaction.atomic def merge(self, uuid): @@ -145,15 +158,6 @@ def merge(self, uuid): self.merged = other - def calc_ai_credits_per_month(self, users): - """Calculate how many AI credits an organization gets per month on this plan - for a given number of users""" - return ( - self.entitlement.base_ai_credits - + (users - self.entitlement.minimum_users) - * self.entitlement.ai_credits_per_user - ) - @transaction.atomic def use_ai_credits(self, amount, user_id, note): """Try to deduct AI credits from the organization's balance diff --git a/documentcloud/organizations/tests/test_models.py b/documentcloud/organizations/tests/test_models.py index d61a4090..9916230c 100644 --- a/documentcloud/organizations/tests/test_models.py +++ b/documentcloud/organizations/tests/test_models.py @@ -1,14 +1,34 @@ +# Standard Library +from datetime import date + # Third Party import pytest # DocumentCloud from documentcloud.organizations.exceptions import InsufficientAICreditsError from documentcloud.organizations.models import Organization -from documentcloud.organizations.tests.factories import OrganizationFactory +from documentcloud.organizations.tests.factories import ( + EntitlementFactory, + OrganizationEntitlementFactory, + OrganizationFactory, + ProfessionalEntitlementFactory, +) from documentcloud.users.models import User from documentcloud.users.tests.factories import UserFactory +def ent_json(entitlement, date_update, quantity=1): + """Helper function for serializing entitlement data""" + return { + "name": entitlement.name, + "slug": entitlement.slug, + "description": entitlement.description, + "resources": entitlement.resources, + "date_update": date_update, + "quantity": quantity, + } + + class TestOrganization: @pytest.mark.django_db() @@ -66,6 +86,171 @@ def test_merge_fks(self): ) +class TestSquareletUpdateDataMultiEntitlement: + """Test cases for update_data with multiple entitlements""" + + def _org_data(self, organization, entitlements): + return { + "name": organization.name, + "slug": organization.slug, + "individual": False, + "private": False, + "entitlements": entitlements, + "card": "", + } + + @pytest.mark.django_db() + def test_two_paid_entitlements_sums_ai_credits(self): + """Two paid entitlements: ai_credits_per_month = sum of both""" + ent1 = ProfessionalEntitlementFactory() # base_ai_credits=2000, min=1 + ent2 = OrganizationEntitlementFactory() # base_ai_credits=5000, min=5 + organization = OrganizationFactory() + date_update = date(2024, 3, 1) + + organization.update_data( + self._org_data( + organization, + [ + ent_json(ent1, date_update, quantity=1), + ent_json(ent2, date_update, quantity=5), + ], + ) + ) + organization.refresh_from_db() + # Professional: 2000 + max(0, 1-1)*0 = 2000 + # Organization: 5000 + max(0, 5-5)*500 = 5000 + assert organization.ai_credits_per_month == 7000 + assert organization.monthly_ai_credits == 7000 + + @pytest.mark.django_db() + def test_paid_and_grant_entitlement_sums_ai_credits(self): + """Paid entitlement + grant entitlement: both contribute to total""" + paid = OrganizationEntitlementFactory() + grant = EntitlementFactory( + name="Grant", + resources={ + "minimum_users": 1, + "base_ai_credits": 500, + "ai_credits_per_user": 0, + "feature_level": 0, + }, + ) + organization = OrganizationFactory() + date_update = date(2024, 3, 1) + + organization.update_data( + self._org_data( + organization, + [ + ent_json(paid, date_update, quantity=5), + ent_json(grant, date_update, quantity=1), + ], + ) + ) + organization.refresh_from_db() + # Organization: 5000 + max(0, 5-5)*500 = 5000, Grant: 500 + assert organization.ai_credits_per_month == 5500 + assert organization.monthly_ai_credits == 5500 + + @pytest.mark.django_db() + def test_primary_entitlement_is_highest_feature_level(self): + """org.entitlement FK points to entitlement with highest feature_level""" + low = ProfessionalEntitlementFactory() # feature_level=1 + high = OrganizationEntitlementFactory() # feature_level=2 + organization = OrganizationFactory() + date_update = date(2024, 3, 1) + + organization.update_data( + self._org_data( + organization, + [ + ent_json(low, date_update, quantity=1), + ent_json(high, date_update, quantity=5), + ], + ) + ) + organization.refresh_from_db() + assert organization.entitlement.slug == high.slug + + @pytest.mark.django_db() + def test_equal_feature_level_tie_breaks_to_first(self): + """Equal feature_level: first entitlement in list wins the FK""" + ent1 = EntitlementFactory( + name="GrantA", + resources={"base_ai_credits": 100, "feature_level": 1}, + ) + ent2 = EntitlementFactory( + name="GrantB", + resources={"base_ai_credits": 200, "feature_level": 1}, + ) + organization = OrganizationFactory() + date_update = date(2024, 3, 1) + + organization.update_data( + self._org_data( + organization, + [ + ent_json(ent1, date_update, quantity=1), + ent_json(ent2, date_update, quantity=1), + ], + ) + ) + organization.refresh_from_db() + assert organization.entitlement.slug == ent1.slug + assert organization.ai_credits_per_month == 300 + + @pytest.mark.django_db() + def test_quantity_below_minimum_does_not_reduce_base(self): + """quantity < minimum_users: base AI credits are not reduced""" + ent = OrganizationEntitlementFactory() # min=5, base=5000, per_user=500 + organization = OrganizationFactory() + + organization.update_data( + self._org_data(organization, [ent_json(ent, date(2024, 3, 1), quantity=2)]) + ) + organization.refresh_from_db() + # max(0, 2-5) = 0, so just base=5000 + assert organization.ai_credits_per_month == 5000 + + @pytest.mark.django_db() + def test_quantity_above_minimum_adds_per_user_credits(self): + """quantity > minimum_users: extra quantity adds per-user AI credits""" + ent = OrganizationEntitlementFactory() # min=5, base=5000, per_user=500 + organization = OrganizationFactory() + + organization.update_data( + self._org_data(organization, [ent_json(ent, date(2024, 3, 1), quantity=8)]) + ) + organization.refresh_from_db() + # 5000 + max(0, 8-5)*500 = 5000 + 1500 = 6500 + assert organization.ai_credits_per_month == 6500 + + @pytest.mark.django_db() + def test_multi_entitlement_monthly_restore(self): + """Monthly restore resets monthly_ai_credits to sum of all entitlements""" + ent1 = ProfessionalEntitlementFactory() + ent2 = OrganizationEntitlementFactory() + organization = OrganizationFactory( + entitlement=ent2, + date_update=date(2024, 2, 1), + ai_credits_per_month=7000, + monthly_ai_credits=1000, + ) + + organization.update_data( + self._org_data( + organization, + [ + ent_json(ent1, date(2024, 3, 1), quantity=1), + ent_json(ent2, date(2024, 3, 1), quantity=5), + ], + ) + ) + organization.refresh_from_db() + assert organization.ai_credits_per_month == 7000 + assert organization.monthly_ai_credits == 7000 + + class TestOrganizationCollective: """Tests for Organization collective resource sharing""" diff --git a/requirements/base.txt b/requirements/base.txt index 2660a3cb..9c09ccc4 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -451,7 +451,7 @@ sqlparse==0.5.4 # via # django # django-debug-toolbar -squarelet-auth==0.1.14 +squarelet-auth==0.1.16 # via -r requirements/base.in stack-data==0.3.0 # via ipython diff --git a/requirements/local.txt b/requirements/local.txt index 0e77b19f..3482d582 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -786,7 +786,7 @@ sqlparse==0.5.4 # -r requirements/base.txt # django # django-debug-toolbar -squarelet-auth==0.1.14 +squarelet-auth==0.1.16 # via -r requirements/base.txt stack-data==0.3.0 # via diff --git a/requirements/production.txt b/requirements/production.txt index 5c5da4a4..e9dd130b 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -622,7 +622,7 @@ sqlparse==0.5.4 # -r requirements/base.txt # django # django-debug-toolbar -squarelet-auth==0.1.14 +squarelet-auth==0.1.16 # via -r requirements/base.txt stack-data==0.3.0 # via diff --git a/tasks.py b/tasks.py index c9aa4e3b..c5d28f44 100755 --- a/tasks.py +++ b/tasks.py @@ -14,6 +14,16 @@ ) WEB_OPEN = "xdg-open {} > /dev/null 2>&1" +@task +def up(c): + """Start the docker images""" + c.run("docker compose up -d") + + +@task +def down(c): + """Shut down the docker images""" + c.run(f"docker compose down") @task def test( @@ -117,12 +127,12 @@ def format(c): """Format your code""" c.run( DJANGO_RUN_USER.format( - cmd="black documentcloud --exclude migrations && " + cmd="sh -c 'black documentcloud --exclude migrations && " "black config/urls.py && " "black config/settings && " "isort documentcloud && " "isort config/urls.py && " - "isort config/settings" + "isort config/settings'" ) ) @@ -233,11 +243,13 @@ def download_tesseract_data(c): """Download Tesseract data files. Needed to be able to do OCR locally.""" c.run("cd config/aws/lambda; ./build.sh") + @task def initialize_minio(c): """Initialize Minio bucket and policies for local development""" c.run(DJANGO_RUN.format(cmd="python manage.py initialize_minio")) + @task def deploy_lambdas(c, staging=False): """Deploy lambda functions on AWS"""