feat: update typesense search index on rfc pub/update (#10575)

* chore: typesense API config for k8s

* feat: DocumentInfo.pub_datetime() helper

* chore(deps): install typesense library

* feat: searchindex (typesense) util module

* feat: sanitize abstract

* feat: add (sanitized) content

* style: ruff ruff on doc/tasks.py

* feat: search index update task

* chore: call the update task

* refactor: better settings management

* ci: update prod settings

* chore: typing

* test: searchindex tests

* test: searchindex task test

* style: ruff ruff

* chore: drop type hints to fix mypy errors

* test: fix tests

* test: improve coverage

* fix: handle missing content blob correctly

Author: jennifer-richards

ietf/api/serializers_rpc.py

@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2025, All Rights Reserved
+# Copyright The IETF Trust 2025-2026, All Rights Reserved
 import datetime
 from pathlib import Path
 from typing import Literal, Optional
@@ -20,6 +20,7 @@
     RfcAuthor,
 )
 from ietf.doc.serializers import RfcAuthorSerializer
+from ietf.doc.tasks import update_rfc_searchindex_task
 from ietf.doc.utils import (
     default_consensus,
     prettify_std_name,
@@ -682,6 +683,8 @@ def update(self, instance, validated_data):
                 stale_subseries_relations.delete()
             if len(rfc_events) > 0:
                 rfc.save_with_history(rfc_events)
+
+        update_rfc_searchindex_task.delay(rfc.rfc_number)
         return rfc
 
 

ietf/api/tests_serializers_rpc.py

@@ -1,4 +1,6 @@
 # Copyright The IETF Trust 2026, All Rights Reserved
+from unittest import mock
+
 from django.utils import timezone
 
 from ietf.utils.test_utils import TestCase
@@ -32,7 +34,8 @@ def test_create(self):
         with self.assertRaises(RuntimeError, msg="serializer does not allow create()"):
             serializer.save()
 
-    def test_update(self):
+    @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task")
+    def test_update(self, mock_update_searchindex_task):
         rfc = WgRfcFactory(pages=10)
         serializer = EditableRfcSerializer(
             instance=rfc,
@@ -56,6 +59,11 @@ def test_update(self):
         )
         self.assertTrue(serializer.is_valid())
         result = serializer.save()
+        self.assertTrue(mock_update_searchindex_task.delay.called)
+        self.assertEqual(
+            mock_update_searchindex_task.delay.call_args,
+            mock.call(rfc.rfc_number),
+        )
         result.refresh_from_db()
         self.assertEqual(result.title, "Yadda yadda yadda")
         self.assertEqual(
@@ -84,7 +92,8 @@ def test_update(self):
             [Document.objects.get(name="fyi999")],
         )
 
-    def test_partial_update(self):
+    @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task")
+    def test_partial_update(self, mock_update_searchindex_task):
         # We could test other permutations of fields, but authors is a partial update
         # we know we are going to use, so verifying that one in particular.
         rfc = WgRfcFactory(pages=10, abstract="do or do not", title="padawan")
@@ -104,6 +113,11 @@ def test_partial_update(self):
         )
         self.assertTrue(serializer.is_valid())
         result = serializer.save()
+        self.assertTrue(mock_update_searchindex_task.delay.called)
+        self.assertEqual(
+            mock_update_searchindex_task.delay.call_args,
+            mock.call(rfc.rfc_number),
+        )
         result.refresh_from_db()
         self.assertEqual(rfc.title, "padawan")
         self.assertEqual(

ietf/api/tests_views_rpc.py

@@ -196,7 +196,8 @@ def test_notify_rfc_published(self, mock_task_delay):
         self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list)
 
     @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]})
-    def test_upload_rfc_files(self):
+    @mock.patch("ietf.api.views_rpc.update_rfc_searchindex_task")
+    def test_upload_rfc_files(self, mock_update_searchindex_task):
         def _valid_post_data():
             """Generate a valid post data dict
 
@@ -217,14 +218,7 @@ def _valid_post_data():
             }
 
         url = urlreverse("ietf.api.purple_api.upload_rfc_files")
-        unused_rfc_number = (
-            Document.objects.filter(rfc_number__isnull=False).aggregate(
-                unused_rfc_number=Max("rfc_number") + 1
-            )["unused_rfc_number"]
-            or 10000
-        )
-
-        rfc = WgRfcFactory(rfc_number=unused_rfc_number)
+        rfc = WgRfcFactory()
         assert isinstance(rfc, Document), "WgRfcFactory should generate a Document"
         with TemporaryDirectory() as rfc_dir:
             settings.RFC_PATH = rfc_dir  # affects overridden settings
@@ -236,15 +230,17 @@ def _valid_post_data():
             # no api key
             r = self.client.post(url, _valid_post_data(), format="multipart")
             self.assertEqual(r.status_code, 403)
+            self.assertFalse(mock_update_searchindex_task.delay.called)
 
             # invalid RFC
             r = self.client.post(
                 url,
-                _valid_post_data() | {"rfc": unused_rfc_number + 1},
+                _valid_post_data() | {"rfc": rfc.rfc_number + 10},
                 format="multipart",
                 headers={"X-Api-Key": "valid-token"},
             )
             self.assertEqual(r.status_code, 400)
+            self.assertFalse(mock_update_searchindex_task.delay.called)
 
             # empty files
             r = self.client.post(
@@ -263,6 +259,7 @@ def _valid_post_data():
                 headers={"X-Api-Key": "valid-token"},
             )
             self.assertEqual(r.status_code, 400)
+            self.assertFalse(mock_update_searchindex_task.delay.called)
 
             # bad file type
             r = self.client.post(
@@ -276,9 +273,10 @@ def _valid_post_data():
                 headers={"X-Api-Key": "valid-token"},
             )
             self.assertEqual(r.status_code, 400)
+            self.assertFalse(mock_update_searchindex_task.delay.called)
 
             # Put a file in the way. Post should fail because replace = False
-            file_in_the_way = (rfc_path / f"rfc{unused_rfc_number}.txt")
+            file_in_the_way = (rfc_path / f"{rfc.name}.txt")
             file_in_the_way.touch()
             r = self.client.post(
                 url,
@@ -287,11 +285,12 @@ def _valid_post_data():
                 headers={"X-Api-Key": "valid-token"},
             )
             self.assertEqual(r.status_code, 409)  # conflict
+            self.assertFalse(mock_update_searchindex_task.delay.called)
             file_in_the_way.unlink()
 
             # Put a blob in the way. Post should fail because replace = False
             blob_in_the_way = Blob.objects.create(
-                bucket="rfc", name=f"txt/rfc{unused_rfc_number}.txt", content=b""
+                bucket="rfc", name=f"txt/{rfc.name}.txt", content=b""
             )
             r = self.client.post(
                 url,
@@ -300,6 +299,7 @@ def _valid_post_data():
                 headers={"X-Api-Key": "valid-token"},
             )
             self.assertEqual(r.status_code, 409)  # conflict
+            self.assertFalse(mock_update_searchindex_task.delay.called)
             blob_in_the_way.delete()
 
             # valid post
@@ -310,8 +310,13 @@ def _valid_post_data():
                 headers={"X-Api-Key": "valid-token"},
             )
             self.assertEqual(r.status_code, 200)
+            self.assertTrue(mock_update_searchindex_task.delay.called)
+            self.assertEqual(
+                mock_update_searchindex_task.delay.call_args,
+                mock.call(rfc.rfc_number),
+            )
             for extension in ["xml", "txt", "html", "pdf", "json"]:
-                filename = f"rfc{unused_rfc_number}.{extension}"
+                filename = f"{rfc.name}.{extension}"
                 self.assertEqual(
                     (rfc_path / filename)
                     .read_text(),
@@ -328,7 +333,7 @@ def _valid_post_data():
                     f"{extension} blob should contain the expected content",
                 )
             # special case for notprepped
-            notprepped_fn = f"rfc{unused_rfc_number}.notprepped.xml"
+            notprepped_fn = f"{rfc.name}.notprepped.xml"
             self.assertEqual(
                 (
                     rfc_path / "prerelease" / notprepped_fn
@@ -347,22 +352,29 @@ def _valid_post_data():
             )
 
             # re-post with replace = False should now fail
+            mock_update_searchindex_task.reset_mock()
             r = self.client.post(
                 url,
                 _valid_post_data(),
                 format="multipart",
                 headers={"X-Api-Key": "valid-token"},
             )
             self.assertEqual(r.status_code, 409)  # conflict
-            
+            self.assertFalse(mock_update_searchindex_task.delay.called)
+
             # re-post with replace = True should succeed
             r = self.client.post(
                 url,
                 _valid_post_data() | {"replace": True},
                 format="multipart",
                 headers={"X-Api-Key": "valid-token"},
             )
-            self.assertEqual(r.status_code, 200)  # conflict
+            self.assertEqual(r.status_code, 200)
+            self.assertTrue(mock_update_searchindex_task.delay.called)
+            self.assertEqual(
+                mock_update_searchindex_task.delay.call_args,
+                mock.call(rfc.rfc_number),
+            )
 
     @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]})
     @mock.patch("ietf.api.views_rpc.create_rfc_index_task")

ietf/api/views_rpc.py

@@ -38,7 +38,7 @@
 from ietf.doc.models import Document, DocHistory, RfcAuthor, DocEvent
 from ietf.doc.serializers import RfcAuthorSerializer
 from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage
-from ietf.doc.tasks import signal_update_rfc_metadata_task
+from ietf.doc.tasks import signal_update_rfc_metadata_task, update_rfc_searchindex_task
 from ietf.person.models import Email, Person
 from ietf.sync.tasks import create_rfc_index_task
 
@@ -516,6 +516,7 @@ def post(self, request):
                     destination.parent.mkdir()
                 shutil.move(ftm, destination)
 
+        update_rfc_searchindex_task.delay(rfc.rfc_number)
         return Response(NotificationAckSerializer().data)
 
 

ietf/doc/models.py

@@ -1285,19 +1285,24 @@ def submission(self):
         s = s.first()
         return s
 
-    def pub_date(self):
-        """Get the publication date for this document
-
-        This is the rfc publication date for RFCs, and the new-revision date for other documents.
-        """
+    def pub_datetime(self):
+        """Get the publication datetime of this document"""
         if self.type_id == "rfc":
             # As of Sept 2022, in ietf.sync.rfceditor.update_docs_from_rfc_index() `published_rfc` events are
             # created with a timestamp whose date *in the PST8PDT timezone* is the official publication date
             # assigned by the RFC editor.
             event = self.latest_event(type='published_rfc')
         else:
             event = self.latest_event(type='new_revision')
-        return event.time.astimezone(RPC_TZINFO).date() if event else None
+        return event.time.astimezone(RPC_TZINFO) if event else None
+
+    def pub_date(self):
+        """Get the publication date for this document
+
+        This is the rfc publication date for RFCs, and the new-revision date for other documents.
+        """
+        pub_datetime = self.pub_datetime()
+        return None if pub_datetime is None else pub_datetime.date()
 
     def is_dochistory(self):
         return False

ietf/doc/tasks.py

@@ -3,6 +3,7 @@
 # Celery task definitions
 #
 import datetime
+
 import debug  # pyflakes:ignore
 
 from celery import shared_task
@@ -11,7 +12,7 @@
 from django.conf import settings
 from django.utils import timezone
 
-from ietf.utils import log
+from ietf.utils import log, searchindex
 from ietf.utils.timezone import datetime_today
 
 from .expire import (
@@ -77,17 +78,19 @@ def expire_last_calls_task():
         try:
             expire_last_call(doc)
         except Exception:
-            log.log(f"ERROR: Failed to expire last call for {doc.file_tag()} (id={doc.pk})")
+            log.log(
+                f"ERROR: Failed to expire last call for {doc.file_tag()} (id={doc.pk})"
+            )
         else:
             log.log(f"Expired last call for {doc.file_tag()} (id={doc.pk})")
 
 
-@shared_task            
+@shared_task
 def generate_idnits2_rfc_status_task():
     outpath = Path(settings.DERIVED_DIR) / "idnits2-rfc-status"
     blob = generate_idnits2_rfc_status()
     try:
-        outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE
+        outpath.write_text(blob, encoding="utf8")  # TODO-BLOBSTORE
     except Exception as e:
         log.log(f"failed to write idnits2-rfc-status: {e}")
 
@@ -97,15 +100,15 @@ def generate_idnits2_rfcs_obsoleted_task():
     outpath = Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted"
     blob = generate_idnits2_rfcs_obsoleted()
     try:
-        outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE
+        outpath.write_text(blob, encoding="utf8")  # TODO-BLOBSTORE
     except Exception as e:
         log.log(f"failed to write idnits2-rfcs-obsoleted: {e}")
 
 
 @shared_task
 def generate_draft_bibxml_files_task(days=7, process_all=False):
     """Generate bibxml files for recently updated docs
-    
+
     If process_all is False (the default), processes only docs with new revisions
     in the last specified number of days.
     """
@@ -117,7 +120,9 @@ def generate_draft_bibxml_files_task(days=7, process_all=False):
         doc__type_id="draft",
     ).order_by("time")
     if not process_all:
-        doc_events = doc_events.filter(time__gte=timezone.now() - datetime.timedelta(days=days))
+        doc_events = doc_events.filter(
+            time__gte=timezone.now() - datetime.timedelta(days=days)
+        )
     for event in doc_events:
         try:
             update_or_create_draft_bibxml_file(event.doc, event.rev)
@@ -132,6 +137,7 @@ def investigate_fragment_task(name_fragment: str):
         "results": investigate_fragment(name_fragment),
     }
 
+
 @shared_task
 def rebuild_reference_relations_task(doc_names: list[str]):
     log.log(f"Task: Rebuilding reference relations for {doc_names}")
@@ -157,6 +163,32 @@ def rebuild_reference_relations_task(doc_names: list[str]):
 def fixup_bofreq_timestamps_task():  # pragma: nocover
     fixup_bofreq_timestamps()
 
+
 @shared_task
 def signal_update_rfc_metadata_task(rfc_number_list=()):
     signal_update_rfc_metadata(rfc_number_list)
+
+
+@shared_task(bind=True)
+def update_rfc_searchindex_task(self, rfc_number: int):
+    """Update the search index for one RFC"""
+    if not searchindex.enabled():
+        log.log("Search indexing is not enabled, skipping")
+        return
+
+    rfc = Document.objects.filter(type_id="rfc", rfc_number=rfc_number).first()
+    if rfc is None:
+        log.log(
+            f"ERROR: Document for rfc{rfc_number} not found, not updating search index"
+        )
+        return
+    try:
+        searchindex.update_or_create_rfc_entry(rfc)
+    except Exception as err:
+        log.log(f"Search index update for {rfc.name} failed ({err})")
+        if isinstance(err, searchindex.RETRYABLE_ERROR_CLASSES):
+            searchindex_settings = searchindex.get_settings()
+            self.retry(
+                countdown=searchindex_settings["TASK_RETRY_DELAY"],
+                max_retries=searchindex_settings["TASK_MAX_RETRIES"],
+            )