diff --git a/.github/workflows/integration-test-cron.yml b/.github/workflows/integration-test-cron.yml
index 004e5c75..162cf723 100644
--- a/.github/workflows/integration-test-cron.yml
+++ b/.github/workflows/integration-test-cron.yml
@@ -127,6 +127,8 @@ jobs:
with:
repository: nextcloud/context_chat_backend
path: context_chat_backend/
+ # todo: remove this
+ ref: feat/vector-db-wipe
persist-credentials: false
- name: Checkout viewer
@@ -410,6 +412,73 @@ jobs:
kill -9 $WORKER1_PID
kill -9 $WORKER2_PID
+ - name: Wipe and reindex only the files provider
+ run: |
+ ./occ context_chat:wipe --force --provider files__default --reindex
+
+ stats=$(./occ context_chat:stats --json)
+ echo "$stats"
+ indexed=$(echo "$stats" | jq '.vectordb_document_counts.files__default // 0')
+ if [ "$indexed" != "0" ]; then
+ echo "Error: expected no indexed files right after the files wipe, got $indexed"
+ exit 1
+ fi
+
+ echo "::group::Waiting up to 10 minutes for the files to be indexed again"
+ for i in {1..60}; do
+ stats=$(timeout 30 ./occ context_chat:stats --json || true)
+ indexed=$(echo "$stats" | jq '.vectordb_document_counts.files__default // 0' 2>/dev/null || echo 0)
+ echo "Attempt $i: indexed=$indexed"
+ if [ "$indexed" -ge 10 ]; then
+ echo "::endgroup::"
+ echo "Files reindexed, $indexed documents"
+ exit 0
+ fi
+ sleep 10
+ done
+ echo "::endgroup::"
+ echo "Error: files were not indexed again after the files wipe"
+ exit 1
+
+ - name: Wipe all the indexed data and check that the queues and the vector DB are empty
+ run: |
+ ./occ context_chat:wipe --force
+
+ stats=$(./occ context_chat:stats --json)
+ echo "$stats"
+ indexed=$(echo "$stats" | jq '[.vectordb_document_counts | to_entries[].value] | add // 0')
+ queued=$(echo "$stats" | jq '[.queued_documents_counts | to_entries[].value] | add // 0')
+ actions=$(echo "$stats" | jq '.queued_actions_count')
+ fs_events=$(echo "$stats" | jq '.queued_fs_events_count')
+ # the queued counts leave out locked items, check those separately
+ queued_locked=$(echo "$stats" | jq '[.queued_documents_locked_counts | to_entries[].value] | add // 0')
+ actions_locked=$(echo "$stats" | jq '.queued_actions_locked_count')
+ backend_available=$(echo "$stats" | jq '.backend_available')
+ echo "indexed=$indexed queued=$queued queued_locked=$queued_locked actions=$actions actions_locked=$actions_locked fs_events=$fs_events backend_available=$backend_available"
+ if [ "$backend_available" != "true" ] || [ "$indexed" != "0" ] || [ "$queued" != "0" ] || [ "$queued_locked" != "0" ] || [ "$actions" != "0" ] || [ "$actions_locked" != "0" ] || [ "$fs_events" != "0" ]; then
+ echo "Error: expected an empty vector DB and empty queues after the wipe"
+ exit 1
+ fi
+
+ - name: Reindex and check that documents get indexed again
+ run: |
+ ./occ context_chat:reindex
+ echo "::group::Waiting up to 10 minutes for documents to be indexed again"
+ for i in {1..60}; do
+ stats=$(timeout 30 ./occ context_chat:stats --json || true)
+ indexed=$(echo "$stats" | jq '.vectordb_document_counts.files__default // 0' 2>/dev/null || echo 0)
+ echo "Attempt $i: indexed=$indexed"
+ if [ "$indexed" -ge 10 ]; then
+ echo "Reindex works, $indexed documents indexed again"
+ echo "::endgroup::"
+ exit 0
+ fi
+ sleep 10
+ done
+ echo "::endgroup::"
+ echo "Error: documents were not indexed again after the reindex"
+ exit 1
+
- name: Show nextcloud logs
if: always()
run: |
diff --git a/.github/workflows/integration-test-file-listener.yml b/.github/workflows/integration-test-file-listener.yml
index 3a5e2ac3..2664d0f0 100644
--- a/.github/workflows/integration-test-file-listener.yml
+++ b/.github/workflows/integration-test-file-listener.yml
@@ -115,6 +115,8 @@ jobs:
with:
repository: nextcloud/context_chat_backend
path: context_chat_backend/
+ # todo: remove this
+ ref: feat/vector-db-wipe
persist-credentials: false
- name: Checkout viewer
diff --git a/appinfo/info.xml b/appinfo/info.xml
index 2b8cda78..7075cb78 100644
--- a/appinfo/info.xml
+++ b/appinfo/info.xml
@@ -47,6 +47,7 @@ Refer to the [Context Chat Backend's readme](https://github.com/nextcloud/contex
OCA\ContextChat\Command\Search
OCA\ContextChat\Command\Statistics
OCA\ContextChat\Command\Reindex
+ OCA\ContextChat\Command\Wipe
diff --git a/lib/Command/Reindex.php b/lib/Command/Reindex.php
index ca3de909..e3b2dc6f 100644
--- a/lib/Command/Reindex.php
+++ b/lib/Command/Reindex.php
@@ -9,10 +9,10 @@
namespace OCA\ContextChat\Command;
-use OCA\ContextChat\BackgroundJobs\SchedulerJob;
-use OCP\BackgroundJob\IJobList;
+use OCA\ContextChat\Service\ReindexService;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
/**
@@ -20,32 +20,55 @@
*
* SchedulerJob -> StorageCrawlJob -> IndexerJob is seeded only by the repair step and
* removes itself once the initial crawl finishes. There is otherwise no way to re-enumerate
- * mounts (e.g. after installing on an instance whose files predate the app, or to recover a crawl
- * that did not complete) short of reinstalling the app. This command re-adds SchedulerJob so the
- * full enumeration runs again; it is a no-op if one is already scheduled, and already-indexed
- * files are skipped.
+ * mounts (e.g. to recover a crawl that did not complete, or after context_chat:wipe) short of
+ * reinstalling the app. This command re-adds SchedulerJob so the full enumeration runs again if one
+ * doesn't exist already.
+ * Content providers' initial import also gets scheduled (e.g. after context_chat:wipe) if it has not run yet.
*/
class Reindex extends Command {
public function __construct(
- private IJobList $jobList,
+ private ReindexService $reindexService,
) {
parent::__construct();
}
protected function configure() {
$this->setName('context_chat:reindex')
- ->setDescription('Schedule a full re-crawl of all the files in all the mounts. Indexed files are not re-indexed when compared against context_chat_backend\'s vector DB. Content providers are not re-indexed.');
+ ->setDescription(
+ 'Schedule a full re-crawl of all the files in all the mounts and the import of the content providers'
+ . ' that were not imported yet. Indexed files are not re-indexed when compared against context_chat_backend\'s vector DB.'
+ )
+ ->addOption(
+ 'provider',
+ 'p',
+ InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY,
+ 'Only reindex this provider, e.g. "files__default" (appId__providerId) for the files.'
+ . ' A named content provider is imported again even if it was imported before. Multiple values can be specified by repeating the option.',
+ );
}
protected function execute(InputInterface $input, OutputInterface $output): int {
- if ($this->jobList->has(SchedulerJob::class, null)) {
- $output->writeln('A full re-crawl is already scheduled; nothing to do.');
+ /** @var list $providerKeys */
+ $providerKeys = $input->getOption('provider');
+ $scope = $providerKeys === [] ? null : $providerKeys;
+
+ if ($scope !== null) {
+ $known = $this->reindexService->getRegisteredProviderKeys();
+ $unknown = array_diff($scope, $known);
+ if ($unknown !== []) {
+ $output->writeln('Unknown provider(s): ' . implode(', ', $unknown) . '');
+ $output->writeln('Registered providers: ' . implode(', ', $known));
+ return 1;
+ }
+ }
+
+ if (!$this->reindexService->scheduleReindex($scope)) {
+ $output->writeln('The re-index is already scheduled; nothing to do.');
return 0;
}
- $this->jobList->add(SchedulerJob::class);
- $output->writeln('Scheduled a full re-crawl. SchedulerJob will enumerate all mounts on its next run.');
+ $output->writeln('Scheduled the re-index. The background jobs will queue the documents on their next run.');
return 0;
}
}
diff --git a/lib/Command/Wipe.php b/lib/Command/Wipe.php
new file mode 100644
index 00000000..df5639be
--- /dev/null
+++ b/lib/Command/Wipe.php
@@ -0,0 +1,140 @@
+setName('context_chat:wipe')
+ ->setDescription(
+ 'Delete the indexed documents from context_chat_backend\'s vector DB and clear the queues.'
+ . ' With --provider option, only the provider\'s data is deleted, the actions queue, and the indexing statistics are left intact.'
+ . ' Nothing gets indexed again until context_chat:reindex is run or the --reindex option is used.'
+ )
+ ->addOption(
+ 'provider',
+ 'p',
+ InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY,
+ 'Only wipe this provider, e.g. "files__default" (appId__providerId) for the files. Multiple values can be specified by repeating the option.',
+ )
+ ->addOption(
+ 'reindex',
+ null,
+ InputOption::VALUE_NONE,
+ 'Schedule a re-crawl of the files and a re-import of the content providers that were wiped',
+ )
+ ->addOption(
+ 'ignore-handed-out',
+ null,
+ InputOption::VALUE_NONE,
+ 'Do not wait for the documents context_chat_backend is still indexing. Only use this when the backend was stopped while indexing them, otherwise they can be written to the vector DB after the wipe.',
+ )
+ ->addOption(
+ 'force',
+ 'f',
+ InputOption::VALUE_NONE,
+ 'Do not ask for confirmation',
+ );
+ }
+
+ protected function execute(InputInterface $input, OutputInterface $output): int {
+ /** @var list $providerKeys */
+ $providerKeys = $input->getOption('provider');
+ $scope = $providerKeys === [] ? null : $providerKeys;
+
+ if ($scope !== null) {
+ try {
+ $known = $this->reindexService->getWipeableProviderKeys();
+ } catch (\RuntimeException $e) {
+ $output->writeln('Could not list the indexed providers: ' . $e->getMessage() . '');
+ return 1;
+ }
+ $unknown = array_diff($scope, $known);
+ if ($unknown !== []) {
+ $output->writeln('Unknown provider(s): ' . implode(', ', $unknown) . '');
+ $output->writeln('Known providers: ' . implode(', ', $known));
+ return 1;
+ }
+ }
+
+ $scopeText = $scope === null ? '' : ' of ' . implode(', ', $scope);
+
+ if (!$input->getOption('force')) {
+ /** @var QuestionHelper $helper */
+ $helper = $this->getHelper('question');
+ $question = new ConfirmationQuestion(
+ 'All the indexed documents in Context Chat'
+ . $scopeText
+ . ' will be deleted. The original documents are not touched. Continue? [y/N] ',
+ false,
+ );
+ if (!$helper->ask($input, $output, $question)) {
+ $output->writeln('Aborted, nothing was deleted. Use --force to skip the confirmation.');
+ return 1;
+ }
+ }
+
+ try {
+ $this->reindexService->wipe(
+ $scope,
+ (bool)$input->getOption('ignore-handed-out'),
+ static function (string $message) use ($output): void {
+ $output->writeln('' . $message . '...');
+ },
+ );
+ } catch (\Throwable $e) {
+ $output->writeln('Wipe failed. The command can be safely run again once the problem is fixed.');
+ $output->writeln('Error: ' . $e->getMessage() . '');
+ $output->writeln('' . $e->getTraceAsString() . '');
+ return 1;
+ }
+ $output->writeln('Deleted all the indexed documents' . $scopeText . ' and cleared the queues.');
+
+ // providers that are not registered anymore cannot be indexed again
+ $reindexScope = $scope === null
+ ? null
+ : array_values(array_intersect($scope, $this->reindexService->getRegisteredProviderKeys()));
+ if ($reindexScope === []) {
+ return 0;
+ }
+
+ if ($input->getOption('reindex')) {
+ try {
+ $this->reindexService->scheduleReindex($reindexScope);
+ } catch (\Throwable $e) {
+ $output->writeln('Scheduling re-index failed. It can be tried again with context_chat:reindex.');
+ $output->writeln('Error: ' . $e->getMessage() . '');
+ $output->writeln('' . $e->getTraceAsString() . '');
+ return 1;
+ }
+ $output->writeln('Scheduled the re-index.');
+ } elseif ($reindexScope === null) {
+ $output->writeln('Run context_chat:reindex to index the documents again.');
+ } else {
+ $output->writeln('Run context_chat:reindex --provider ' . implode(' --provider ', $reindexScope) . ' to index the documents again.');
+ }
+
+ return 0;
+ }
+}
diff --git a/lib/Controller/QueueController.php b/lib/Controller/QueueController.php
index f35eaa14..d260afd5 100644
--- a/lib/Controller/QueueController.php
+++ b/lib/Controller/QueueController.php
@@ -17,6 +17,7 @@
use OCA\ContextChat\Db\QueueContentItemMapper;
use OCA\ContextChat\Db\QueueFile;
use OCA\ContextChat\Db\QueueMapper;
+use OCA\ContextChat\Service\FetchGuard;
use OCA\ContextChat\Service\ProviderConfigService;
use OCA\ContextChat\Type\Source;
use OCP\AppFramework\Http;
@@ -47,6 +48,7 @@ public function __construct(
private IJobList $jobList,
private ITimeFactory $timeFactory,
private QueueMapper $queueMapper,
+ private FetchGuard $fetchGuard,
string $corsMethods = 'PUT, POST, GET, DELETE, PATCH',
string $corsAllowedHeaders = 'Authorization, Content-Type, Accept, OCS-APIRequest',
int $corsMaxAge = 1728000,
@@ -106,6 +108,13 @@ public function getDocumentsQueueItems(
if ($n > $maxN) {
$n = $maxN;
}
+ // hand out nothing, the backend waits and asks again
+ if ($this->fetchGuard->isPaused()) {
+ return new DataResponse([
+ 'files' => new \stdClass(),
+ 'content_providers' => new \stdClass(),
+ ]);
+ }
try {
$files = [];
while (count($files) < $n) {
@@ -216,6 +225,10 @@ public function countDocumentsQueueItems(QueueMapper $queueMapper, QueueContentI
#[ExAppRequired]
#[ApiRoute(verb: 'GET', url: '/queues/actions/')]
public function getActionsQueueItems(QueueActionMapper $queueActionMapper, int $n = 512) : DataResponse {
+ if ($this->fetchGuard->isPaused()) {
+ // stdClass is used to represent an empty object in the response
+ return new DataResponse(['actions' => new \stdClass()]);
+ }
try {
$actions = [];
while (count($actions) < $n) {
diff --git a/lib/Db/QueueContentItemMapper.php b/lib/Db/QueueContentItemMapper.php
index 2cc79cfc..b61f1cd6 100644
--- a/lib/Db/QueueContentItemMapper.php
+++ b/lib/Db/QueueContentItemMapper.php
@@ -71,6 +71,17 @@ public function removeFromQueue(array $ids): void {
}
}
+ /**
+ * @throws \OCP\DB\Exception
+ */
+ public function removeProvider(string $appId, string $providerId): void {
+ $qb = $this->db->getQueryBuilder();
+ $qb->delete($this->getTableName())
+ ->where($qb->expr()->eq('app_id', $qb->createNamedParameter($appId)))
+ ->andWhere($qb->expr()->eq('provider_id', $qb->createNamedParameter($providerId)))
+ ->executeStatement();
+ }
+
/**
* @throws \OCP\DB\Exception
* @return array
diff --git a/lib/Service/FetchGuard.php b/lib/Service/FetchGuard.php
new file mode 100644
index 00000000..4ede4c99
--- /dev/null
+++ b/lib/Service/FetchGuard.php
@@ -0,0 +1,126 @@
+timeFactory->getTime() + $this->getDrainTimeout() + $this->getRequestTimeout() + self::MARGIN;
+ $this->appConfig->setAppValueInt(self::CONFIG_KEY, $until, lazy: true);
+ return $until;
+ }
+
+ /**
+ * Lifts the pause, unless another one was started since.
+ */
+ public function resume(int $until): void {
+ if ($this->getPausedUntil() === $until) {
+ $this->appConfig->deleteAppValue(self::CONFIG_KEY);
+ }
+ }
+
+ public function isPaused(): bool {
+ return $this->getPausedUntil() > $this->timeFactory->getTime();
+ }
+
+ /**
+ * Waits for the documents the backend took before the pause to be reported back. A document that
+ * is still being indexed would be written to the vector DB after it was wiped.
+ *
+ * @param ?callable(int): void $onWaiting called with the number of documents still being indexed,
+ * once before waiting and again whenever that number changes
+ * @throws \RuntimeException when they are not reported back in time
+ * @throws \OCP\DB\Exception
+ */
+ public function waitForHandedOutDocuments(?callable $onWaiting = null): void {
+ $deadline = $this->timeFactory->getTime() + $this->getDrainTimeout();
+ $reported = null;
+ while (($locked = $this->countHandedOutDocuments()) > 0) {
+ if ($onWaiting !== null && $locked !== $reported) {
+ $onWaiting($locked);
+ $reported = $locked;
+ }
+ if ($this->timeFactory->getTime() >= $deadline) {
+ throw new \RuntimeException(
+ $locked . ' documents are still being indexed by context_chat_backend after waiting '
+ . $this->getDrainTimeout() . ' seconds. Try again later. If the backend was stopped while'
+ . ' indexing them, they would only be released after a day, use --ignore-handed-out then.'
+ );
+ }
+ sleep(1);
+ }
+ }
+
+ /**
+ * @throws \OCP\DB\Exception
+ */
+ private function countHandedOutDocuments(): int {
+ return $this->queueMapper->countLocked() + intval(array_sum($this->contentQueue->countLocked()));
+ }
+
+ private function getDrainTimeout(): int {
+ return $this->appConfig->getAppValueInt('wipe_drain_timeout', self::DEFAULT_DRAIN_TIMEOUT, lazy: true);
+ }
+
+ private function getRequestTimeout(): int {
+ return (int)$this->appConfig->getAppValueString(
+ 'request_timeout',
+ strval(Application::CC_DEFAULT_REQUEST_TIMEOUT),
+ lazy: true,
+ );
+ }
+
+ /**
+ * Read from the database instead of IAppConfig. IAppConfig values are cached by each web server
+ * for a few seconds, and all of them must stop handing out items before the wipe starts.
+ * This ensures that we get the most up-to-date value for the pause state.
+ */
+ private function getPausedUntil(): int {
+ $qb = $this->db->getQueryBuilder();
+ $result = $qb->select('configvalue')
+ ->from('appconfig')
+ ->where($qb->expr()->eq('appid', $qb->createNamedParameter(Application::APP_ID)))
+ ->andWhere($qb->expr()->eq('configkey', $qb->createNamedParameter(self::CONFIG_KEY)))
+ ->executeQuery();
+ $value = $result->fetchOne();
+ $result->closeCursor();
+ return $value === false ? 0 : (int)$value;
+ }
+}
diff --git a/lib/Service/LangRopeService.php b/lib/Service/LangRopeService.php
index 96150bcc..84e8cf93 100644
--- a/lib/Service/LangRopeService.php
+++ b/lib/Service/LangRopeService.php
@@ -189,14 +189,13 @@ private function requestToExApp(
*/
public function getIndexedDocumentsCounts(): array {
$response = $this->requestToExApp('/countIndexedDocuments', 'POST', extraOptions: ['timeout' => 10]);
- if ($response === []) {
- // No documents indexed yet
- return [];
- }
- if (!isset($response[ProviderConfigService::getDefaultProviderKey()])) {
- throw new \RuntimeException("Malformed indexed documents count response from Context Chat Backend (ExApp): '"
- . ProviderConfigService::getDefaultProviderKey() . "' key is missing, response: " . strval(json_encode($response)));
+ foreach ($response as $count) {
+ if (!is_int($count)) {
+ throw new \RuntimeException('Malformed indexed documents count response from Context Chat Backend (ExApp): '
+ . strval(json_encode($response)));
+ }
}
+ // only the providers with indexed documents are listed
return $response;
}
@@ -224,6 +223,16 @@ public function deleteProvider(string $providerKey): void {
$this->requestToExApp('/deleteProvider', 'POST', $params);
}
+ /**
+ * Delete all the indexed documents from the backend's vector DB
+ *
+ * @return void
+ * @throws \RuntimeException
+ */
+ public function deleteAll(): void {
+ $this->requestToExApp('/deleteAll', 'POST');
+ }
+
/**
* @param string $userId
* @return void
diff --git a/lib/Service/ProviderConfigService.php b/lib/Service/ProviderConfigService.php
index 344c98e0..f513596d 100644
--- a/lib/Service/ProviderConfigService.php
+++ b/lib/Service/ProviderConfigService.php
@@ -112,6 +112,53 @@ public function updateProvider(
$this->appConfig->setAppValueString('providers', json_encode($providers), lazy: true);
}
+ /**
+ * Mark all the content providers, except the files provider, as not initiated
+ * so their initial import runs again
+ */
+ public function resetProvidersInitiation(): void {
+ $providers = $this->getProviders();
+ foreach ($providers as $key => $provider) {
+ if ($key !== self::getDefaultProviderKey()) {
+ $providers[$key]['isInitiated'] = false;
+ }
+ }
+ $this->appConfig->setAppValueString('providers', json_encode($providers), lazy: true);
+ }
+
+ /**
+ * Get the parts of a provider key
+ *
+ * @param string $providerKey The provider key in the format "appId__providerId"
+ * @return array{0: string, 1: string} The appId and providerId
+ * @throws \RuntimeException when the provider key format is invalid
+ */
+ public function getProviderParts(string $providerKey): array {
+ $identifierValues = explode('__', $providerKey, 2);
+ if (count($identifierValues) !== 2) {
+ throw new \RuntimeException('Invalid provider key format: ' . $providerKey);
+ }
+ return $identifierValues;
+ }
+
+ /**
+ * Mark a specific provider as initiated or not initiated
+ *
+ * @param string $providerKey The provider key in the format "appId__providerId"
+ * @param bool $isInitiated
+ * @return void
+ * @throws \RuntimeException when the provider key format is invalid
+ */
+ public function setProviderInitiated(string $providerKey, bool $isInitiated): void {
+ $provider = $this->getProvider($providerKey);
+ if ($provider === null) {
+ return;
+ }
+ [$appId, $providerId] = $this->getProviderParts($providerKey);
+ /** @psalm-suppress ArgumentTypeCoercion */
+ $this->updateProvider($appId, $providerId, $provider['classString'], $isInitiated);
+ }
+
/**
* @param string $appId
* @param ?string $providerId
diff --git a/lib/Service/ReindexService.php b/lib/Service/ReindexService.php
new file mode 100644
index 00000000..409c9369
--- /dev/null
+++ b/lib/Service/ReindexService.php
@@ -0,0 +1,260 @@
+
+ */
+ public function getRegisteredProviderKeys(): array {
+ return $this->uniqueSorted(array_merge(
+ [ProviderConfigService::getDefaultProviderKey()],
+ array_keys($this->providerConfig->getProviders()),
+ ));
+ }
+
+ /**
+ * Keys of the providers that can be wiped: the registered ones and the ones that still have
+ * documents in the vector DB, e.g. of an app that has been removed since.
+ *
+ * @return list
+ * @throws \RuntimeException when the backend could not be reached
+ */
+ public function getWipeableProviderKeys(): array {
+ return $this->uniqueSorted(array_merge(
+ $this->getRegisteredProviderKeys(),
+ array_keys($this->langRopeService->getIndexedDocumentsCounts()),
+ ));
+ }
+
+ /**
+ * @param array $keys
+ * @return list
+ */
+ private function uniqueSorted(array $keys): array {
+ $keys = array_values(array_unique(array_map(strval(...), $keys)));
+ sort($keys);
+ return $keys;
+ }
+
+ /**
+ * Schedule the full filesystem crawl and the initial import of the content providers.
+ * Jobs that are already scheduled are not scheduled again.
+ *
+ * @param list|null $providerKeys null means everything. Without a scope only the content
+ * providers that have not been imported yet are imported (which is all of them after a full wipe),
+ * content providers explicitly specified are set as not initiated and scheduled for import.
+ * @return bool whether anything new was scheduled
+ */
+ public function scheduleReindex(?array $providerKeys = null): bool {
+ $scheduled = false;
+ $filesKey = ProviderConfigService::getDefaultProviderKey();
+
+ foreach ($this->providerConfig->getProviders() as $key => $provider) {
+ if ($key === $filesKey || $provider['classString'] === '') {
+ continue;
+ }
+ if ($providerKeys === null) {
+ // a full wipe turns all providers as not initiated so they will be scheduled for import and not skipped
+ if ($provider['isInitiated']) {
+ continue;
+ }
+ } elseif (in_array($key, $providerKeys, true)) {
+ $this->providerConfig->setProviderInitiated($key, false);
+ } else {
+ continue;
+ }
+ if ($this->jobList->has(InitialContentImportJob::class, $provider['classString'])) {
+ continue;
+ }
+ $this->jobList->add(InitialContentImportJob::class, $provider['classString']);
+ $scheduled = true;
+ }
+
+ if (($providerKeys === null || in_array($filesKey, $providerKeys, true))
+ && !$this->jobList->has(SchedulerJob::class, null)
+ ) {
+ $this->jobList->add(SchedulerJob::class);
+ $scheduled = true;
+ }
+ return $scheduled;
+ }
+
+ /**
+ * Delete everything that was queued or indexed, here and in context_chat_backend's vector DB.
+ * No queue items are handed to the backend until the wipe is over, and the documents it is still
+ * indexing are waited for, so that its workers, possibly running elsewhere on Kubernetes, cannot
+ * write to the vector DB while it is wiped.
+ * It is safe to run this again when the backend request fails.
+ *
+ * @param list|null $providerKeys null means everything including the actions queue, otherwise only the given providers
+ * @param bool $ignoreHandedOut do not wait for the documents the backend is indexing, for when some documents are stuck
+ * in the queue for 24h due to error or if the backend stopped while indexing them/having a lock
+ * @param ?callable(string): void $onStep called with a short description before each step, for progress output
+ * @throws \OCP\DB\Exception
+ * @throws \RuntimeException when the vector DB could not be wiped
+ */
+ public function wipe(?array $providerKeys = null, bool $ignoreHandedOut = false, ?callable $onStep = null): void {
+ $onStep = $onStep ?? static function (string $_): void {
+ };
+
+ $onStep('Pausing the hand-out of queued documents and actions to context_chat_backend');
+ $pausedUntil = $this->fetchGuard->pause();
+ try {
+ if ($ignoreHandedOut) {
+ $onStep('Not waiting for the documents context_chat_backend may still be indexing (--ignore-handed-out)');
+ } else {
+ $onStep('Checking for documents context_chat_backend is still indexing');
+ $this->fetchGuard->waitForHandedOutDocuments(static function (int $locked) use ($onStep): void {
+ $onStep('Waiting for context_chat_backend to finish indexing ' . $locked . ' documents');
+ });
+ }
+
+ if ($providerKeys === null) {
+ $this->wipeAll($onStep);
+ return;
+ }
+
+ foreach (array_unique($providerKeys) as $providerKey) {
+ $this->logger->info('Wiping the Context Chat data of provider ' . $providerKey);
+ if ($providerKey === ProviderConfigService::getDefaultProviderKey()) {
+ $onStep('Removing the crawl jobs, clearing the files indexing queue and the file system events, resetting the indexing statistics');
+ $this->clearFilesState();
+ } else {
+ $onStep('Removing the import job and clearing the indexing queue of ' . $providerKey);
+ $this->clearContentProviderState($providerKey);
+ }
+ $onStep('Deleting the documents of ' . $providerKey . ' from the vector DB');
+ $this->langRopeService->deleteProvider($providerKey);
+ }
+ } finally {
+ $onStep('Resuming the hand-out of queued documents and actions');
+ $this->fetchGuard->resume($pausedUntil);
+ }
+ }
+
+ /**
+ * @param callable(string): void $onStep
+ */
+ private function wipeAll(callable $onStep): void {
+ $this->logger->info('Wiping all the Context Chat data');
+
+ $onStep('Removing the scheduled crawl and content import jobs');
+ // stop the crawl and the content provider imports from queueing more items
+ foreach ([SchedulerJob::class, StorageCrawlJob::class, InitialContentImportJob::class] as $jobClass) {
+ $this->jobList->remove($jobClass);
+ }
+ $onStep('Clearing the indexing, actions and file system events queues');
+ $this->clearTables(self::QUEUE_TABLES);
+ $onStep('Resetting the indexing statistics and the content providers\' import state');
+ $this->resetStats();
+ $this->providerConfig->resetProvidersInitiation();
+
+ $onStep('Deleting all the documents from the vector DB');
+ $this->langRopeService->deleteAll();
+ $this->logger->info('Wiped all the Context Chat data');
+ }
+
+ /**
+ * The actions queue is left alone, it is shared by all the providers and the users.
+ * Leftover actions for the wiped documents are skipped by the backend.
+ */
+ private function clearFilesState(): void {
+ foreach ([SchedulerJob::class, StorageCrawlJob::class] as $jobClass) {
+ $this->jobList->remove($jobClass);
+ }
+ $this->clearTables(self::FILES_QUEUE_TABLES);
+ // the stats only track the files
+ $this->resetStats();
+ }
+
+ private function clearContentProviderState(string $providerKey): void {
+ $provider = $this->providerConfig->getProvider($providerKey);
+ if ($provider !== null && $provider['classString'] !== '') {
+ $this->jobList->remove(InitialContentImportJob::class, $provider['classString']);
+ }
+
+ [$appId, $providerId] = $this->providerConfig->getProviderParts($providerKey);
+ $this->contentQueue->removeProvider($appId, $providerId);
+
+ if ($provider !== null) {
+ $this->providerConfig->setProviderInitiated($providerKey, false);
+ }
+ }
+
+ /**
+ * @param list $tables
+ * @throws \RuntimeException
+ */
+ private function clearTables(array $tables): void {
+ $this->db->beginTransaction();
+ try {
+ foreach ($tables as $table) {
+ $this->db->getQueryBuilder()->delete($table)->executeStatement();
+ }
+ $this->db->commit();
+ } catch (\Throwable $e) {
+ $this->db->rollBack();
+ throw new \RuntimeException('Failed to clear tables ' . implode(', ', $tables) . ': ' . $e->getMessage(), 0, $e);
+ }
+ }
+
+ private function resetStats(): void {
+ foreach (self::STATS_CONFIG_KEYS as $key) {
+ $this->appConfig->deleteAppValue($key);
+ }
+ // the time taken for the next complete index is measured from here
+ $this->appConfig->setAppValueInt('installed_time', $this->timeFactory->getTime(), lazy: true);
+ }
+}
diff --git a/lib/Service/StatisticsService.php b/lib/Service/StatisticsService.php
index 7d214ef0..c3086b52 100644
--- a/lib/Service/StatisticsService.php
+++ b/lib/Service/StatisticsService.php
@@ -69,9 +69,7 @@ public function getStatistics(): array {
$this->logger->error($e->getMessage(), ['exception' => $e]);
$stats['backend_available'] = false;
}
- if (empty($stats['vectordb_document_counts'] ?? [])) {
- $stats['vectordb_document_counts'] = [ ProviderConfigService::getDefaultProviderKey() => 0 ];
- }
+ $stats['vectordb_document_counts'] = ($stats['vectordb_document_counts'] ?? []) + [ ProviderConfigService::getDefaultProviderKey() => 0 ];
try {
$queued_files_count = $this->queueService->count();
diff --git a/tests/integration/ReindexServiceTest.php b/tests/integration/ReindexServiceTest.php
new file mode 100644
index 00000000..cda6319d
--- /dev/null
+++ b/tests/integration/ReindexServiceTest.php
@@ -0,0 +1,500 @@
+> */
+ private array $savedConfig;
+
+ protected function setUp(): void {
+ parent::setUp();
+ $this->db = Server::get(IDBConnection::class);
+ $this->jobList = Server::get(IJobList::class);
+ // the app scoped config is only available from the app's container
+ $container = (new Application())->getContainer();
+ $this->appConfig = $container->get(IAppConfig::class);
+ $this->providerConfig = $container->get(ProviderConfigService::class);
+ $this->langRope = $this->createMock(LangRopeService::class);
+ $this->fetchGuard = $this->newGuard(Server::get(ITimeFactory::class));
+ $time = $this->createMock(ITimeFactory::class);
+ $time->method('getTime')->willReturn(self::NOW);
+
+ $this->service = new ReindexService(
+ $this->db,
+ $this->jobList,
+ $this->appConfig,
+ $this->providerConfig,
+ $this->langRope,
+ Server::get(QueueContentItemMapper::class),
+ $this->fetchGuard,
+ $time,
+ Server::get(Logger::class),
+ );
+
+ $this->savedConfig = $this->readConfig();
+ $this->clearState();
+ $this->appConfig->setAppValueString('providers', '', lazy: true);
+ }
+
+ protected function tearDown(): void {
+ $this->clearState();
+ // the wipe resets the stats and the providers, put back what the instance had before
+ $qb = $this->db->getQueryBuilder();
+ $qb->delete('appconfig')
+ ->where($qb->expr()->eq('appid', $qb->createNamedParameter(Application::APP_ID)))
+ ->executeStatement();
+ foreach ($this->savedConfig as $row) {
+ $qb = $this->db->getQueryBuilder();
+ $qb->insert('appconfig');
+ foreach ($row as $column => $value) {
+ $qb->setValue($column, $qb->createNamedParameter($value));
+ }
+ $qb->executeStatement();
+ }
+ Server::get(\OCP\IAppConfig::class)->clearCache(true);
+ parent::tearDown();
+ }
+
+ /** @return list> */
+ private function readConfig(): array {
+ $qb = $this->db->getQueryBuilder();
+ $result = $qb->select('*')->from('appconfig')
+ ->where($qb->expr()->eq('appid', $qb->createNamedParameter(Application::APP_ID)))
+ ->executeQuery();
+ $rows = $result->fetchAll();
+ $result->closeCursor();
+ return $rows;
+ }
+
+ private function clearState(): void {
+ foreach (ReindexService::QUEUE_TABLES as $table) {
+ $this->db->getQueryBuilder()->delete($table)->executeStatement();
+ }
+ foreach ([SchedulerJob::class, StorageCrawlJob::class, InitialContentImportJob::class] as $jobClass) {
+ $this->jobList->remove($jobClass);
+ }
+ }
+
+ private function seedIndexedState(): void {
+ $file = new QueueFile();
+ $file->setFileId(42);
+ $file->setStorageId(1);
+ $file->setRootId(1);
+ $file->setUpdate(false);
+ Server::get(QueueMapper::class)->insertIntoQueue($file);
+
+ foreach (['someapp', 'otherapp'] as $appId) {
+ $item = new QueueContentItem();
+ $item->setItemId('item1');
+ $item->setAppId($appId);
+ $item->setProviderId('prov');
+ $item->setTitle('title');
+ $item->setContent('content');
+ $item->setDocumentType('text');
+ $item->setLastModified(new \DateTime());
+ $item->setUsers('admin');
+ Server::get(QueueContentItemMapper::class)->insert($item);
+ }
+
+ $action = new QueueAction();
+ $action->setType(ActionType::DELETE_SOURCE_IDS);
+ $action->setPayload('{"sourceIds":["files__default: 42"]}');
+ Server::get(QueueActionMapper::class)->insertIntoQueue($action);
+
+ Server::get(FsEventMapper::class)->insertRow(FsEventType::CREATE->value, 'admin', 42);
+
+ $this->jobList->add(StorageCrawlJob::class, ['storage_id' => 1, 'root_id' => 1, 'overridden_root' => 1, 'last_file_id' => 0]);
+ $this->jobList->add(InitialContentImportJob::class, self::PROVIDER_CLASS);
+ $this->jobList->add(InitialContentImportJob::class, self::OTHER_PROVIDER_CLASS);
+
+ $this->appConfig->setAppValueInt('last_indexed_time', 123, lazy: true);
+ $this->appConfig->setAppValueInt('last_enqueued_db_id', 7, lazy: true);
+ $this->appConfig->setAppValueInt('eligible_files_count', 99, lazy: true);
+ $this->appConfig->setAppValueInt('eligible_files_count_time', 456, lazy: true);
+
+ $this->providerConfig->updateProvider('files', 'default', '', true);
+ $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, true);
+ $this->providerConfig->updateProvider('otherapp', 'prov', self::OTHER_PROVIDER_CLASS, true);
+ }
+
+ /** @return list */
+ private function contentQueueAppIds(): array {
+ $qb = $this->db->getQueryBuilder();
+ $result = $qb->selectDistinct('app_id')->from('context_chat_content_queue')->executeQuery();
+ $appIds = [];
+ while (($appId = $result->fetchOne()) !== false) {
+ $appIds[] = $appId;
+ }
+ $result->closeCursor();
+ return $appIds;
+ }
+
+ private function rowCount(string $table): int {
+ $qb = $this->db->getQueryBuilder();
+ return (int)$qb->select($qb->func()->count('*'))->from($table)->executeQuery()->fetchOne();
+ }
+
+ public function testWipeClearsAllLocalStateAndTheBackend(): void {
+ $this->seedIndexedState();
+ $this->langRope->expects($this->once())->method('deleteAll');
+ $this->langRope->expects($this->never())->method('deleteProvider');
+
+ $this->service->wipe();
+
+ foreach (ReindexService::QUEUE_TABLES as $table) {
+ $this->assertSame(0, $this->rowCount($table), $table . ' should be empty');
+ }
+ $this->assertFalse($this->jobList->has(StorageCrawlJob::class, ['storage_id' => 1, 'root_id' => 1, 'overridden_root' => 1, 'last_file_id' => 0]));
+ $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS));
+ $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::OTHER_PROVIDER_CLASS));
+ $this->assertSame(0, $this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true));
+ $this->assertSame(-1, $this->appConfig->getAppValueInt('last_enqueued_db_id', -1, lazy: true));
+ $this->assertSame(0, $this->appConfig->getAppValueInt('eligible_files_count_time', 0, lazy: true));
+ $this->assertSame(self::NOW, $this->appConfig->getAppValueInt('installed_time', 0, lazy: true));
+
+ $providers = $this->providerConfig->getProviders();
+ $this->assertFalse($providers['someapp__prov']['isInitiated'], 'content providers must be imported again');
+ $this->assertTrue($providers[ProviderConfigService::getDefaultProviderKey()]['isInitiated'], 'files are re-crawled by SchedulerJob instead');
+ }
+
+ public function testWipeClearsLocalStateBeforeTheBackendFails(): void {
+ $this->seedIndexedState();
+ $this->langRope->method('deleteAll')->willThrowException(new \RuntimeException('backend down'));
+
+ try {
+ $this->service->wipe();
+ $this->fail('wipe() must report the backend failure');
+ } catch (\RuntimeException $e) {
+ $this->assertSame('backend down', $e->getMessage());
+ }
+
+ // nothing may be indexed or pruned against a half-wiped vector DB, and a re-run must work
+ foreach (ReindexService::QUEUE_TABLES as $table) {
+ $this->assertSame(0, $this->rowCount($table), $table . ' should be empty');
+ }
+ }
+
+ public function testScheduleReindexAfterWipeQueuesCrawlAndProviderImportsOnce(): void {
+ $this->seedIndexedState();
+ $this->service->wipe();
+
+ $this->assertTrue($this->service->scheduleReindex());
+ $this->assertFalse($this->service->scheduleReindex(), 'an already scheduled crawl is not scheduled again');
+
+ $this->assertTrue($this->jobList->has(SchedulerJob::class, null));
+ $this->assertTrue($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS));
+ $importJobs = iterator_to_array($this->jobList->getJobsIterator(InitialContentImportJob::class, null, 0), false);
+ $this->assertCount(2, $importJobs);
+ }
+
+ public function testScheduleReindexSkipsInitiatedProviders(): void {
+ $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, true);
+
+ $this->service->scheduleReindex();
+
+ $this->assertTrue($this->jobList->has(SchedulerJob::class, null));
+ $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS));
+ }
+
+ public function testWipeFilesOnlyLeavesContentProvidersAndActions(): void {
+ $this->seedIndexedState();
+ $this->langRope->expects($this->never())->method('deleteAll');
+ $this->langRope->expects($this->once())->method('deleteProvider')->with('files__default');
+
+ $this->service->wipe(['files__default']);
+
+ $this->assertSame(0, $this->rowCount('context_chat_queue'));
+ $this->assertSame(0, $this->rowCount('context_chat_fs_events'));
+ $this->assertSame(2, $this->rowCount('context_chat_content_queue'), 'content providers must be kept');
+ $this->assertSame(1, $this->rowCount('context_chat_action_queue'), 'actions of other providers and users must be kept');
+ $this->assertFalse($this->jobList->has(StorageCrawlJob::class, ['storage_id' => 1, 'root_id' => 1, 'overridden_root' => 1, 'last_file_id' => 0]));
+ $this->assertTrue($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS));
+ $this->assertSame(0, $this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true));
+ $this->assertTrue($this->providerConfig->getProviders()['someapp__prov']['isInitiated']);
+ }
+
+ public function testWipeOneContentProviderLeavesTheRest(): void {
+ $this->seedIndexedState();
+ $this->langRope->expects($this->never())->method('deleteAll');
+ $this->langRope->expects($this->once())->method('deleteProvider')->with('someapp__prov');
+
+ $this->service->wipe(['someapp__prov']);
+
+ $this->assertSame(['otherapp'], $this->contentQueueAppIds());
+ $this->assertSame(1, $this->rowCount('context_chat_queue'), 'files must be kept');
+ $this->assertSame(1, $this->rowCount('context_chat_fs_events'), 'files must be kept');
+ $this->assertSame(1, $this->rowCount('context_chat_action_queue'));
+ $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS));
+ $this->assertTrue($this->jobList->has(InitialContentImportJob::class, self::OTHER_PROVIDER_CLASS));
+ $this->assertSame(123, $this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true), 'file stats must be kept');
+
+ $providers = $this->providerConfig->getProviders();
+ $this->assertFalse($providers['someapp__prov']['isInitiated']);
+ $this->assertTrue($providers['otherapp__prov']['isInitiated']);
+ }
+
+ public function testWipeOfAnUnregisteredProviderOnlyDeletesInTheBackend(): void {
+ $this->seedIndexedState();
+ $this->langRope->expects($this->once())->method('deleteProvider')->with('removedapp__prov');
+
+ $this->service->wipe(['removedapp__prov']);
+
+ $this->assertSame(2, $this->rowCount('context_chat_content_queue'));
+ $this->assertArrayNotHasKey('removedapp__prov', $this->providerConfig->getProviders());
+ }
+
+ public function testScheduleReindexOfANamedProviderReimportsItEvenIfImported(): void {
+ $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, true);
+ $this->providerConfig->updateProvider('otherapp', 'prov', self::OTHER_PROVIDER_CLASS, true);
+
+ $this->assertTrue($this->service->scheduleReindex(['someapp__prov']));
+
+ $this->assertTrue($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS));
+ $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::OTHER_PROVIDER_CLASS));
+ $this->assertFalse($this->jobList->has(SchedulerJob::class, null), 'files are not in the scope');
+ // InitialContentImportJob skips initiated providers
+ $this->assertFalse($this->providerConfig->getProviders()['someapp__prov']['isInitiated']);
+ }
+
+ public function testScheduleReindexOfFilesOnly(): void {
+ $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, false);
+
+ $this->assertTrue($this->service->scheduleReindex(['files__default']));
+ $this->assertFalse($this->service->scheduleReindex(['files__default']));
+
+ $this->assertTrue($this->jobList->has(SchedulerJob::class, null));
+ $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS));
+ }
+
+ public function testWipeableProviderKeysIncludeProvidersOnlyKnownToTheBackend(): void {
+ $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, true);
+ $this->langRope->method('getIndexedDocumentsCounts')->willReturn(['files__default' => 3, 'removedapp__prov' => 1]);
+
+ $this->assertSame(['files__default', 'someapp__prov'], $this->service->getRegisteredProviderKeys());
+ $this->assertSame(['files__default', 'removedapp__prov', 'someapp__prov'], $this->service->getWipeableProviderKeys());
+ }
+
+ private function newGuard(ITimeFactory $time): FetchGuard {
+ return new FetchGuard($this->db, $this->appConfig, $time, Server::get(QueueMapper::class), Server::get(QueueContentItemMapper::class));
+ }
+
+ /** @return ITimeFactory&MockObject a clock that advances one second per sleep(1) of the guard */
+ private function clock(int &$now): ITimeFactory {
+ $time = $this->createMock(ITimeFactory::class);
+ $time->method('getTime')->willReturnCallback(function () use (&$now) {
+ return $now;
+ });
+ return $time;
+ }
+
+ public function testFetchingIsPausedOnlyWhileTheBackendIsWiped(): void {
+ $this->seedIndexedState();
+ $seen = null;
+ $this->langRope->method('deleteAll')->willReturnCallback(function () use (&$seen) {
+ $seen = $this->fetchGuard->isPaused();
+ });
+
+ $this->service->wipe();
+
+ $this->assertTrue($seen, 'nothing may be handed out while the vector DB is wiped');
+ $this->assertFalse($this->fetchGuard->isPaused());
+ }
+
+ public function testFetchingResumesWhenTheBackendFails(): void {
+ $this->seedIndexedState();
+ $this->langRope->method('deleteAll')->willThrowException(new \RuntimeException('backend down'));
+
+ try {
+ $this->service->wipe();
+ } catch (\RuntimeException) {
+ }
+
+ $this->assertFalse($this->fetchGuard->isPaused());
+ }
+
+ public function testWipeIsRefusedWhileDocumentsAreStillBeingIndexed(): void {
+ $this->seedIndexedState();
+ $queueMapper = Server::get(QueueMapper::class);
+ $queueMapper->lock($queueMapper->getMaxId());
+ $this->appConfig->setAppValueInt('wipe_drain_timeout', 0, lazy: true);
+ $this->langRope->expects($this->never())->method('deleteAll');
+
+ try {
+ $this->service->wipe();
+ $this->fail('the wipe must not run while a document is being indexed');
+ } catch (\RuntimeException $e) {
+ $this->assertStringContainsString('1 documents are still being indexed', $e->getMessage());
+ } finally {
+ $this->appConfig->deleteAppValue('wipe_drain_timeout');
+ }
+
+ $this->assertSame(1, $queueMapper->countLocked(), 'nothing may be cleared when the wipe is refused');
+ $this->assertFalse($this->fetchGuard->isPaused());
+ }
+
+ public function testResumeDoesNotLiftANewerPause(): void {
+ $now = self::NOW;
+ $guard = $this->newGuard($this->clock($now));
+
+ $first = $guard->pause();
+ $now += 10;
+ $second = $guard->pause();
+ $guard->resume($first);
+
+ $this->assertTrue($guard->isPaused(), 'the second wipe is still running');
+ $guard->resume($second);
+ $this->assertFalse($guard->isPaused());
+ }
+
+ public function testPauseExpiresOnItsOwn(): void {
+ $now = self::NOW;
+ $guard = $this->newGuard($this->clock($now));
+
+ $until = $guard->pause();
+ $this->assertTrue($guard->isPaused());
+ $now = $until;
+
+ $this->assertFalse($guard->isPaused(), 'a wipe that died must not stop the indexing forever');
+ $guard->resume($until);
+ }
+
+ public function testWaitEndsOnceTheHandedOutBatchIsReportedBack(): void {
+ $queueMapper = Server::get(QueueMapper::class);
+ $file = new QueueFile();
+ $file->setFileId(43);
+ $file->setStorageId(1);
+ $file->setRootId(1);
+ $file->setUpdate(false);
+ $id = $queueMapper->insertIntoQueue($file)->getId();
+ $queueMapper->lock($id);
+
+ $checks = 0;
+ $time = $this->createMock(ITimeFactory::class);
+ $time->method('getTime')->willReturnCallback(function () use (&$checks, $queueMapper, $id) {
+ // the backend reports the batch back while the wipe is waiting
+ if (++$checks === 3) {
+ $queueMapper->removeFromQueue([$id]);
+ }
+ return self::NOW;
+ });
+
+ $reported = [];
+ $this->newGuard($time)->waitForHandedOutDocuments(function (int $locked) use (&$reported) {
+ $reported[] = $locked;
+ });
+
+ $this->assertSame(0, $queueMapper->countLocked());
+ $this->assertSame([1], $reported, 'the count is reported once, not on every poll');
+ }
+
+ /**
+ * The steps are reported in order, each one before its work: what the admin reads is what runs next.
+ */
+ public function testWipeReportsEachStepBeforeDoingIt(): void {
+ $this->seedIndexedState();
+ $events = [];
+ $this->langRope->method('deleteAll')->willReturnCallback(function () use (&$events) {
+ $events[] = 'backend:deleteAll';
+ });
+
+ $this->service->wipe(null, false, function (string $message) use (&$events) {
+ // mark the moment the queues are emptied, to check the order against the work
+ $events[] = $message . ($this->rowCount('context_chat_queue') === 0 ? ' [queues empty]' : '');
+ });
+
+ $this->assertSame([
+ 'Pausing the hand-out of queued documents and actions to context_chat_backend',
+ 'Checking for documents context_chat_backend is still indexing',
+ 'Removing the scheduled crawl and content import jobs',
+ 'Clearing the indexing, actions and file system events queues',
+ 'Resetting the indexing statistics and the content providers\' import state [queues empty]',
+ 'Deleting all the documents from the vector DB [queues empty]',
+ 'backend:deleteAll',
+ 'Resuming the hand-out of queued documents and actions [queues empty]',
+ ], $events);
+ }
+
+ public function testWipeOfOneProviderReportsItsSteps(): void {
+ $this->seedIndexedState();
+ $events = [];
+ $this->langRope->method('deleteProvider')->willReturnCallback(function (string $key) use (&$events) {
+ $events[] = 'backend:deleteProvider ' . $key;
+ });
+
+ $this->service->wipe(['someapp__prov'], true, function (string $message) use (&$events) {
+ $events[] = $message;
+ });
+
+ $this->assertSame([
+ 'Pausing the hand-out of queued documents and actions to context_chat_backend',
+ 'Not waiting for the documents context_chat_backend may still be indexing (--ignore-handed-out)',
+ 'Removing the import job and clearing the indexing queue of someapp__prov',
+ 'Deleting the documents of someapp__prov from the vector DB',
+ 'backend:deleteProvider someapp__prov',
+ 'Resuming the hand-out of queued documents and actions',
+ ], $events);
+ }
+
+ public function testResumeIsReportedWhenTheBackendFails(): void {
+ $this->seedIndexedState();
+ $this->langRope->method('deleteAll')->willThrowException(new \RuntimeException('backend down'));
+ $events = [];
+
+ try {
+ $this->service->wipe(null, false, function (string $message) use (&$events) {
+ $events[] = $message;
+ });
+ $this->fail('the backend failure must be reported');
+ } catch (\RuntimeException) {
+ }
+
+ $this->assertSame('Deleting all the documents from the vector DB', $events[count($events) - 2]);
+ $this->assertSame('Resuming the hand-out of queued documents and actions', end($events));
+ }
+}