diff --git a/.github/workflows/integration-test-cron.yml b/.github/workflows/integration-test-cron.yml index 004e5c75..162cf723 100644 --- a/.github/workflows/integration-test-cron.yml +++ b/.github/workflows/integration-test-cron.yml @@ -127,6 +127,8 @@ jobs: with: repository: nextcloud/context_chat_backend path: context_chat_backend/ + # todo: remove this + ref: feat/vector-db-wipe persist-credentials: false - name: Checkout viewer @@ -410,6 +412,73 @@ jobs: kill -9 $WORKER1_PID kill -9 $WORKER2_PID + - name: Wipe and reindex only the files provider + run: | + ./occ context_chat:wipe --force --provider files__default --reindex + + stats=$(./occ context_chat:stats --json) + echo "$stats" + indexed=$(echo "$stats" | jq '.vectordb_document_counts.files__default // 0') + if [ "$indexed" != "0" ]; then + echo "Error: expected no indexed files right after the files wipe, got $indexed" + exit 1 + fi + + echo "::group::Waiting up to 10 minutes for the files to be indexed again" + for i in {1..60}; do + stats=$(timeout 30 ./occ context_chat:stats --json || true) + indexed=$(echo "$stats" | jq '.vectordb_document_counts.files__default // 0' 2>/dev/null || echo 0) + echo "Attempt $i: indexed=$indexed" + if [ "$indexed" -ge 10 ]; then + echo "::endgroup::" + echo "Files reindexed, $indexed documents" + exit 0 + fi + sleep 10 + done + echo "::endgroup::" + echo "Error: files were not indexed again after the files wipe" + exit 1 + + - name: Wipe all the indexed data and check that the queues and the vector DB are empty + run: | + ./occ context_chat:wipe --force + + stats=$(./occ context_chat:stats --json) + echo "$stats" + indexed=$(echo "$stats" | jq '[.vectordb_document_counts | to_entries[].value] | add // 0') + queued=$(echo "$stats" | jq '[.queued_documents_counts | to_entries[].value] | add // 0') + actions=$(echo "$stats" | jq '.queued_actions_count') + fs_events=$(echo "$stats" | jq '.queued_fs_events_count') + # the queued counts leave out locked items, check those separately + queued_locked=$(echo "$stats" | jq '[.queued_documents_locked_counts | to_entries[].value] | add // 0') + actions_locked=$(echo "$stats" | jq '.queued_actions_locked_count') + backend_available=$(echo "$stats" | jq '.backend_available') + echo "indexed=$indexed queued=$queued queued_locked=$queued_locked actions=$actions actions_locked=$actions_locked fs_events=$fs_events backend_available=$backend_available" + if [ "$backend_available" != "true" ] || [ "$indexed" != "0" ] || [ "$queued" != "0" ] || [ "$queued_locked" != "0" ] || [ "$actions" != "0" ] || [ "$actions_locked" != "0" ] || [ "$fs_events" != "0" ]; then + echo "Error: expected an empty vector DB and empty queues after the wipe" + exit 1 + fi + + - name: Reindex and check that documents get indexed again + run: | + ./occ context_chat:reindex + echo "::group::Waiting up to 10 minutes for documents to be indexed again" + for i in {1..60}; do + stats=$(timeout 30 ./occ context_chat:stats --json || true) + indexed=$(echo "$stats" | jq '.vectordb_document_counts.files__default // 0' 2>/dev/null || echo 0) + echo "Attempt $i: indexed=$indexed" + if [ "$indexed" -ge 10 ]; then + echo "Reindex works, $indexed documents indexed again" + echo "::endgroup::" + exit 0 + fi + sleep 10 + done + echo "::endgroup::" + echo "Error: documents were not indexed again after the reindex" + exit 1 + - name: Show nextcloud logs if: always() run: | diff --git a/.github/workflows/integration-test-file-listener.yml b/.github/workflows/integration-test-file-listener.yml index 3a5e2ac3..2664d0f0 100644 --- a/.github/workflows/integration-test-file-listener.yml +++ b/.github/workflows/integration-test-file-listener.yml @@ -115,6 +115,8 @@ jobs: with: repository: nextcloud/context_chat_backend path: context_chat_backend/ + # todo: remove this + ref: feat/vector-db-wipe persist-credentials: false - name: Checkout viewer diff --git a/appinfo/info.xml b/appinfo/info.xml index 2b8cda78..7075cb78 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -47,6 +47,7 @@ Refer to the [Context Chat Backend's readme](https://github.com/nextcloud/contex OCA\ContextChat\Command\Search OCA\ContextChat\Command\Statistics OCA\ContextChat\Command\Reindex + OCA\ContextChat\Command\Wipe diff --git a/lib/Command/Reindex.php b/lib/Command/Reindex.php index ca3de909..e3b2dc6f 100644 --- a/lib/Command/Reindex.php +++ b/lib/Command/Reindex.php @@ -9,10 +9,10 @@ namespace OCA\ContextChat\Command; -use OCA\ContextChat\BackgroundJobs\SchedulerJob; -use OCP\BackgroundJob\IJobList; +use OCA\ContextChat\Service\ReindexService; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; /** @@ -20,32 +20,55 @@ * * SchedulerJob -> StorageCrawlJob -> IndexerJob is seeded only by the repair step and * removes itself once the initial crawl finishes. There is otherwise no way to re-enumerate - * mounts (e.g. after installing on an instance whose files predate the app, or to recover a crawl - * that did not complete) short of reinstalling the app. This command re-adds SchedulerJob so the - * full enumeration runs again; it is a no-op if one is already scheduled, and already-indexed - * files are skipped. + * mounts (e.g. to recover a crawl that did not complete, or after context_chat:wipe) short of + * reinstalling the app. This command re-adds SchedulerJob so the full enumeration runs again if one + * doesn't exist already. + * Content providers' initial import also gets scheduled (e.g. after context_chat:wipe) if it has not run yet. */ class Reindex extends Command { public function __construct( - private IJobList $jobList, + private ReindexService $reindexService, ) { parent::__construct(); } protected function configure() { $this->setName('context_chat:reindex') - ->setDescription('Schedule a full re-crawl of all the files in all the mounts. Indexed files are not re-indexed when compared against context_chat_backend\'s vector DB. Content providers are not re-indexed.'); + ->setDescription( + 'Schedule a full re-crawl of all the files in all the mounts and the import of the content providers' + . ' that were not imported yet. Indexed files are not re-indexed when compared against context_chat_backend\'s vector DB.' + ) + ->addOption( + 'provider', + 'p', + InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY, + 'Only reindex this provider, e.g. "files__default" (appId__providerId) for the files.' + . ' A named content provider is imported again even if it was imported before. Multiple values can be specified by repeating the option.', + ); } protected function execute(InputInterface $input, OutputInterface $output): int { - if ($this->jobList->has(SchedulerJob::class, null)) { - $output->writeln('A full re-crawl is already scheduled; nothing to do.'); + /** @var list $providerKeys */ + $providerKeys = $input->getOption('provider'); + $scope = $providerKeys === [] ? null : $providerKeys; + + if ($scope !== null) { + $known = $this->reindexService->getRegisteredProviderKeys(); + $unknown = array_diff($scope, $known); + if ($unknown !== []) { + $output->writeln('Unknown provider(s): ' . implode(', ', $unknown) . ''); + $output->writeln('Registered providers: ' . implode(', ', $known)); + return 1; + } + } + + if (!$this->reindexService->scheduleReindex($scope)) { + $output->writeln('The re-index is already scheduled; nothing to do.'); return 0; } - $this->jobList->add(SchedulerJob::class); - $output->writeln('Scheduled a full re-crawl. SchedulerJob will enumerate all mounts on its next run.'); + $output->writeln('Scheduled the re-index. The background jobs will queue the documents on their next run.'); return 0; } } diff --git a/lib/Command/Wipe.php b/lib/Command/Wipe.php new file mode 100644 index 00000000..df5639be --- /dev/null +++ b/lib/Command/Wipe.php @@ -0,0 +1,140 @@ +setName('context_chat:wipe') + ->setDescription( + 'Delete the indexed documents from context_chat_backend\'s vector DB and clear the queues.' + . ' With --provider option, only the provider\'s data is deleted, the actions queue, and the indexing statistics are left intact.' + . ' Nothing gets indexed again until context_chat:reindex is run or the --reindex option is used.' + ) + ->addOption( + 'provider', + 'p', + InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY, + 'Only wipe this provider, e.g. "files__default" (appId__providerId) for the files. Multiple values can be specified by repeating the option.', + ) + ->addOption( + 'reindex', + null, + InputOption::VALUE_NONE, + 'Schedule a re-crawl of the files and a re-import of the content providers that were wiped', + ) + ->addOption( + 'ignore-handed-out', + null, + InputOption::VALUE_NONE, + 'Do not wait for the documents context_chat_backend is still indexing. Only use this when the backend was stopped while indexing them, otherwise they can be written to the vector DB after the wipe.', + ) + ->addOption( + 'force', + 'f', + InputOption::VALUE_NONE, + 'Do not ask for confirmation', + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + /** @var list $providerKeys */ + $providerKeys = $input->getOption('provider'); + $scope = $providerKeys === [] ? null : $providerKeys; + + if ($scope !== null) { + try { + $known = $this->reindexService->getWipeableProviderKeys(); + } catch (\RuntimeException $e) { + $output->writeln('Could not list the indexed providers: ' . $e->getMessage() . ''); + return 1; + } + $unknown = array_diff($scope, $known); + if ($unknown !== []) { + $output->writeln('Unknown provider(s): ' . implode(', ', $unknown) . ''); + $output->writeln('Known providers: ' . implode(', ', $known)); + return 1; + } + } + + $scopeText = $scope === null ? '' : ' of ' . implode(', ', $scope); + + if (!$input->getOption('force')) { + /** @var QuestionHelper $helper */ + $helper = $this->getHelper('question'); + $question = new ConfirmationQuestion( + 'All the indexed documents in Context Chat' + . $scopeText + . ' will be deleted. The original documents are not touched. Continue? [y/N] ', + false, + ); + if (!$helper->ask($input, $output, $question)) { + $output->writeln('Aborted, nothing was deleted. Use --force to skip the confirmation.'); + return 1; + } + } + + try { + $this->reindexService->wipe( + $scope, + (bool)$input->getOption('ignore-handed-out'), + static function (string $message) use ($output): void { + $output->writeln('' . $message . '...'); + }, + ); + } catch (\Throwable $e) { + $output->writeln('Wipe failed. The command can be safely run again once the problem is fixed.'); + $output->writeln('Error: ' . $e->getMessage() . ''); + $output->writeln('' . $e->getTraceAsString() . ''); + return 1; + } + $output->writeln('Deleted all the indexed documents' . $scopeText . ' and cleared the queues.'); + + // providers that are not registered anymore cannot be indexed again + $reindexScope = $scope === null + ? null + : array_values(array_intersect($scope, $this->reindexService->getRegisteredProviderKeys())); + if ($reindexScope === []) { + return 0; + } + + if ($input->getOption('reindex')) { + try { + $this->reindexService->scheduleReindex($reindexScope); + } catch (\Throwable $e) { + $output->writeln('Scheduling re-index failed. It can be tried again with context_chat:reindex.'); + $output->writeln('Error: ' . $e->getMessage() . ''); + $output->writeln('' . $e->getTraceAsString() . ''); + return 1; + } + $output->writeln('Scheduled the re-index.'); + } elseif ($reindexScope === null) { + $output->writeln('Run context_chat:reindex to index the documents again.'); + } else { + $output->writeln('Run context_chat:reindex --provider ' . implode(' --provider ', $reindexScope) . ' to index the documents again.'); + } + + return 0; + } +} diff --git a/lib/Controller/QueueController.php b/lib/Controller/QueueController.php index f35eaa14..d260afd5 100644 --- a/lib/Controller/QueueController.php +++ b/lib/Controller/QueueController.php @@ -17,6 +17,7 @@ use OCA\ContextChat\Db\QueueContentItemMapper; use OCA\ContextChat\Db\QueueFile; use OCA\ContextChat\Db\QueueMapper; +use OCA\ContextChat\Service\FetchGuard; use OCA\ContextChat\Service\ProviderConfigService; use OCA\ContextChat\Type\Source; use OCP\AppFramework\Http; @@ -47,6 +48,7 @@ public function __construct( private IJobList $jobList, private ITimeFactory $timeFactory, private QueueMapper $queueMapper, + private FetchGuard $fetchGuard, string $corsMethods = 'PUT, POST, GET, DELETE, PATCH', string $corsAllowedHeaders = 'Authorization, Content-Type, Accept, OCS-APIRequest', int $corsMaxAge = 1728000, @@ -106,6 +108,13 @@ public function getDocumentsQueueItems( if ($n > $maxN) { $n = $maxN; } + // hand out nothing, the backend waits and asks again + if ($this->fetchGuard->isPaused()) { + return new DataResponse([ + 'files' => new \stdClass(), + 'content_providers' => new \stdClass(), + ]); + } try { $files = []; while (count($files) < $n) { @@ -216,6 +225,10 @@ public function countDocumentsQueueItems(QueueMapper $queueMapper, QueueContentI #[ExAppRequired] #[ApiRoute(verb: 'GET', url: '/queues/actions/')] public function getActionsQueueItems(QueueActionMapper $queueActionMapper, int $n = 512) : DataResponse { + if ($this->fetchGuard->isPaused()) { + // stdClass is used to represent an empty object in the response + return new DataResponse(['actions' => new \stdClass()]); + } try { $actions = []; while (count($actions) < $n) { diff --git a/lib/Db/QueueContentItemMapper.php b/lib/Db/QueueContentItemMapper.php index 2cc79cfc..b61f1cd6 100644 --- a/lib/Db/QueueContentItemMapper.php +++ b/lib/Db/QueueContentItemMapper.php @@ -71,6 +71,17 @@ public function removeFromQueue(array $ids): void { } } + /** + * @throws \OCP\DB\Exception + */ + public function removeProvider(string $appId, string $providerId): void { + $qb = $this->db->getQueryBuilder(); + $qb->delete($this->getTableName()) + ->where($qb->expr()->eq('app_id', $qb->createNamedParameter($appId))) + ->andWhere($qb->expr()->eq('provider_id', $qb->createNamedParameter($providerId))) + ->executeStatement(); + } + /** * @throws \OCP\DB\Exception * @return array diff --git a/lib/Service/FetchGuard.php b/lib/Service/FetchGuard.php new file mode 100644 index 00000000..4ede4c99 --- /dev/null +++ b/lib/Service/FetchGuard.php @@ -0,0 +1,126 @@ +timeFactory->getTime() + $this->getDrainTimeout() + $this->getRequestTimeout() + self::MARGIN; + $this->appConfig->setAppValueInt(self::CONFIG_KEY, $until, lazy: true); + return $until; + } + + /** + * Lifts the pause, unless another one was started since. + */ + public function resume(int $until): void { + if ($this->getPausedUntil() === $until) { + $this->appConfig->deleteAppValue(self::CONFIG_KEY); + } + } + + public function isPaused(): bool { + return $this->getPausedUntil() > $this->timeFactory->getTime(); + } + + /** + * Waits for the documents the backend took before the pause to be reported back. A document that + * is still being indexed would be written to the vector DB after it was wiped. + * + * @param ?callable(int): void $onWaiting called with the number of documents still being indexed, + * once before waiting and again whenever that number changes + * @throws \RuntimeException when they are not reported back in time + * @throws \OCP\DB\Exception + */ + public function waitForHandedOutDocuments(?callable $onWaiting = null): void { + $deadline = $this->timeFactory->getTime() + $this->getDrainTimeout(); + $reported = null; + while (($locked = $this->countHandedOutDocuments()) > 0) { + if ($onWaiting !== null && $locked !== $reported) { + $onWaiting($locked); + $reported = $locked; + } + if ($this->timeFactory->getTime() >= $deadline) { + throw new \RuntimeException( + $locked . ' documents are still being indexed by context_chat_backend after waiting ' + . $this->getDrainTimeout() . ' seconds. Try again later. If the backend was stopped while' + . ' indexing them, they would only be released after a day, use --ignore-handed-out then.' + ); + } + sleep(1); + } + } + + /** + * @throws \OCP\DB\Exception + */ + private function countHandedOutDocuments(): int { + return $this->queueMapper->countLocked() + intval(array_sum($this->contentQueue->countLocked())); + } + + private function getDrainTimeout(): int { + return $this->appConfig->getAppValueInt('wipe_drain_timeout', self::DEFAULT_DRAIN_TIMEOUT, lazy: true); + } + + private function getRequestTimeout(): int { + return (int)$this->appConfig->getAppValueString( + 'request_timeout', + strval(Application::CC_DEFAULT_REQUEST_TIMEOUT), + lazy: true, + ); + } + + /** + * Read from the database instead of IAppConfig. IAppConfig values are cached by each web server + * for a few seconds, and all of them must stop handing out items before the wipe starts. + * This ensures that we get the most up-to-date value for the pause state. + */ + private function getPausedUntil(): int { + $qb = $this->db->getQueryBuilder(); + $result = $qb->select('configvalue') + ->from('appconfig') + ->where($qb->expr()->eq('appid', $qb->createNamedParameter(Application::APP_ID))) + ->andWhere($qb->expr()->eq('configkey', $qb->createNamedParameter(self::CONFIG_KEY))) + ->executeQuery(); + $value = $result->fetchOne(); + $result->closeCursor(); + return $value === false ? 0 : (int)$value; + } +} diff --git a/lib/Service/LangRopeService.php b/lib/Service/LangRopeService.php index 96150bcc..84e8cf93 100644 --- a/lib/Service/LangRopeService.php +++ b/lib/Service/LangRopeService.php @@ -189,14 +189,13 @@ private function requestToExApp( */ public function getIndexedDocumentsCounts(): array { $response = $this->requestToExApp('/countIndexedDocuments', 'POST', extraOptions: ['timeout' => 10]); - if ($response === []) { - // No documents indexed yet - return []; - } - if (!isset($response[ProviderConfigService::getDefaultProviderKey()])) { - throw new \RuntimeException("Malformed indexed documents count response from Context Chat Backend (ExApp): '" - . ProviderConfigService::getDefaultProviderKey() . "' key is missing, response: " . strval(json_encode($response))); + foreach ($response as $count) { + if (!is_int($count)) { + throw new \RuntimeException('Malformed indexed documents count response from Context Chat Backend (ExApp): ' + . strval(json_encode($response))); + } } + // only the providers with indexed documents are listed return $response; } @@ -224,6 +223,16 @@ public function deleteProvider(string $providerKey): void { $this->requestToExApp('/deleteProvider', 'POST', $params); } + /** + * Delete all the indexed documents from the backend's vector DB + * + * @return void + * @throws \RuntimeException + */ + public function deleteAll(): void { + $this->requestToExApp('/deleteAll', 'POST'); + } + /** * @param string $userId * @return void diff --git a/lib/Service/ProviderConfigService.php b/lib/Service/ProviderConfigService.php index 344c98e0..f513596d 100644 --- a/lib/Service/ProviderConfigService.php +++ b/lib/Service/ProviderConfigService.php @@ -112,6 +112,53 @@ public function updateProvider( $this->appConfig->setAppValueString('providers', json_encode($providers), lazy: true); } + /** + * Mark all the content providers, except the files provider, as not initiated + * so their initial import runs again + */ + public function resetProvidersInitiation(): void { + $providers = $this->getProviders(); + foreach ($providers as $key => $provider) { + if ($key !== self::getDefaultProviderKey()) { + $providers[$key]['isInitiated'] = false; + } + } + $this->appConfig->setAppValueString('providers', json_encode($providers), lazy: true); + } + + /** + * Get the parts of a provider key + * + * @param string $providerKey The provider key in the format "appId__providerId" + * @return array{0: string, 1: string} The appId and providerId + * @throws \RuntimeException when the provider key format is invalid + */ + public function getProviderParts(string $providerKey): array { + $identifierValues = explode('__', $providerKey, 2); + if (count($identifierValues) !== 2) { + throw new \RuntimeException('Invalid provider key format: ' . $providerKey); + } + return $identifierValues; + } + + /** + * Mark a specific provider as initiated or not initiated + * + * @param string $providerKey The provider key in the format "appId__providerId" + * @param bool $isInitiated + * @return void + * @throws \RuntimeException when the provider key format is invalid + */ + public function setProviderInitiated(string $providerKey, bool $isInitiated): void { + $provider = $this->getProvider($providerKey); + if ($provider === null) { + return; + } + [$appId, $providerId] = $this->getProviderParts($providerKey); + /** @psalm-suppress ArgumentTypeCoercion */ + $this->updateProvider($appId, $providerId, $provider['classString'], $isInitiated); + } + /** * @param string $appId * @param ?string $providerId diff --git a/lib/Service/ReindexService.php b/lib/Service/ReindexService.php new file mode 100644 index 00000000..409c9369 --- /dev/null +++ b/lib/Service/ReindexService.php @@ -0,0 +1,260 @@ + + */ + public function getRegisteredProviderKeys(): array { + return $this->uniqueSorted(array_merge( + [ProviderConfigService::getDefaultProviderKey()], + array_keys($this->providerConfig->getProviders()), + )); + } + + /** + * Keys of the providers that can be wiped: the registered ones and the ones that still have + * documents in the vector DB, e.g. of an app that has been removed since. + * + * @return list + * @throws \RuntimeException when the backend could not be reached + */ + public function getWipeableProviderKeys(): array { + return $this->uniqueSorted(array_merge( + $this->getRegisteredProviderKeys(), + array_keys($this->langRopeService->getIndexedDocumentsCounts()), + )); + } + + /** + * @param array $keys + * @return list + */ + private function uniqueSorted(array $keys): array { + $keys = array_values(array_unique(array_map(strval(...), $keys))); + sort($keys); + return $keys; + } + + /** + * Schedule the full filesystem crawl and the initial import of the content providers. + * Jobs that are already scheduled are not scheduled again. + * + * @param list|null $providerKeys null means everything. Without a scope only the content + * providers that have not been imported yet are imported (which is all of them after a full wipe), + * content providers explicitly specified are set as not initiated and scheduled for import. + * @return bool whether anything new was scheduled + */ + public function scheduleReindex(?array $providerKeys = null): bool { + $scheduled = false; + $filesKey = ProviderConfigService::getDefaultProviderKey(); + + foreach ($this->providerConfig->getProviders() as $key => $provider) { + if ($key === $filesKey || $provider['classString'] === '') { + continue; + } + if ($providerKeys === null) { + // a full wipe turns all providers as not initiated so they will be scheduled for import and not skipped + if ($provider['isInitiated']) { + continue; + } + } elseif (in_array($key, $providerKeys, true)) { + $this->providerConfig->setProviderInitiated($key, false); + } else { + continue; + } + if ($this->jobList->has(InitialContentImportJob::class, $provider['classString'])) { + continue; + } + $this->jobList->add(InitialContentImportJob::class, $provider['classString']); + $scheduled = true; + } + + if (($providerKeys === null || in_array($filesKey, $providerKeys, true)) + && !$this->jobList->has(SchedulerJob::class, null) + ) { + $this->jobList->add(SchedulerJob::class); + $scheduled = true; + } + return $scheduled; + } + + /** + * Delete everything that was queued or indexed, here and in context_chat_backend's vector DB. + * No queue items are handed to the backend until the wipe is over, and the documents it is still + * indexing are waited for, so that its workers, possibly running elsewhere on Kubernetes, cannot + * write to the vector DB while it is wiped. + * It is safe to run this again when the backend request fails. + * + * @param list|null $providerKeys null means everything including the actions queue, otherwise only the given providers + * @param bool $ignoreHandedOut do not wait for the documents the backend is indexing, for when some documents are stuck + * in the queue for 24h due to error or if the backend stopped while indexing them/having a lock + * @param ?callable(string): void $onStep called with a short description before each step, for progress output + * @throws \OCP\DB\Exception + * @throws \RuntimeException when the vector DB could not be wiped + */ + public function wipe(?array $providerKeys = null, bool $ignoreHandedOut = false, ?callable $onStep = null): void { + $onStep = $onStep ?? static function (string $_): void { + }; + + $onStep('Pausing the hand-out of queued documents and actions to context_chat_backend'); + $pausedUntil = $this->fetchGuard->pause(); + try { + if ($ignoreHandedOut) { + $onStep('Not waiting for the documents context_chat_backend may still be indexing (--ignore-handed-out)'); + } else { + $onStep('Checking for documents context_chat_backend is still indexing'); + $this->fetchGuard->waitForHandedOutDocuments(static function (int $locked) use ($onStep): void { + $onStep('Waiting for context_chat_backend to finish indexing ' . $locked . ' documents'); + }); + } + + if ($providerKeys === null) { + $this->wipeAll($onStep); + return; + } + + foreach (array_unique($providerKeys) as $providerKey) { + $this->logger->info('Wiping the Context Chat data of provider ' . $providerKey); + if ($providerKey === ProviderConfigService::getDefaultProviderKey()) { + $onStep('Removing the crawl jobs, clearing the files indexing queue and the file system events, resetting the indexing statistics'); + $this->clearFilesState(); + } else { + $onStep('Removing the import job and clearing the indexing queue of ' . $providerKey); + $this->clearContentProviderState($providerKey); + } + $onStep('Deleting the documents of ' . $providerKey . ' from the vector DB'); + $this->langRopeService->deleteProvider($providerKey); + } + } finally { + $onStep('Resuming the hand-out of queued documents and actions'); + $this->fetchGuard->resume($pausedUntil); + } + } + + /** + * @param callable(string): void $onStep + */ + private function wipeAll(callable $onStep): void { + $this->logger->info('Wiping all the Context Chat data'); + + $onStep('Removing the scheduled crawl and content import jobs'); + // stop the crawl and the content provider imports from queueing more items + foreach ([SchedulerJob::class, StorageCrawlJob::class, InitialContentImportJob::class] as $jobClass) { + $this->jobList->remove($jobClass); + } + $onStep('Clearing the indexing, actions and file system events queues'); + $this->clearTables(self::QUEUE_TABLES); + $onStep('Resetting the indexing statistics and the content providers\' import state'); + $this->resetStats(); + $this->providerConfig->resetProvidersInitiation(); + + $onStep('Deleting all the documents from the vector DB'); + $this->langRopeService->deleteAll(); + $this->logger->info('Wiped all the Context Chat data'); + } + + /** + * The actions queue is left alone, it is shared by all the providers and the users. + * Leftover actions for the wiped documents are skipped by the backend. + */ + private function clearFilesState(): void { + foreach ([SchedulerJob::class, StorageCrawlJob::class] as $jobClass) { + $this->jobList->remove($jobClass); + } + $this->clearTables(self::FILES_QUEUE_TABLES); + // the stats only track the files + $this->resetStats(); + } + + private function clearContentProviderState(string $providerKey): void { + $provider = $this->providerConfig->getProvider($providerKey); + if ($provider !== null && $provider['classString'] !== '') { + $this->jobList->remove(InitialContentImportJob::class, $provider['classString']); + } + + [$appId, $providerId] = $this->providerConfig->getProviderParts($providerKey); + $this->contentQueue->removeProvider($appId, $providerId); + + if ($provider !== null) { + $this->providerConfig->setProviderInitiated($providerKey, false); + } + } + + /** + * @param list $tables + * @throws \RuntimeException + */ + private function clearTables(array $tables): void { + $this->db->beginTransaction(); + try { + foreach ($tables as $table) { + $this->db->getQueryBuilder()->delete($table)->executeStatement(); + } + $this->db->commit(); + } catch (\Throwable $e) { + $this->db->rollBack(); + throw new \RuntimeException('Failed to clear tables ' . implode(', ', $tables) . ': ' . $e->getMessage(), 0, $e); + } + } + + private function resetStats(): void { + foreach (self::STATS_CONFIG_KEYS as $key) { + $this->appConfig->deleteAppValue($key); + } + // the time taken for the next complete index is measured from here + $this->appConfig->setAppValueInt('installed_time', $this->timeFactory->getTime(), lazy: true); + } +} diff --git a/lib/Service/StatisticsService.php b/lib/Service/StatisticsService.php index 7d214ef0..c3086b52 100644 --- a/lib/Service/StatisticsService.php +++ b/lib/Service/StatisticsService.php @@ -69,9 +69,7 @@ public function getStatistics(): array { $this->logger->error($e->getMessage(), ['exception' => $e]); $stats['backend_available'] = false; } - if (empty($stats['vectordb_document_counts'] ?? [])) { - $stats['vectordb_document_counts'] = [ ProviderConfigService::getDefaultProviderKey() => 0 ]; - } + $stats['vectordb_document_counts'] = ($stats['vectordb_document_counts'] ?? []) + [ ProviderConfigService::getDefaultProviderKey() => 0 ]; try { $queued_files_count = $this->queueService->count(); diff --git a/tests/integration/ReindexServiceTest.php b/tests/integration/ReindexServiceTest.php new file mode 100644 index 00000000..cda6319d --- /dev/null +++ b/tests/integration/ReindexServiceTest.php @@ -0,0 +1,500 @@ +> */ + private array $savedConfig; + + protected function setUp(): void { + parent::setUp(); + $this->db = Server::get(IDBConnection::class); + $this->jobList = Server::get(IJobList::class); + // the app scoped config is only available from the app's container + $container = (new Application())->getContainer(); + $this->appConfig = $container->get(IAppConfig::class); + $this->providerConfig = $container->get(ProviderConfigService::class); + $this->langRope = $this->createMock(LangRopeService::class); + $this->fetchGuard = $this->newGuard(Server::get(ITimeFactory::class)); + $time = $this->createMock(ITimeFactory::class); + $time->method('getTime')->willReturn(self::NOW); + + $this->service = new ReindexService( + $this->db, + $this->jobList, + $this->appConfig, + $this->providerConfig, + $this->langRope, + Server::get(QueueContentItemMapper::class), + $this->fetchGuard, + $time, + Server::get(Logger::class), + ); + + $this->savedConfig = $this->readConfig(); + $this->clearState(); + $this->appConfig->setAppValueString('providers', '', lazy: true); + } + + protected function tearDown(): void { + $this->clearState(); + // the wipe resets the stats and the providers, put back what the instance had before + $qb = $this->db->getQueryBuilder(); + $qb->delete('appconfig') + ->where($qb->expr()->eq('appid', $qb->createNamedParameter(Application::APP_ID))) + ->executeStatement(); + foreach ($this->savedConfig as $row) { + $qb = $this->db->getQueryBuilder(); + $qb->insert('appconfig'); + foreach ($row as $column => $value) { + $qb->setValue($column, $qb->createNamedParameter($value)); + } + $qb->executeStatement(); + } + Server::get(\OCP\IAppConfig::class)->clearCache(true); + parent::tearDown(); + } + + /** @return list> */ + private function readConfig(): array { + $qb = $this->db->getQueryBuilder(); + $result = $qb->select('*')->from('appconfig') + ->where($qb->expr()->eq('appid', $qb->createNamedParameter(Application::APP_ID))) + ->executeQuery(); + $rows = $result->fetchAll(); + $result->closeCursor(); + return $rows; + } + + private function clearState(): void { + foreach (ReindexService::QUEUE_TABLES as $table) { + $this->db->getQueryBuilder()->delete($table)->executeStatement(); + } + foreach ([SchedulerJob::class, StorageCrawlJob::class, InitialContentImportJob::class] as $jobClass) { + $this->jobList->remove($jobClass); + } + } + + private function seedIndexedState(): void { + $file = new QueueFile(); + $file->setFileId(42); + $file->setStorageId(1); + $file->setRootId(1); + $file->setUpdate(false); + Server::get(QueueMapper::class)->insertIntoQueue($file); + + foreach (['someapp', 'otherapp'] as $appId) { + $item = new QueueContentItem(); + $item->setItemId('item1'); + $item->setAppId($appId); + $item->setProviderId('prov'); + $item->setTitle('title'); + $item->setContent('content'); + $item->setDocumentType('text'); + $item->setLastModified(new \DateTime()); + $item->setUsers('admin'); + Server::get(QueueContentItemMapper::class)->insert($item); + } + + $action = new QueueAction(); + $action->setType(ActionType::DELETE_SOURCE_IDS); + $action->setPayload('{"sourceIds":["files__default: 42"]}'); + Server::get(QueueActionMapper::class)->insertIntoQueue($action); + + Server::get(FsEventMapper::class)->insertRow(FsEventType::CREATE->value, 'admin', 42); + + $this->jobList->add(StorageCrawlJob::class, ['storage_id' => 1, 'root_id' => 1, 'overridden_root' => 1, 'last_file_id' => 0]); + $this->jobList->add(InitialContentImportJob::class, self::PROVIDER_CLASS); + $this->jobList->add(InitialContentImportJob::class, self::OTHER_PROVIDER_CLASS); + + $this->appConfig->setAppValueInt('last_indexed_time', 123, lazy: true); + $this->appConfig->setAppValueInt('last_enqueued_db_id', 7, lazy: true); + $this->appConfig->setAppValueInt('eligible_files_count', 99, lazy: true); + $this->appConfig->setAppValueInt('eligible_files_count_time', 456, lazy: true); + + $this->providerConfig->updateProvider('files', 'default', '', true); + $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, true); + $this->providerConfig->updateProvider('otherapp', 'prov', self::OTHER_PROVIDER_CLASS, true); + } + + /** @return list */ + private function contentQueueAppIds(): array { + $qb = $this->db->getQueryBuilder(); + $result = $qb->selectDistinct('app_id')->from('context_chat_content_queue')->executeQuery(); + $appIds = []; + while (($appId = $result->fetchOne()) !== false) { + $appIds[] = $appId; + } + $result->closeCursor(); + return $appIds; + } + + private function rowCount(string $table): int { + $qb = $this->db->getQueryBuilder(); + return (int)$qb->select($qb->func()->count('*'))->from($table)->executeQuery()->fetchOne(); + } + + public function testWipeClearsAllLocalStateAndTheBackend(): void { + $this->seedIndexedState(); + $this->langRope->expects($this->once())->method('deleteAll'); + $this->langRope->expects($this->never())->method('deleteProvider'); + + $this->service->wipe(); + + foreach (ReindexService::QUEUE_TABLES as $table) { + $this->assertSame(0, $this->rowCount($table), $table . ' should be empty'); + } + $this->assertFalse($this->jobList->has(StorageCrawlJob::class, ['storage_id' => 1, 'root_id' => 1, 'overridden_root' => 1, 'last_file_id' => 0])); + $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS)); + $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::OTHER_PROVIDER_CLASS)); + $this->assertSame(0, $this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true)); + $this->assertSame(-1, $this->appConfig->getAppValueInt('last_enqueued_db_id', -1, lazy: true)); + $this->assertSame(0, $this->appConfig->getAppValueInt('eligible_files_count_time', 0, lazy: true)); + $this->assertSame(self::NOW, $this->appConfig->getAppValueInt('installed_time', 0, lazy: true)); + + $providers = $this->providerConfig->getProviders(); + $this->assertFalse($providers['someapp__prov']['isInitiated'], 'content providers must be imported again'); + $this->assertTrue($providers[ProviderConfigService::getDefaultProviderKey()]['isInitiated'], 'files are re-crawled by SchedulerJob instead'); + } + + public function testWipeClearsLocalStateBeforeTheBackendFails(): void { + $this->seedIndexedState(); + $this->langRope->method('deleteAll')->willThrowException(new \RuntimeException('backend down')); + + try { + $this->service->wipe(); + $this->fail('wipe() must report the backend failure'); + } catch (\RuntimeException $e) { + $this->assertSame('backend down', $e->getMessage()); + } + + // nothing may be indexed or pruned against a half-wiped vector DB, and a re-run must work + foreach (ReindexService::QUEUE_TABLES as $table) { + $this->assertSame(0, $this->rowCount($table), $table . ' should be empty'); + } + } + + public function testScheduleReindexAfterWipeQueuesCrawlAndProviderImportsOnce(): void { + $this->seedIndexedState(); + $this->service->wipe(); + + $this->assertTrue($this->service->scheduleReindex()); + $this->assertFalse($this->service->scheduleReindex(), 'an already scheduled crawl is not scheduled again'); + + $this->assertTrue($this->jobList->has(SchedulerJob::class, null)); + $this->assertTrue($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS)); + $importJobs = iterator_to_array($this->jobList->getJobsIterator(InitialContentImportJob::class, null, 0), false); + $this->assertCount(2, $importJobs); + } + + public function testScheduleReindexSkipsInitiatedProviders(): void { + $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, true); + + $this->service->scheduleReindex(); + + $this->assertTrue($this->jobList->has(SchedulerJob::class, null)); + $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS)); + } + + public function testWipeFilesOnlyLeavesContentProvidersAndActions(): void { + $this->seedIndexedState(); + $this->langRope->expects($this->never())->method('deleteAll'); + $this->langRope->expects($this->once())->method('deleteProvider')->with('files__default'); + + $this->service->wipe(['files__default']); + + $this->assertSame(0, $this->rowCount('context_chat_queue')); + $this->assertSame(0, $this->rowCount('context_chat_fs_events')); + $this->assertSame(2, $this->rowCount('context_chat_content_queue'), 'content providers must be kept'); + $this->assertSame(1, $this->rowCount('context_chat_action_queue'), 'actions of other providers and users must be kept'); + $this->assertFalse($this->jobList->has(StorageCrawlJob::class, ['storage_id' => 1, 'root_id' => 1, 'overridden_root' => 1, 'last_file_id' => 0])); + $this->assertTrue($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS)); + $this->assertSame(0, $this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true)); + $this->assertTrue($this->providerConfig->getProviders()['someapp__prov']['isInitiated']); + } + + public function testWipeOneContentProviderLeavesTheRest(): void { + $this->seedIndexedState(); + $this->langRope->expects($this->never())->method('deleteAll'); + $this->langRope->expects($this->once())->method('deleteProvider')->with('someapp__prov'); + + $this->service->wipe(['someapp__prov']); + + $this->assertSame(['otherapp'], $this->contentQueueAppIds()); + $this->assertSame(1, $this->rowCount('context_chat_queue'), 'files must be kept'); + $this->assertSame(1, $this->rowCount('context_chat_fs_events'), 'files must be kept'); + $this->assertSame(1, $this->rowCount('context_chat_action_queue')); + $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS)); + $this->assertTrue($this->jobList->has(InitialContentImportJob::class, self::OTHER_PROVIDER_CLASS)); + $this->assertSame(123, $this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true), 'file stats must be kept'); + + $providers = $this->providerConfig->getProviders(); + $this->assertFalse($providers['someapp__prov']['isInitiated']); + $this->assertTrue($providers['otherapp__prov']['isInitiated']); + } + + public function testWipeOfAnUnregisteredProviderOnlyDeletesInTheBackend(): void { + $this->seedIndexedState(); + $this->langRope->expects($this->once())->method('deleteProvider')->with('removedapp__prov'); + + $this->service->wipe(['removedapp__prov']); + + $this->assertSame(2, $this->rowCount('context_chat_content_queue')); + $this->assertArrayNotHasKey('removedapp__prov', $this->providerConfig->getProviders()); + } + + public function testScheduleReindexOfANamedProviderReimportsItEvenIfImported(): void { + $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, true); + $this->providerConfig->updateProvider('otherapp', 'prov', self::OTHER_PROVIDER_CLASS, true); + + $this->assertTrue($this->service->scheduleReindex(['someapp__prov'])); + + $this->assertTrue($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS)); + $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::OTHER_PROVIDER_CLASS)); + $this->assertFalse($this->jobList->has(SchedulerJob::class, null), 'files are not in the scope'); + // InitialContentImportJob skips initiated providers + $this->assertFalse($this->providerConfig->getProviders()['someapp__prov']['isInitiated']); + } + + public function testScheduleReindexOfFilesOnly(): void { + $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, false); + + $this->assertTrue($this->service->scheduleReindex(['files__default'])); + $this->assertFalse($this->service->scheduleReindex(['files__default'])); + + $this->assertTrue($this->jobList->has(SchedulerJob::class, null)); + $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS)); + } + + public function testWipeableProviderKeysIncludeProvidersOnlyKnownToTheBackend(): void { + $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, true); + $this->langRope->method('getIndexedDocumentsCounts')->willReturn(['files__default' => 3, 'removedapp__prov' => 1]); + + $this->assertSame(['files__default', 'someapp__prov'], $this->service->getRegisteredProviderKeys()); + $this->assertSame(['files__default', 'removedapp__prov', 'someapp__prov'], $this->service->getWipeableProviderKeys()); + } + + private function newGuard(ITimeFactory $time): FetchGuard { + return new FetchGuard($this->db, $this->appConfig, $time, Server::get(QueueMapper::class), Server::get(QueueContentItemMapper::class)); + } + + /** @return ITimeFactory&MockObject a clock that advances one second per sleep(1) of the guard */ + private function clock(int &$now): ITimeFactory { + $time = $this->createMock(ITimeFactory::class); + $time->method('getTime')->willReturnCallback(function () use (&$now) { + return $now; + }); + return $time; + } + + public function testFetchingIsPausedOnlyWhileTheBackendIsWiped(): void { + $this->seedIndexedState(); + $seen = null; + $this->langRope->method('deleteAll')->willReturnCallback(function () use (&$seen) { + $seen = $this->fetchGuard->isPaused(); + }); + + $this->service->wipe(); + + $this->assertTrue($seen, 'nothing may be handed out while the vector DB is wiped'); + $this->assertFalse($this->fetchGuard->isPaused()); + } + + public function testFetchingResumesWhenTheBackendFails(): void { + $this->seedIndexedState(); + $this->langRope->method('deleteAll')->willThrowException(new \RuntimeException('backend down')); + + try { + $this->service->wipe(); + } catch (\RuntimeException) { + } + + $this->assertFalse($this->fetchGuard->isPaused()); + } + + public function testWipeIsRefusedWhileDocumentsAreStillBeingIndexed(): void { + $this->seedIndexedState(); + $queueMapper = Server::get(QueueMapper::class); + $queueMapper->lock($queueMapper->getMaxId()); + $this->appConfig->setAppValueInt('wipe_drain_timeout', 0, lazy: true); + $this->langRope->expects($this->never())->method('deleteAll'); + + try { + $this->service->wipe(); + $this->fail('the wipe must not run while a document is being indexed'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('1 documents are still being indexed', $e->getMessage()); + } finally { + $this->appConfig->deleteAppValue('wipe_drain_timeout'); + } + + $this->assertSame(1, $queueMapper->countLocked(), 'nothing may be cleared when the wipe is refused'); + $this->assertFalse($this->fetchGuard->isPaused()); + } + + public function testResumeDoesNotLiftANewerPause(): void { + $now = self::NOW; + $guard = $this->newGuard($this->clock($now)); + + $first = $guard->pause(); + $now += 10; + $second = $guard->pause(); + $guard->resume($first); + + $this->assertTrue($guard->isPaused(), 'the second wipe is still running'); + $guard->resume($second); + $this->assertFalse($guard->isPaused()); + } + + public function testPauseExpiresOnItsOwn(): void { + $now = self::NOW; + $guard = $this->newGuard($this->clock($now)); + + $until = $guard->pause(); + $this->assertTrue($guard->isPaused()); + $now = $until; + + $this->assertFalse($guard->isPaused(), 'a wipe that died must not stop the indexing forever'); + $guard->resume($until); + } + + public function testWaitEndsOnceTheHandedOutBatchIsReportedBack(): void { + $queueMapper = Server::get(QueueMapper::class); + $file = new QueueFile(); + $file->setFileId(43); + $file->setStorageId(1); + $file->setRootId(1); + $file->setUpdate(false); + $id = $queueMapper->insertIntoQueue($file)->getId(); + $queueMapper->lock($id); + + $checks = 0; + $time = $this->createMock(ITimeFactory::class); + $time->method('getTime')->willReturnCallback(function () use (&$checks, $queueMapper, $id) { + // the backend reports the batch back while the wipe is waiting + if (++$checks === 3) { + $queueMapper->removeFromQueue([$id]); + } + return self::NOW; + }); + + $reported = []; + $this->newGuard($time)->waitForHandedOutDocuments(function (int $locked) use (&$reported) { + $reported[] = $locked; + }); + + $this->assertSame(0, $queueMapper->countLocked()); + $this->assertSame([1], $reported, 'the count is reported once, not on every poll'); + } + + /** + * The steps are reported in order, each one before its work: what the admin reads is what runs next. + */ + public function testWipeReportsEachStepBeforeDoingIt(): void { + $this->seedIndexedState(); + $events = []; + $this->langRope->method('deleteAll')->willReturnCallback(function () use (&$events) { + $events[] = 'backend:deleteAll'; + }); + + $this->service->wipe(null, false, function (string $message) use (&$events) { + // mark the moment the queues are emptied, to check the order against the work + $events[] = $message . ($this->rowCount('context_chat_queue') === 0 ? ' [queues empty]' : ''); + }); + + $this->assertSame([ + 'Pausing the hand-out of queued documents and actions to context_chat_backend', + 'Checking for documents context_chat_backend is still indexing', + 'Removing the scheduled crawl and content import jobs', + 'Clearing the indexing, actions and file system events queues', + 'Resetting the indexing statistics and the content providers\' import state [queues empty]', + 'Deleting all the documents from the vector DB [queues empty]', + 'backend:deleteAll', + 'Resuming the hand-out of queued documents and actions [queues empty]', + ], $events); + } + + public function testWipeOfOneProviderReportsItsSteps(): void { + $this->seedIndexedState(); + $events = []; + $this->langRope->method('deleteProvider')->willReturnCallback(function (string $key) use (&$events) { + $events[] = 'backend:deleteProvider ' . $key; + }); + + $this->service->wipe(['someapp__prov'], true, function (string $message) use (&$events) { + $events[] = $message; + }); + + $this->assertSame([ + 'Pausing the hand-out of queued documents and actions to context_chat_backend', + 'Not waiting for the documents context_chat_backend may still be indexing (--ignore-handed-out)', + 'Removing the import job and clearing the indexing queue of someapp__prov', + 'Deleting the documents of someapp__prov from the vector DB', + 'backend:deleteProvider someapp__prov', + 'Resuming the hand-out of queued documents and actions', + ], $events); + } + + public function testResumeIsReportedWhenTheBackendFails(): void { + $this->seedIndexedState(); + $this->langRope->method('deleteAll')->willThrowException(new \RuntimeException('backend down')); + $events = []; + + try { + $this->service->wipe(null, false, function (string $message) use (&$events) { + $events[] = $message; + }); + $this->fail('the backend failure must be reported'); + } catch (\RuntimeException) { + } + + $this->assertSame('Deleting all the documents from the vector DB', $events[count($events) - 2]); + $this->assertSame('Resuming the hand-out of queued documents and actions', end($events)); + } +}