diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index 87e56cd3cb..c2016d7f70 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -102,14 +102,15 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO services.AddSingleton(s => { var handlers = new WorkItemHandlers(); + handlers.Register(s.GetRequiredService); + handlers.Register(s.GetRequiredService); + handlers.Register(s.GetRequiredService); + handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); - handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); + handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); - handlers.Register(s.GetRequiredService); - handlers.Register(s.GetRequiredService); - handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); return handlers; }); diff --git a/src/Exceptionless.Core/Jobs/EventPostsJob.cs b/src/Exceptionless.Core/Jobs/EventPostsJob.cs index 19027d53cb..605ee3ee6c 100644 --- a/src/Exceptionless.Core/Jobs/EventPostsJob.cs +++ b/src/Exceptionless.Core/Jobs/EventPostsJob.cs @@ -5,7 +5,7 @@ using Exceptionless.Core.Plugins.EventParser; using Exceptionless.Core.Queues.Models; using Exceptionless.Core.Repositories; -using Exceptionless.Core.Repositories.Base; +using Foundatio.Repositories.Exceptions; using Exceptionless.Core.Services; using Exceptionless.Core.Validation; using FluentValidation; diff --git a/src/Exceptionless.Core/Jobs/EventUserDescriptionsJob.cs b/src/Exceptionless.Core/Jobs/EventUserDescriptionsJob.cs index c4c783dd7c..8b4514b6b3 100644 --- a/src/Exceptionless.Core/Jobs/EventUserDescriptionsJob.cs +++ b/src/Exceptionless.Core/Jobs/EventUserDescriptionsJob.cs @@ -1,7 +1,7 @@ using Exceptionless.Core.Models.Data; using Exceptionless.Core.Queues.Models; using Exceptionless.Core.Repositories; -using Exceptionless.Core.Repositories.Base; +using Foundatio.Repositories.Exceptions; using Foundatio.Jobs; using Foundatio.Queues; using Foundatio.Repositories.Extensions; diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/FixStackStatsWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/FixStackStatsWorkItemHandler.cs new file mode 100644 index 0000000000..a40ad6c746 --- /dev/null +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/FixStackStatsWorkItemHandler.cs @@ -0,0 +1,155 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Repositories; +using Exceptionless.DateTimeExtensions; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Repositories; +using Foundatio.Repositories.Models; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Jobs.WorkItemHandlers; + +public class FixStackStatsWorkItemHandler : WorkItemHandlerBase +{ + private readonly IStackRepository _stackRepository; + private readonly IEventRepository _eventRepository; + private readonly ILockProvider _lockProvider; + private readonly TimeProvider _timeProvider; + + public FixStackStatsWorkItemHandler(IStackRepository stackRepository, IEventRepository eventRepository, ILockProvider lockProvider, TimeProvider timeProvider, ILoggerFactory loggerFactory) + : base(loggerFactory) + { + _stackRepository = stackRepository; + _eventRepository = eventRepository; + _lockProvider = lockProvider; + _timeProvider = timeProvider; + } + + public override Task GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default) + { + return _lockProvider.AcquireAsync(nameof(FixStackStatsWorkItemHandler), TimeSpan.FromHours(1), cancellationToken); + } + + public override async Task HandleItemAsync(WorkItemContext context) + { + var wi = context.GetData(); + var utcEnd = wi.UtcEnd ?? _timeProvider.GetUtcNow().UtcDateTime; + + Log.LogInformation("Starting stack stats repair for {UtcStart:O} to {UtcEnd:O}. OrganizationId={Organization}", wi.UtcStart, utcEnd, wi.OrganizationId); + await context.ReportProgressAsync(0, $"Starting stack stats repair for window {wi.UtcStart:O} – {utcEnd:O}"); + + var organizationIds = await GetOrganizationIdsAsync(wi, utcEnd); + Log.LogInformation("Found {OrganizationCount} organizations to process", organizationIds.Count); + + int repaired = 0; + int skipped = 0; + + for (int index = 0; index < organizationIds.Count; index++) + { + if (context.CancellationToken.IsCancellationRequested) + break; + + var (organizationRepaired, organizationSkipped) = await ProcessOrganizationAsync(context, organizationIds[index], wi.UtcStart, utcEnd); + repaired += organizationRepaired; + skipped += organizationSkipped; + + int percentage = (int)Math.Min(99, (index + 1) * 100.0 / organizationIds.Count); + await context.ReportProgressAsync(percentage, $"Organization {index + 1}/{organizationIds.Count} ({percentage}%): repaired {repaired}, skipped {skipped}"); + } + + Log.LogInformation("Stack stats repair complete: Repaired={Repaired} Skipped={Skipped}", repaired, skipped); + await context.ReportProgressAsync(100, $"Done. Repaired {repaired} stacks, skipped={skipped}."); + } + + private async Task> GetOrganizationIdsAsync(FixStackStatsWorkItem wi, DateTime utcEnd) + { + if (wi.OrganizationId is not null) + return [wi.OrganizationId]; + + var countResult = await _eventRepository.CountAsync(q => q + .DateRange(wi.UtcStart, utcEnd, (PersistentEvent e) => e.Date) + .Index(wi.UtcStart, utcEnd) + .AggregationsExpression("terms:(organization_id~65536)")); + + return countResult.Aggregations.Terms("terms_organization_id")?.Buckets + .Select(b => b.Key) + .ToList() ?? []; + } + + private async Task<(int Repaired, int Skipped)> ProcessOrganizationAsync(WorkItemContext context, string organizationId, DateTime utcStart, DateTime utcEnd) + { + using var _ = Log.BeginScope(new ExceptionlessState().Organization(organizationId)); + await context.RenewLockAsync(); + + var countResult = await _eventRepository.CountAsync(q => q + .Organization(organizationId) + .DateRange(utcStart, utcEnd, (PersistentEvent e) => e.Date) + .Index(utcStart, utcEnd) + .AggregationsExpression("terms:(stack_id~65536 min:date max:date)")); + + var stackBuckets = countResult.Aggregations.Terms("terms_stack_id")?.Buckets ?? []; + if (stackBuckets.Count is 0) + return (0, 0); + + var statsByStackId = new Dictionary(stackBuckets.Count); + foreach (var bucket in stackBuckets) + { + var firstOccurrence = bucket.Aggregations.Min("min_date")?.Value; + var lastOccurrence = bucket.Aggregations.Max("max_date")?.Value; + if (firstOccurrence is null || lastOccurrence is null || bucket.Total is null) + continue; + + statsByStackId[bucket.Key] = new StackEventStats(firstOccurrence.Value, lastOccurrence.Value, bucket.Total.Value); + } + + int repaired = 0; + int skipped = 0; + + foreach (string[] batch in statsByStackId.Keys.Chunk(100)) + { + if (context.CancellationToken.IsCancellationRequested) + break; + + await context.RenewLockAsync(); + + var stacks = await _stackRepository.GetByIdsAsync(batch); + foreach (var stack in stacks) + { + if (!statsByStackId.TryGetValue(stack.Id, out var stats)) + { + skipped++; + continue; + } + + bool shouldUpdateFirst = stack.FirstOccurrence.IsAfter(stats.FirstOccurrence); + bool shouldUpdateLast = stack.LastOccurrence.IsBefore(stats.LastOccurrence); + bool shouldUpdateTotal = stats.TotalOccurrences > stack.TotalOccurrences; + if (!shouldUpdateFirst && !shouldUpdateLast && !shouldUpdateTotal) + { + skipped++; + continue; + } + + var newFirst = shouldUpdateFirst ? stats.FirstOccurrence : stack.FirstOccurrence; + var newLast = shouldUpdateLast ? stats.LastOccurrence : stack.LastOccurrence; + long newTotal = shouldUpdateTotal ? stats.TotalOccurrences : stack.TotalOccurrences; + + Log.LogInformation( + "Repairing stack {StackId}: first={OldFirst:O}->{NewFirst:O} last={OldLast:O}->{NewLast:O} total={OldTotal}->{NewTotal}", + stack.Id, + stack.FirstOccurrence, newFirst, + stack.LastOccurrence, newLast, + stack.TotalOccurrences, newTotal); + + await _stackRepository.SetEventCounterAsync(stack.Id, newFirst, newLast, newTotal, sendNotifications: false); + repaired++; + } + } + + Log.LogDebug("Processed organization: Repaired={Repaired} Skipped={Skipped}", repaired, skipped); + return (repaired, skipped); + } +} + +internal record StackEventStats(DateTime FirstOccurrence, DateTime LastOccurrence, long TotalOccurrences); diff --git a/src/Exceptionless.Core/Models/WorkItems/FixStackStatsWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/FixStackStatsWorkItem.cs new file mode 100644 index 0000000000..0585d33536 --- /dev/null +++ b/src/Exceptionless.Core/Models/WorkItems/FixStackStatsWorkItem.cs @@ -0,0 +1,14 @@ +namespace Exceptionless.Core.Models.WorkItems; + +public record FixStackStatsWorkItem +{ + public DateTime UtcStart { get; init; } + + public DateTime? UtcEnd { get; init; } + + /// + /// When set, only stacks belonging to this organization are repaired. + /// When null, all organizations with events in the time window are processed. + /// + public string? OrganizationId { get; init; } +} diff --git a/src/Exceptionless.Core/Repositories/Exceptions/DocumentNotFoundException.cs b/src/Exceptionless.Core/Repositories/Exceptions/DocumentNotFoundException.cs deleted file mode 100644 index ad669b9621..0000000000 --- a/src/Exceptionless.Core/Repositories/Exceptions/DocumentNotFoundException.cs +++ /dev/null @@ -1,22 +0,0 @@ -namespace Exceptionless.Core.Repositories.Base; - -public class DocumentNotFoundException : ApplicationException -{ - public DocumentNotFoundException(string id, string? message = null) : base(message) - { - Id = id; - } - - public string Id { get; init; } - - public override string ToString() - { - if (!String.IsNullOrEmpty(Message)) - return Message; - - if (!String.IsNullOrEmpty(Id)) - return $"Document \"{Id}\" could not be found"; - - return base.ToString(); - } -} diff --git a/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs b/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs index 9ec7693745..13199d28c6 100644 --- a/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs +++ b/src/Exceptionless.Core/Repositories/Interfaces/IStackRepository.cs @@ -11,6 +11,7 @@ public interface IStackRepository : IRepositoryOwnedByOrganizationAndProject> GetExpiredSnoozedStatuses(DateTime utcNow, CommandOptionsDescriptor? options = null); Task MarkAsRegressedAsync(string stackId); Task IncrementEventCounterAsync(string organizationId, string projectId, string stackId, DateTime minOccurrenceDateUtc, DateTime maxOccurrenceDateUtc, int count, bool sendNotifications = true); + Task SetEventCounterAsync(string stackId, DateTime firstOccurrenceUtc, DateTime lastOccurrenceUtc, long totalOccurrences, bool sendNotifications = true); Task> GetStacksForCleanupAsync(string organizationId, DateTime cutoff); Task> GetSoftDeleted(); Task SoftDeleteByProjectIdAsync(string organizationId, string projectId); diff --git a/src/Exceptionless.Core/Repositories/StackRepository.cs b/src/Exceptionless.Core/Repositories/StackRepository.cs index 3b79e84296..27bb6bc6f3 100644 --- a/src/Exceptionless.Core/Repositories/StackRepository.cs +++ b/src/Exceptionless.Core/Repositories/StackRepository.cs @@ -3,9 +3,9 @@ using Exceptionless.Core.Repositories.Configuration; using FluentValidation; using Foundatio.Repositories; +using Foundatio.Repositories.Exceptions; using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; -using Microsoft.Extensions.Logging; using Nest; namespace Exceptionless.Core.Repositories; @@ -63,37 +63,90 @@ Instant parseDate(def dt) { if (ctx._source.total_occurrences == 0 || parseDate(ctx._source.first_occurrence).isAfter(parseDate(params.minOccurrenceDateUtc))) { ctx._source.first_occurrence = params.minOccurrenceDateUtc; } + if (parseDate(ctx._source.last_occurrence).isBefore(parseDate(params.maxOccurrenceDateUtc))) { ctx._source.last_occurrence = params.maxOccurrenceDateUtc; } + if (parseDate(ctx._source.updated_utc).isBefore(parseDate(params.updatedUtc))) { ctx._source.updated_utc = params.updatedUtc; } + ctx._source.total_occurrences += params.count;"; - var request = new UpdateRequest(ElasticIndex.GetIndex(stackId), stackId) + var operation = new ScriptPatch(script.TrimScript()) { - Script = new InlineScript(script.TrimScript()) + Params = new Dictionary(4) { - Params = new Dictionary(3) { - { "minOccurrenceDateUtc", minOccurrenceDateUtc }, - { "maxOccurrenceDateUtc", maxOccurrenceDateUtc }, - { "count", count }, - { "updatedUtc", _timeProvider.GetUtcNow().UtcDateTime } - } + { "minOccurrenceDateUtc", minOccurrenceDateUtc }, + { "maxOccurrenceDateUtc", maxOccurrenceDateUtc }, + { "count", count }, + { "updatedUtc", _timeProvider.GetUtcNow().UtcDateTime } } }; - var result = await _client.UpdateAsync(request); - if (!result.IsValid) + try + { + await PatchAsync(stackId, operation, o => o.Notifications(false)); + } + catch (DocumentNotFoundException) { - _logger.LogError(result.OriginalException, "Error occurred incrementing total event occurrences on stack {Stack}. Error: {Message}", stackId, result.ServerError?.Error); - return result.ServerError?.Status == 404; + return true; } - await Cache.RemoveAsync(stackId); if (sendNotifications) - await PublishMessageAsync(CreateEntityChanged(ChangeType.Saved, organizationId, projectId, null, stackId), TimeSpan.FromSeconds(1.5)); + await PublishMessageAsync(CreateEntityChanged(ChangeType.Saved, organizationId, projectId, null, stackId)); + + return true; + } + + public async Task SetEventCounterAsync(string stackId, DateTime firstOccurrenceUtc, DateTime lastOccurrenceUtc, long totalOccurrences, bool sendNotifications = true) + { + const string script = @" +Instant parseDate(def dt) { + if (dt != null) { + try { + return Instant.parse(dt); + } catch(DateTimeParseException e) {} + } + return Instant.MIN; +} + +if (ctx._source.total_occurrences == null || ctx._source.total_occurrences < params.totalOccurrences) { + ctx._source.total_occurrences = params.totalOccurrences; +} + +if (parseDate(ctx._source.first_occurrence).isAfter(parseDate(params.firstOccurrenceUtc))) { + ctx._source.first_occurrence = params.firstOccurrenceUtc; +} + +if (parseDate(ctx._source.last_occurrence).isBefore(parseDate(params.lastOccurrenceUtc))) { + ctx._source.last_occurrence = params.lastOccurrenceUtc; +} + +if (parseDate(ctx._source.updated_utc).isBefore(parseDate(params.updatedUtc))) { + ctx._source.updated_utc = params.updatedUtc; +}"; + + var operation = new ScriptPatch(script.TrimScript()) + { + Params = new Dictionary(4) + { + { "firstOccurrenceUtc", firstOccurrenceUtc }, + { "lastOccurrenceUtc", lastOccurrenceUtc }, + { "totalOccurrences", totalOccurrences }, + { "updatedUtc", _timeProvider.GetUtcNow().UtcDateTime } + } + }; + + try + { + await PatchAsync(stackId, operation, o => o.Notifications(sendNotifications)); + } + catch (DocumentNotFoundException) + { + return true; + } return true; } diff --git a/src/Exceptionless.Web/Controllers/AdminController.cs b/src/Exceptionless.Web/Controllers/AdminController.cs index e3c7b2aef5..dfdef0339a 100644 --- a/src/Exceptionless.Web/Controllers/AdminController.cs +++ b/src/Exceptionless.Web/Controllers/AdminController.cs @@ -9,6 +9,7 @@ using Exceptionless.Core.Repositories; using Exceptionless.Core.Repositories.Configuration; using Exceptionless.Core.Utility; +using Exceptionless.DateTimeExtensions; using Exceptionless.Web.Extensions; using Foundatio.Jobs; using Foundatio.Messaging; @@ -155,8 +156,11 @@ public async Task RequeueAsync(string? path = null, bool archive } [HttpGet("maintenance/{name:minlength(1)}")] - public async Task RunJobAsync(string name) + public async Task RunJobAsync(string name, DateTime? utcStart = null, DateTime? utcEnd = null, string? organizationId = null) { + if (!ModelState.IsValid) + return ValidationProblem(ModelState); + switch (name.ToLowerInvariant()) { case "indexes": @@ -184,6 +188,23 @@ public async Task RunJobAsync(string name) case "reset-verify-email-address-token-and-expiration": await _workItemQueue.EnqueueAsync(new UserMaintenanceWorkItem { ResetVerifyEmailAddressToken = true }); break; + case "fix-stack-stats": + var defaultUtcStart = new DateTime(2026, 2, 10, 0, 0, 0, DateTimeKind.Utc); + var effectiveUtcStart = utcStart ?? defaultUtcStart; + + if (utcEnd.HasValue && utcEnd.Value.IsBefore(effectiveUtcStart)) + { + ModelState.AddModelError(nameof(utcEnd), "utcEnd must be greater than or equal to utcStart."); + return ValidationProblem(ModelState); + } + + await _workItemQueue.EnqueueAsync(new FixStackStatsWorkItem + { + UtcStart = effectiveUtcStart, + UtcEnd = utcEnd, + OrganizationId = organizationId + }); + break; default: return NotFound(); } diff --git a/tests/Exceptionless.Tests/Controllers/AdminControllerTests.cs b/tests/Exceptionless.Tests/Controllers/AdminControllerTests.cs new file mode 100644 index 0000000000..b03ae571ce --- /dev/null +++ b/tests/Exceptionless.Tests/Controllers/AdminControllerTests.cs @@ -0,0 +1,175 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Utility; +using Exceptionless.Tests.Extensions; +using Exceptionless.Tests.Utility; +using Foundatio.Jobs; +using Foundatio.Queues; +using Foundatio.Repositories; +using Xunit; + +namespace Exceptionless.Tests.Controllers; + +public class AdminControllerTests : IntegrationTestsBase +{ + private readonly WorkItemJob _workItemJob; + private readonly IQueue _workItemQueue; + private readonly IStackRepository _stackRepository; + private readonly IEventRepository _eventRepository; + private readonly StackData _stackData; + private readonly EventData _eventData; + + public AdminControllerTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) + { + _workItemJob = GetService(); + _workItemQueue = GetService>(); + _stackRepository = GetService(); + _eventRepository = GetService(); + _stackData = GetService(); + _eventData = GetService(); + } + + protected override async Task ResetDataAsync() + { + await base.ResetDataAsync(); + var service = GetService(); + await service.CreateDataAsync(); + } + + [Fact] + public async Task RunJobAsync_WhenFixStackStatsWithExplicitUtcWindow_ShouldRepairStatsEndToEnd() + { + // Arrange + TimeProvider.SetUtcNow(new DateTime(2026, 2, 15, 12, 0, 0, DateTimeKind.Utc)); + var stack = await CreateCorruptedStackWithEventAsync(new DateTimeOffset(2026, 2, 14, 0, 0, 0, TimeSpan.Zero)); + + // Act + await SendRequestAsync(r => r + .AsGlobalAdminUser() + .AppendPaths("admin", "maintenance", "fix-stack-stats") + .QueryString("utcStart", "2026-02-10T00:00:00Z") + .QueryString("utcEnd", "2026-02-23T00:00:00Z") + .StatusCodeShouldBeOk()); + + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + stack = await _stackRepository.GetByIdAsync(stack.Id); + + // Assert + Assert.NotNull(stack); + Assert.Equal(1, stack.TotalOccurrences); + Assert.Equal(new DateTime(2026, 2, 14, 0, 0, 0, DateTimeKind.Utc), stack.FirstOccurrence); + Assert.Equal(new DateTime(2026, 2, 14, 0, 0, 0, DateTimeKind.Utc), stack.LastOccurrence); + } + + [Fact] + public async Task RunJobAsync_WhenFixStackStatsWindowIsOmitted_ShouldUseDefaultStartAndCurrentUtcEnd() + { + // Arrange + TimeProvider.SetUtcNow(new DateTime(2026, 2, 5, 12, 0, 0, DateTimeKind.Utc)); + var beforeWindow = await CreateCorruptedStackWithEventAsync(new DateTimeOffset(2026, 2, 5, 12, 0, 0, TimeSpan.Zero)); + + TimeProvider.SetUtcNow(new DateTime(2026, 2, 15, 12, 0, 0, DateTimeKind.Utc)); + var inWindow = await CreateCorruptedStackWithEventAsync(new DateTimeOffset(2026, 2, 15, 12, 0, 0, TimeSpan.Zero)); + + // Act + await SendRequestAsync(r => r + .AsGlobalAdminUser() + .AppendPaths("admin", "maintenance", "fix-stack-stats") + .StatusCodeShouldBeOk()); + + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + beforeWindow = await _stackRepository.GetByIdAsync(beforeWindow.Id); + inWindow = await _stackRepository.GetByIdAsync(inWindow.Id); + + // Assert + Assert.NotNull(beforeWindow); + Assert.NotNull(inWindow); + Assert.Equal(0, beforeWindow.TotalOccurrences); + Assert.Equal(1, inWindow.TotalOccurrences); + } + + [Fact] + public async Task RunJobAsync_WhenFixStackStatsUsesOffsetUtcTimestamp_ShouldAcceptModelBindingValue() + { + // Arrange + TimeProvider.SetUtcNow(new DateTime(2026, 2, 15, 12, 0, 0, DateTimeKind.Utc)); + var stack = await CreateCorruptedStackWithEventAsync(new DateTimeOffset(2026, 2, 14, 0, 0, 0, TimeSpan.Zero)); + + // Act + await SendRequestAsync(r => r + .AsGlobalAdminUser() + .AppendPaths("admin", "maintenance", "fix-stack-stats") + .QueryString("utcStart", "2026-02-10T00:00:00+00:00") + .QueryString("utcEnd", "2026-02-23T00:00:00+00:00") + .StatusCodeShouldBeOk()); + + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + stack = await _stackRepository.GetByIdAsync(stack.Id); + var stats = await _workItemQueue.GetQueueStatsAsync(); + + // Assert + Assert.NotNull(stack); + Assert.Equal(1, stack.TotalOccurrences); + Assert.Equal(1, stats.Enqueued); + } + + [Fact] + public async Task RunJobAsync_WhenFixStackStatsEndDateIsBeforeStartDate_ShouldReturnUnprocessableEntity() + { + // Arrange + var response = await SendRequestAsAsync(r => r + .AsGlobalAdminUser() + .AppendPaths("admin", "maintenance", "fix-stack-stats") + .QueryString("utcStart", "2026-02-20T00:00:00Z") + .QueryString("utcEnd", "2026-02-10T00:00:00Z") + .StatusCodeShouldBeUnprocessableEntity()); + + // Act + var stats = await _workItemQueue.GetQueueStatsAsync(); + + // Assert + Assert.NotNull(response); + Assert.True(response.Errors.ContainsKey("utc_end")); + Assert.Equal(0, stats.Enqueued); + } + + [Fact] + public async Task RunJobAsync_WhenFixStackStatsStartDateIsInvalid_ShouldReturnBadRequestAndNotQueueWorkItem() + { + // Arrange + await SendRequestAsync(r => r + .AsGlobalAdminUser() + .AppendPaths("admin", "maintenance", "fix-stack-stats") + .QueryString("utcStart", "not-a-dateZ") + .StatusCodeShouldBeBadRequest()); + + // Act + var stats = await _workItemQueue.GetQueueStatsAsync(); + + // Assert + Assert.Equal(0, stats.Enqueued); + } + + private async Task CreateCorruptedStackWithEventAsync(DateTimeOffset occurrenceDate) + { + var utcOccurrenceDate = occurrenceDate.UtcDateTime; + var stack = _stackData.GenerateStack(generateId: true, + organizationId: TestConstants.OrganizationId, + projectId: TestConstants.ProjectId, + totalOccurrences: 0, + utcFirstOccurrence: utcOccurrenceDate.AddDays(1), + utcLastOccurrence: utcOccurrenceDate.AddDays(-1)); + + stack = await _stackRepository.AddAsync(stack, o => o.ImmediateConsistency()); + + await _eventRepository.AddAsync( + [_eventData.GenerateEvent(TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id, occurrenceDate: occurrenceDate)], + o => o.ImmediateConsistency()); + + await RefreshDataAsync(); + return stack; + } +} diff --git a/tests/Exceptionless.Tests/Jobs/FixStackStatsJobTests.cs b/tests/Exceptionless.Tests/Jobs/FixStackStatsJobTests.cs new file mode 100644 index 0000000000..fd58d41944 --- /dev/null +++ b/tests/Exceptionless.Tests/Jobs/FixStackStatsJobTests.cs @@ -0,0 +1,270 @@ +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Repositories; +using Exceptionless.Tests.Utility; +using Foundatio.Jobs; +using Foundatio.Queues; +using Foundatio.Repositories; +using Xunit; + +namespace Exceptionless.Tests.Jobs; + +public class FixStackStatsJobTests : IntegrationTestsBase +{ + private readonly WorkItemJob _workItemJob; + private readonly IQueue _workItemQueue; + private readonly IStackRepository _stackRepository; + private readonly IEventRepository _eventRepository; + private readonly StackData _stackData; + private readonly EventData _eventData; + + private static readonly DateTime DefaultWindowStart = new(2026, 2, 10, 0, 0, 0, DateTimeKind.Utc); + private static readonly DateTime DefaultWindowEnd = new(2026, 2, 23, 0, 0, 0, DateTimeKind.Utc); + private static readonly DateTime InWindowDate = new(2026, 2, 15, 12, 0, 0, DateTimeKind.Utc); + + public FixStackStatsJobTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) + { + _workItemJob = GetService(); + _workItemQueue = GetService>(); + _stackRepository = GetService(); + _eventRepository = GetService(); + _stackData = GetService(); + _eventData = GetService(); + } + + [Fact] + public async Task RunUntilEmptyAsync_WhenStackIsInBugWindowWithCorruptCounters_ShouldRebuildStackStatsFromEvents() + { + // Arrange + // Simulate the corrupted state: stack created in bug window with TotalOccurrences = 0 + TimeProvider.SetUtcNow(InWindowDate); + + var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, + organizationId: TestConstants.OrganizationId, + projectId: TestConstants.ProjectId, + utcFirstOccurrence: InWindowDate, + utcLastOccurrence: InWindowDate, + totalOccurrences: 0) + , o => o.ImmediateConsistency()); + + // Events exist with known occurrence dates — as if they were posted but the Redis + // ValueTuple bug caused stack stat increments to be silently dropped. + var first = new DateTimeOffset(2026, 2, 11, 0, 0, 0, TimeSpan.Zero); + var middle = new DateTimeOffset(2026, 2, 15, 12, 0, 0, TimeSpan.Zero); + var last = new DateTimeOffset(2026, 2, 20, 0, 0, 0, TimeSpan.Zero); + await _eventRepository.AddAsync( + [ + _eventData.GenerateEvent(organizationId: TestConstants.OrganizationId, + projectId: TestConstants.ProjectId, stackId: stack.Id, occurrenceDate: first), + _eventData.GenerateEvent(organizationId: TestConstants.OrganizationId, + projectId: TestConstants.ProjectId, stackId: stack.Id, occurrenceDate: middle), + _eventData.GenerateEvent(organizationId: TestConstants.OrganizationId, + projectId: TestConstants.ProjectId, stackId: stack.Id, occurrenceDate: last), + ], + o => o.ImmediateConsistency()); + + // Act + await _workItemQueue.EnqueueAsync(new FixStackStatsWorkItem + { + UtcStart = DefaultWindowStart, + UtcEnd = DefaultWindowEnd + }); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + stack = await _stackRepository.GetByIdAsync(stack.Id); + + // Assert + Assert.NotNull(stack); + Assert.Equal(3, stack.TotalOccurrences); + Assert.Equal(first.UtcDateTime, stack.FirstOccurrence); + Assert.Equal(last.UtcDateTime, stack.LastOccurrence); + } + + [Fact] + public async Task RunUntilEmptyAsync_WhenAllEventsAreBeforeWindowStart_ShouldSkipRepair() + { + // Arrange + // All events for this stack are before the window start — the handler won't find them + // in the event aggregation, so the stack should not be touched. + TimeProvider.SetUtcNow(new DateTime(2026, 2, 5, 12, 0, 0, DateTimeKind.Utc)); + + var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, + organizationId: TestConstants.OrganizationId, + projectId: TestConstants.ProjectId, + totalOccurrences: 0), o => o.ImmediateConsistency()); + + await _eventRepository.AddAsync( + [ + _eventData.GenerateEvent(organizationId: TestConstants.OrganizationId, + projectId: TestConstants.ProjectId, stackId: stack.Id, + occurrenceDate: new DateTimeOffset(2026, 2, 5, 12, 0, 0, TimeSpan.Zero)) + ], + o => o.ImmediateConsistency()); + + // Act + await _workItemQueue.EnqueueAsync(new FixStackStatsWorkItem + { + UtcStart = DefaultWindowStart, // Feb 10 — after this stack's events (Feb 5) + UtcEnd = DefaultWindowEnd + }); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + stack = await _stackRepository.GetByIdAsync(stack.Id); + + // Assert + Assert.NotNull(stack); + Assert.Equal(0, stack.TotalOccurrences); // Not touched — events outside window + } + + [Fact] + public async Task RunUntilEmptyAsync_WhenAllEventsAreAfterWindowEnd_ShouldSkipRepair() + { + // Arrange + // All events for this stack are after the window end — excluded from the aggregation. + TimeProvider.SetUtcNow(new DateTime(2026, 2, 24, 0, 0, 0, DateTimeKind.Utc)); + + var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, + organizationId: TestConstants.OrganizationId, + projectId: TestConstants.ProjectId, + totalOccurrences: 0), o => o.ImmediateConsistency()); + + await _eventRepository.AddAsync( + [ + _eventData.GenerateEvent(organizationId: TestConstants.OrganizationId, + projectId: TestConstants.ProjectId, stackId: stack.Id, + occurrenceDate: new DateTimeOffset(2026, 2, 24, 0, 0, 0, TimeSpan.Zero)) + ], + o => o.ImmediateConsistency()); + + // Act + await _workItemQueue.EnqueueAsync(new FixStackStatsWorkItem + { + UtcStart = DefaultWindowStart, + UtcEnd = DefaultWindowEnd // Feb 23 — before this stack's events (Feb 24) + }); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + stack = await _stackRepository.GetByIdAsync(stack.Id); + + // Assert + Assert.NotNull(stack); + Assert.Equal(0, stack.TotalOccurrences); // Not touched — events outside window + } + + [Fact] + public async Task RunUntilEmptyAsync_WhenOrganizationIdIsSpecified_ShouldOnlyRepairThatOrg() + { + // Arrange + TimeProvider.SetUtcNow(InWindowDate); + + // Stack in the target org with corrupted counters + var targetStack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, + organizationId: TestConstants.OrganizationId, + projectId: TestConstants.ProjectId, + utcFirstOccurrence: InWindowDate.AddDays(1), // wrong: too late + utcLastOccurrence: InWindowDate.AddDays(-1), // wrong: too early + totalOccurrences: 0), o => o.ImmediateConsistency()); + + await _eventRepository.AddAsync( + [_eventData.GenerateEvent(TestConstants.OrganizationId, TestConstants.ProjectId, targetStack.Id, + occurrenceDate: new DateTimeOffset(InWindowDate, TimeSpan.Zero))], + o => o.ImmediateConsistency()); + + // Stack in a different org — should not be touched + const string otherOrgId = TestConstants.OrganizationId2; + const string otherProjectId = "1ecd0826e447ad1e78877ab9"; + var otherStack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, + organizationId: otherOrgId, + projectId: otherProjectId, + utcFirstOccurrence: InWindowDate.AddDays(1), + utcLastOccurrence: InWindowDate.AddDays(-1), + totalOccurrences: 0), o => o.ImmediateConsistency()); + + await _eventRepository.AddAsync( + [_eventData.GenerateEvent(otherOrgId, otherProjectId, otherStack.Id, + occurrenceDate: new DateTimeOffset(InWindowDate, TimeSpan.Zero))], + o => o.ImmediateConsistency()); + + // Act: repair only the target org + await _workItemQueue.EnqueueAsync(new FixStackStatsWorkItem + { + UtcStart = DefaultWindowStart, + UtcEnd = DefaultWindowEnd, + OrganizationId = TestConstants.OrganizationId + }); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + targetStack = await _stackRepository.GetByIdAsync(targetStack.Id); + otherStack = await _stackRepository.GetByIdAsync(otherStack.Id); + + // Assert + Assert.NotNull(targetStack); + Assert.Equal(1, targetStack.TotalOccurrences); // Fixed + + Assert.NotNull(otherStack); + Assert.Equal(0, otherStack.TotalOccurrences); // Not touched — different org + } + + [Fact] + public async Task RunUntilEmptyAsync_WhenStackHasNoEvents_ShouldLeaveCountersUnchanged() + { + // Arrange + // Stack is in the bug window but has no events — should be left as-is. + TimeProvider.SetUtcNow(InWindowDate); + + var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, + organizationId: TestConstants.OrganizationId, + projectId: TestConstants.ProjectId, + totalOccurrences: 0), o => o.ImmediateConsistency()); + + // Act + await _workItemQueue.EnqueueAsync(new FixStackStatsWorkItem + { + UtcStart = DefaultWindowStart, + UtcEnd = DefaultWindowEnd + }); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + stack = await _stackRepository.GetByIdAsync(stack.Id); + + // Assert + Assert.NotNull(stack); + Assert.Equal(0, stack.TotalOccurrences); // Not touched — no events to derive stats from + } + + [Fact] + public async Task RunUntilEmptyAsync_WhenAggregatedTotalIsLowerThanCurrent_ShouldNotDecreaseTotalOccurrences() + { + // Arrange + TimeProvider.SetUtcNow(InWindowDate); + + var occurrenceDate = new DateTime(2026, 2, 15, 12, 0, 0, DateTimeKind.Utc); + + var stack = await _stackRepository.AddAsync(_stackData.GenerateStack(generateId: true, + organizationId: TestConstants.OrganizationId, + projectId: TestConstants.ProjectId, + utcFirstOccurrence: occurrenceDate, + utcLastOccurrence: occurrenceDate, + totalOccurrences: 10), o => o.ImmediateConsistency()); + + await _eventRepository.AddAsync( + [ + _eventData.GenerateEvent(TestConstants.OrganizationId, TestConstants.ProjectId, stack.Id, + occurrenceDate: new DateTimeOffset(occurrenceDate, TimeSpan.Zero)) + ], + o => o.ImmediateConsistency()); + + // Act + await _workItemQueue.EnqueueAsync(new FixStackStatsWorkItem + { + UtcStart = DefaultWindowStart, + UtcEnd = DefaultWindowEnd + }); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + stack = await _stackRepository.GetByIdAsync(stack.Id); + + // Assert + Assert.NotNull(stack); + Assert.Equal(10, stack.TotalOccurrences); + } +} diff --git a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs index 6e7ba9c3c4..c8e0cf9528 100644 --- a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs @@ -1,6 +1,5 @@ using System.Diagnostics; using System.Text.Json; -using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; using Exceptionless.Core.Repositories; diff --git a/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs index 979497eabf..bd765b8129 100644 --- a/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs @@ -168,6 +168,50 @@ public async Task CanIncrementEventCounterAsync() Assert.Equal(utcNow.AddDays(1), stack.LastOccurrence); } + [Fact] + public async Task SetEventCounterAsync_WhenIncomingValuesAreOlderOrLower_ShouldOnlyApplyMonotonicUpdates() + { + // Arrange + var originalFirst = new DateTime(2026, 2, 15, 0, 0, 0, DateTimeKind.Utc); + var originalLast = new DateTime(2026, 2, 16, 0, 0, 0, DateTimeKind.Utc); + var stack = await _repository.AddAsync(_stackData.GenerateStack( + generateId: true, + organizationId: TestConstants.OrganizationId, + projectId: TestConstants.ProjectId, + utcFirstOccurrence: originalFirst, + utcLastOccurrence: originalLast, + totalOccurrences: 10), o => o.ImmediateConsistency()); + // Act + await _repository.SetEventCounterAsync( + stack.Id, + originalFirst.AddDays(1), + originalLast.AddDays(-1), + 5, + sendNotifications: false); + + var unchanged = await _repository.GetByIdAsync(stack.Id); + + // Assert + Assert.Equal(10, unchanged.TotalOccurrences); + Assert.Equal(originalFirst, unchanged.FirstOccurrence); + Assert.Equal(originalLast, unchanged.LastOccurrence); + + // Act + await _repository.SetEventCounterAsync( + stack.Id, + originalFirst.AddDays(-1), + originalLast.AddDays(1), + 15, + sendNotifications: false); + + var updated = await _repository.GetByIdAsync(stack.Id); + + // Assert + Assert.Equal(15, updated.TotalOccurrences); + Assert.Equal(originalFirst.AddDays(-1), updated.FirstOccurrence); + Assert.Equal(originalLast.AddDays(1), updated.LastOccurrence); + } + [Fact] public async Task CanFindManyAsync() { @@ -199,11 +243,16 @@ public async Task GetStacksForCleanupAsync() var openStack10DaysOldWithReference = _stackData.GenerateStack(id: TestConstants.StackId3, utcLastOccurrence: utcNow.SubtractDays(10), status: StackStatus.Open); openStack10DaysOldWithReference.References.Add("test"); - await _repository.AddAsync(new List { - _stackData.GenerateStack(id: TestConstants.StackId, utcLastOccurrence: utcNow.SubtractDays(5), status: StackStatus.Open), - _stackData.GenerateStack(id: TestConstants.StackId2, utcLastOccurrence: utcNow.SubtractDays(10), status: StackStatus.Open), + await _repository.AddAsync( + new List + { + _stackData.GenerateStack(id: TestConstants.StackId, utcLastOccurrence: utcNow.SubtractDays(5), + status: StackStatus.Open), + _stackData.GenerateStack(id: TestConstants.StackId2, utcLastOccurrence: utcNow.SubtractDays(10), + status: StackStatus.Open), openStack10DaysOldWithReference, - _stackData.GenerateStack(id: TestConstants.StackId4, utcLastOccurrence: utcNow.SubtractDays(10), status: StackStatus.Fixed) + _stackData.GenerateStack(id: TestConstants.StackId4, utcLastOccurrence: utcNow.SubtractDays(10), + status: StackStatus.Fixed) }, o => o.ImmediateConsistency()); var stacks = await _repository.GetStacksForCleanupAsync(TestConstants.OrganizationId, utcNow.SubtractDays(8)); diff --git a/tests/http/admin.http b/tests/http/admin.http index e0f7742354..f02571be7c 100644 --- a/tests/http/admin.http +++ b/tests/http/admin.http @@ -85,3 +85,11 @@ Authorization: Bearer {{token}} GET {{apiUrl}}/admin/maintenance/update-project-default-bot-lists Authorization: Bearer {{token}} +### Fix Stack Stats (defaults: utcStart=2026-02-10T00:00:00Z, utcEnd=null=>now) +GET {{apiUrl}}/admin/maintenance/fix-stack-stats +Authorization: Bearer {{token}} + +### Fix Stack Stats (explicit UTC window) +GET {{apiUrl}}/admin/maintenance/fix-stack-stats?utcStart=2026-02-10T00:00:00Z&utcEnd=2026-02-23T00:00:00Z +Authorization: Bearer {{token}} +