From 43651f903db3b2fae2c594e1ddcc3882d35711ff Mon Sep 17 00:00:00 2001 From: Gibah Joseph Date: Thu, 20 Aug 2026 13:10:52 +0100 Subject: [PATCH 1/5] fix: enhance error handling and messaging for URL submissions and processing --- src/controllers/resultsController.js | 7 ++-- src/filters/validationMessageLookup.js | 2 +- src/views/check/error-redirect.html | 21 ++++++----- src/views/check/url.html | 3 +- test/unit/views/check/errorRedirect.test.js | 40 +++++++++++++++++++++ test/unit/views/check/url.test.js | 17 +++++++++ 6 files changed, 77 insertions(+), 13 deletions(-) create mode 100644 test/unit/views/check/errorRedirect.test.js create mode 100644 test/unit/views/check/url.test.js diff --git a/src/controllers/resultsController.js b/src/controllers/resultsController.js index 6bb100116..6bbd1ee34 100644 --- a/src/controllers/resultsController.js +++ b/src/controllers/resultsController.js @@ -85,15 +85,16 @@ export async function getRequestDataMiddleware (req, res, next) { export async function checkForErroredResponse (req, res, next) { // Sentry metrics will count repeatedly if page reloads - not complete solution. if (req.locals.requestData.response?.error) { - const { errMsg } = req.locals.requestData.response.error + const processingError = req.locals.requestData.response.error + const { errMsg } = processingError if (errMsg && errMsg.length > 0) { Sentry.metrics.count('url_submission.async_processing_failure', 1, { attributes: { error_message: errMsg } }) // Disable as this is not an error we want to track in Sentry, only need metrics Sentry.getCurrentScope().setTag('async_handled_processing_error', true) - return next(new MiddlewareError(errMsg, 500, { template: 'check/error-redirect.html' })) + return next(new MiddlewareError(errMsg, 500, { template: 'check/error-redirect.html', errorDetail: processingError })) } else { Sentry.metrics.count('url_submission.async_processing_failure', 1, { attributes: { error_message: 'unknown' } }) - return next(new MiddlewareError('An unknown error occurred when processing your endpoint', 500, { template: 'check/error-redirect.html' })) + return next(new MiddlewareError('An unknown error occurred when processing your endpoint', 500, { template: 'check/error-redirect.html', errorDetail: processingError })) } } next() diff --git a/src/filters/validationMessageLookup.js b/src/filters/validationMessageLookup.js index 3415ed708..f9ae7fcab 100644 --- a/src/filters/validationMessageLookup.js +++ b/src/filters/validationMessageLookup.js @@ -31,7 +31,7 @@ const validationMessages = { exists: 'The URL does not exist. Check the URL you\'ve entered is correct', filetype: 'The file referenced by URL must be a CSV, GeoJSON, GML or GeoPackage file', size: 'The file referenced by URL must be smaller than 100MB', - restricted403: 'The URL must be accessible' + restricted403: 'We could not access this URL automatically. It may be protected by a bot-protection service, such as Cloudflare or Imperva.' }, validationResult: { required: 'Sorry, there is a problem with the service. Try again later' diff --git a/src/views/check/error-redirect.html b/src/views/check/error-redirect.html index 2742f7b0f..4bed505b2 100644 --- a/src/views/check/error-redirect.html +++ b/src/views/check/error-redirect.html @@ -9,6 +9,10 @@ {% block content %} {% set errorMessage = err.message | default("An error occurred, please try again later.", true) %} +{% set errorCode = err.errorDetail.errCode %} +{% set exceptionType = err.errorDetail.exceptionType %} +{% set contentType = err.errorDetail.contentType %} +{% set plugin = err.errorDetail.plugin %} {{ govukErrorSummary({ titleText: "There’s a problem", errorList: [ @@ -22,7 +26,7 @@
{# Display specific error details for certain errors else show error message if available #} - {% if errorMessage == "SSL certificate verification failed" %} + {% if exceptionType == "SSLError" or exceptionType == "SSLCertVerificationError" %}

The link we’re trying to access is not loading using a secure connection. We could not verify the Secure Sockets Layer (SSL) certificate.

@@ -38,7 +42,11 @@

Contact your IT team for support if you need it.

- {% elif errorMessage == "The selected file must be a CSV, GeoJSON, GML or GeoPackage file" %} + {% elif errorCode == "403" or errorCode == 403 %} +

You must host the URL on a server which does not block access due to set permissions.

+

The website may also use bot protection, such as Cloudflare or Imperva, which blocks automated downloads.

+

Contact your IT team for support if you need it, referencing a 'HTTP status code 403' error.

+ {% elif contentType and "text/html" in contentType %}

The URL returns a HTML webpage. This must be a data file.

Provide a direct link to the data file in one of the following formats:

@@ -52,10 +60,7 @@

Contact your IT team for support if you need it.

- {% elif errorMessage == "The URL must be accessible" %} -

You must host the URL on a server which does not block access due to set permissions.

-

Contact your IT team for support if you need it, referencing a 'HTTP status code 403' error.

- {% elif errorMessage == "URL must be the data layer" %} + {% elif plugin == "arcgis" and (errorCode == "200" or errorCode == 200) %}

The URL you have provided is an ArcGIS link, which is not the data layer.

The link to the data layer ends with a forward slash, followed by a number. For example, /8.

If you believe you have provided the information correctly and the problem persists, contact us at {% endif %} {# Specific case where we don't want duplication of contact reasons #} - {% if errorMessage != "URL must be the data layer" %} + {% if plugin != "arcgis" or (errorCode != "200" and errorCode != 200) %}

Please try again or contact support if the problem persists.

{% endif %} @@ -80,4 +85,4 @@ text: "Back", href: "javascript:window.history.back()" }) }} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/src/views/check/url.html b/src/views/check/url.html index 813168028..80ba696b5 100644 --- a/src/views/check/url.html +++ b/src/views/check/url.html @@ -62,6 +62,7 @@

If you are not sure about how to find your endpoint details, contact your GIS specialist, or DigitalLand@communities.gov.uk.

{% if 'url' in errors and errors['url'].type == 'restricted403' %}

You must host the URL on a server which does not block access due to set permissions.

+

The website may also use bot protection, such as Cloudflare or Imperva, which blocks automated downloads.

Contact your IT team for support if you need it, referencing a 'HTTP status code 403' error.

{% endif %} {% endset %} @@ -90,4 +91,4 @@
-{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/test/unit/views/check/errorRedirect.test.js b/test/unit/views/check/errorRedirect.test.js new file mode 100644 index 000000000..e8eb48f65 --- /dev/null +++ b/test/unit/views/check/errorRedirect.test.js @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest' +import { setupNunjucks } from '../../../../src/serverSetup/nunjucks.js' + +const nunjucks = setupNunjucks({ datasetNameMapping: new Map() }) + +const renderError = (errorDetail) => nunjucks.render('check/error-redirect.html', { + err: { + message: 'A user-facing message that may change', + errorDetail + } +}) + +describe('check error redirect page', () => { + it('uses the exception type for SSL certificate errors', () => { + const html = renderError({ exceptionType: 'SSLCertVerificationError' }) + + expect(html).toContain('We could not verify the Secure Sockets Layer (SSL) certificate') + }) + + it('uses the 403 error code for inaccessible URLs', () => { + const html = renderError({ errCode: '403', contentType: 'text/html' }) + + expect(html).toContain("referencing a 'HTTP status code 403' error") + expect(html).toContain('bot protection, such as Cloudflare or Imperva') + expect(html).not.toContain('The URL returns a HTML webpage') + }) + + it('uses the content type for HTML responses', () => { + const html = renderError({ contentType: 'text/html; charset=UTF-8' }) + + expect(html).toContain('The URL returns a HTML webpage') + }) + + it('uses the error code and plugin for ArcGIS layer errors', () => { + const html = renderError({ errCode: '200', plugin: 'arcgis' }) + + expect(html).toContain('The URL you have provided is an ArcGIS link, which is not the data layer') + expect(html).not.toContain('contact support') + }) +}) diff --git a/test/unit/views/check/url.test.js b/test/unit/views/check/url.test.js new file mode 100644 index 000000000..a1119d8d8 --- /dev/null +++ b/test/unit/views/check/url.test.js @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest' +import { setupNunjucks } from '../../../../src/serverSetup/nunjucks.js' + +const nunjucks = setupNunjucks({ datasetNameMapping: new Map() }) + +describe('check URL page', () => { + it('explains that bot protection can cause a 403 error', () => { + const html = nunjucks.render('check/url.html', { + errors: { url: { type: 'restricted403' } }, + options: {}, + data: { check: { url: '' } } + }) + + expect(html).toContain('bot protection, such as Cloudflare or Imperva') + expect(html).toContain('We could not access this URL automatically. It may be protected by a bot-protection service, such as Cloudflare or Imperva.') + }) +}) From 2d7094293d3cecebe1214fd3ff58d632c345c92d Mon Sep 17 00:00:00 2001 From: Gibah Joseph Date: Wed, 26 Aug 2026 14:57:13 +0100 Subject: [PATCH 2/5] fix: enhance error handling for 403 responses and include organisation details in error redirects --- src/controllers/resultsController.js | 12 ++++++++-- src/utils/errors.js | 4 ++++ src/views/check/error-redirect.html | 25 +++++++++++++++++++-- test/unit/views/check/errorRedirect.test.js | 25 ++++++++++++++------- 4 files changed, 54 insertions(+), 12 deletions(-) diff --git a/src/controllers/resultsController.js b/src/controllers/resultsController.js index 6bbd1ee34..bfd4de3c4 100644 --- a/src/controllers/resultsController.js +++ b/src/controllers/resultsController.js @@ -9,6 +9,7 @@ import performanceDbApi from '../services/performanceDbApi.js' import { isFeatureEnabled } from '../utils/features.js' import { splitByLeading } from '../utils/table.js' import { MiddlewareError } from '../utils/errors.js' +import { orgIdToName } from '../utils/orgIdToName.js' const isIssueDetailsPageEnabled = isFeatureEnabled('checkIssueDetailsPage') const failedFileRequestTemplate = 'results/failedFileRequest' @@ -87,14 +88,21 @@ export async function checkForErroredResponse (req, res, next) { if (req.locals.requestData.response?.error) { const processingError = req.locals.requestData.response.error const { errMsg } = processingError + const organisationId = req.locals.requestData.getParams()?.organisationName + const errorOptions = { + template: 'check/error-redirect.html', + errorDetail: processingError, + organisationId, + organisationName: organisationId ? orgIdToName(organisationId) : undefined + } if (errMsg && errMsg.length > 0) { Sentry.metrics.count('url_submission.async_processing_failure', 1, { attributes: { error_message: errMsg } }) // Disable as this is not an error we want to track in Sentry, only need metrics Sentry.getCurrentScope().setTag('async_handled_processing_error', true) - return next(new MiddlewareError(errMsg, 500, { template: 'check/error-redirect.html', errorDetail: processingError })) + return next(new MiddlewareError(errMsg, 500, errorOptions)) } else { Sentry.metrics.count('url_submission.async_processing_failure', 1, { attributes: { error_message: 'unknown' } }) - return next(new MiddlewareError('An unknown error occurred when processing your endpoint', 500, { template: 'check/error-redirect.html', errorDetail: processingError })) + return next(new MiddlewareError('An unknown error occurred when processing your endpoint', 500, errorOptions)) } } next() diff --git a/src/utils/errors.js b/src/utils/errors.js index 5ce28f535..1f74ddae7 100644 --- a/src/utils/errors.js +++ b/src/utils/errors.js @@ -22,6 +22,8 @@ export class MiddlewareError extends Error { * @param {Object} [options] - Additional options * @param {string} [options.template] - Custom error template path * @param {Error} [options.cause] - Cause of the error + * @param {string} [options.organisationId] - Organisation ID for a return link + * @param {string} [options.organisationName] - Organisation name for a return link */ constructor (message, statusCode, options = {}) { super(message, options) @@ -31,5 +33,7 @@ export class MiddlewareError extends Error { this.statusCode = statusCode this.template = options?.template ?? 'errorPages/error.njk' this.errorDetail = options?.errorDetail + this.organisationId = options?.organisationId + this.organisationName = options?.organisationName } } diff --git a/src/views/check/error-redirect.html b/src/views/check/error-redirect.html index 4bed505b2..d6a765360 100644 --- a/src/views/check/error-redirect.html +++ b/src/views/check/error-redirect.html @@ -13,6 +13,26 @@ {% set exceptionType = err.errorDetail.exceptionType %} {% set contentType = err.errorDetail.contentType %} {% set plugin = err.errorDetail.plugin %} +{% if errorCode == "403" or errorCode == 403 %} +
+
+

We cannot access your endpoint URL

+ +

You need to:

+
    +
  • host the URL on a server that does not block access with set permissions
    • +
  • remove any bot protection that blocks automated downloads
    • +
+ +

If you need help, contact your IT team for support and tell them that you have a ‘HTTP status code 403’ error.

+

If your IT team needs the Planning Data platform IP addresses, email digitalland@communities.gov.uk.

+ + {% if err.organisationId and err.organisationName %} +

Return to {{ err.organisationName }} overview

+ {% endif %} +
+
+{% else %} {{ govukErrorSummary({ titleText: "There’s a problem", errorList: [ @@ -44,7 +64,6 @@

Contact your IT team for support if you need it.

{% elif errorCode == "403" or errorCode == 403 %}

You must host the URL on a server which does not block access due to set permissions.

-

The website may also use bot protection, such as Cloudflare or Imperva, which blocks automated downloads.

Contact your IT team for support if you need it, referencing a 'HTTP status code 403' error.

{% elif contentType and "text/html" in contentType %}

The URL returns a HTML webpage. This must be a data file.

@@ -77,12 +96,14 @@ {% endif %} - +{% endif %} {% endblock %} {% block beforeContent %} +{% if err.errorDetail.errCode != "403" and err.errorDetail.errCode != 403 %} {{ govukBackLink({ text: "Back", href: "javascript:window.history.back()" }) }} +{% endif %} {% endblock %} diff --git a/test/unit/views/check/errorRedirect.test.js b/test/unit/views/check/errorRedirect.test.js index e8eb48f65..26271d7ba 100644 --- a/test/unit/views/check/errorRedirect.test.js +++ b/test/unit/views/check/errorRedirect.test.js @@ -3,10 +3,11 @@ import { setupNunjucks } from '../../../../src/serverSetup/nunjucks.js' const nunjucks = setupNunjucks({ datasetNameMapping: new Map() }) -const renderError = (errorDetail) => nunjucks.render('check/error-redirect.html', { +const renderError = (errorDetail, organisation = {}) => nunjucks.render('check/error-redirect.html', { err: { message: 'A user-facing message that may change', - errorDetail + errorDetail, + ...organisation } }) @@ -17,12 +18,20 @@ describe('check error redirect page', () => { expect(html).toContain('We could not verify the Secure Sockets Layer (SSL) certificate') }) - it('uses the 403 error code for inaccessible URLs', () => { - const html = renderError({ errCode: '403', contentType: 'text/html' }) - - expect(html).toContain("referencing a 'HTTP status code 403' error") - expect(html).toContain('bot protection, such as Cloudflare or Imperva') - expect(html).not.toContain('The URL returns a HTML webpage') + it('renders the dedicated 403 page and LPA overview link', () => { + const html = renderError({ errCode: '403', contentType: 'text/html' }, { + organisationId: 'local-authority:ABC', + organisationName: 'Example Council' + }) + + expect(html).toContain('We cannot access your endpoint URL') + expect(html).toContain('host the URL on a server that does not block access with set permissions') + expect(html).toContain('remove any bot protection that blocks automated downloads') + expect(html).toContain('‘HTTP status code 403’ error') + expect(html).toContain('digitalland@communities.gov.uk') + expect(html).toContain('href="/organisations/local-authority%3AABC"') + expect(html).toContain('Return to Example Council overview') + expect(html).not.toContain('There’s a problem') }) it('uses the content type for HTML responses', () => { From 881645efc5377cb2a6f760f2d2c14679ed24536e Mon Sep 17 00:00:00 2001 From: Gibah Joseph Date: Wed, 26 Aug 2026 15:48:56 +0100 Subject: [PATCH 3/5] fix: handle locally detected 403 errors with detailed organisation information in redirects --- src/controllers/submitUrlController.js | 13 +++++++++++++ test/unit/submitUrlController.test.js | 24 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/src/controllers/submitUrlController.js b/src/controllers/submitUrlController.js index 1fdd5f7ba..680a2e856 100644 --- a/src/controllers/submitUrlController.js +++ b/src/controllers/submitUrlController.js @@ -7,6 +7,8 @@ import { types } from '../utils/logging.js' import axios from 'axios' import { allowedFileTypes } from '../utils/utils.js' import config from '../../config/index.js' +import { MiddlewareError } from '../utils/errors.js' +import { orgIdToName } from '../utils/orgIdToName.js' const HTTP_STATUS_METHOD_NOT_ALLOWED = 405 const HTTP_STATUS_BLOCKED = 403 @@ -17,6 +19,17 @@ class SubmitUrlController extends UploadController { const localValidationErrorType = await SubmitUrlController.localUrlValidation(req.body.url) if (localValidationErrorType) { + if (localValidationErrorType === 'restricted403') { + const organisationId = req.sessionModel.get('orgId') + const organisationName = req.sessionModel.get('lpa') ?? orgIdToName(organisationId) + return next(new MiddlewareError('We cannot access your endpoint URL', 403, { + template: 'check/error-redirect.html', + errorDetail: { errCode: '403' }, + organisationId, + organisationName + })) + } + const error = { key: 'url', type: localValidationErrorType diff --git a/test/unit/submitUrlController.test.js b/test/unit/submitUrlController.test.js index 184eebf8b..53f6b42e3 100644 --- a/test/unit/submitUrlController.test.js +++ b/test/unit/submitUrlController.test.js @@ -126,6 +126,30 @@ describe('SubmitUrlController', async () => { expect(asyncRequestApi.postUrlRequest).not.toHaveBeenCalled() expect(next).toHaveBeenCalled() }) + + it('should render the dedicated page for a locally detected 403', async () => { + const localUrlValidation = vi.spyOn(SubmitUrlController, 'localUrlValidation').mockResolvedValue('restricted403') + const req = { + body: { url: 'http://example.com' }, + sessionModel: { + get: vi.fn((key) => ({ orgId: 'local-authority:ABC', lpa: 'Example Council' })[key]) + }, + session: { id: '1234' } + } + const next = vi.fn() + + await submitUrlController.post(req, {}, next) + + expect(asyncRequestApi.postUrlRequest).not.toHaveBeenCalled() + expect(next).toHaveBeenCalledWith(expect.objectContaining({ + statusCode: 403, + template: 'check/error-redirect.html', + errorDetail: { errCode: '403' }, + organisationId: 'local-authority:ABC', + organisationName: 'Example Council' + })) + localUrlValidation.mockRestore() + }) }) describe('getHeadRequest', () => { From 9aadf53310ab9816ef0ebff28867db70376fdeb7 Mon Sep 17 00:00:00 2001 From: Gibah Joseph Date: Wed, 26 Aug 2026 16:06:49 +0100 Subject: [PATCH 4/5] updated page content --- src/views/check/error-redirect.html | 54 +++++++++++++---------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/src/views/check/error-redirect.html b/src/views/check/error-redirect.html index d6a765360..b107a0de6 100644 --- a/src/views/check/error-redirect.html +++ b/src/views/check/error-redirect.html @@ -13,35 +13,18 @@ {% set exceptionType = err.errorDetail.exceptionType %} {% set contentType = err.errorDetail.contentType %} {% set plugin = err.errorDetail.plugin %} -{% if errorCode == "403" or errorCode == 403 %} -
-
-

We cannot access your endpoint URL

- -

You need to:

-
    -
  • host the URL on a server that does not block access with set permissions
    • -
  • remove any bot protection that blocks automated downloads
    • -
- -

If you need help, contact your IT team for support and tell them that you have a ‘HTTP status code 403’ error.

-

If your IT team needs the Planning Data platform IP addresses, email digitalland@communities.gov.uk.

- {% if err.organisationId and err.organisationName %} -

Return to {{ err.organisationName }} overview

- {% endif %} -
-
-{% else %} +{% if errorCode != "403" and errorCode != 403 %} {{ govukErrorSummary({ - titleText: "There’s a problem", - errorList: [ - { - text: errorMessage, - href: "#bad-upload" - } - ] + titleText: "There’s a problem", + errorList: [ + { + text: errorMessage, + href: "#bad-upload" + } + ] }) }} +{% endif %}
@@ -63,8 +46,20 @@

We cannot access your endpoint URL

Contact your IT team for support if you need it.

{% elif errorCode == "403" or errorCode == 403 %} -

You must host the URL on a server which does not block access due to set permissions.

-

Contact your IT team for support if you need it, referencing a 'HTTP status code 403' error.

+

We cannot access your endpoint URL

+ +

You need to:

+
    +
  • host the URL on a server that does not block access with set permissions
    • +
  • remove any bot protection that blocks automated downloads
    • +
+ +

If you need help, contact your IT team for support and tell them that you have a ‘HTTP status code 403’ error.

+

If your IT team needs the Planning Data platform IP addresses, email digitalland@communities.gov.uk.

+ + {% if err.organisationId and err.organisationName %} +

Return to {{ err.organisationName }} overview

+ {% endif %} {% elif contentType and "text/html" in contentType %}

The URL returns a HTML webpage. This must be a data file.

@@ -90,13 +85,12 @@

We cannot access your endpoint URL

{% endif %} {# Specific case where we don't want duplication of contact reasons #} - {% if plugin != "arcgis" or (errorCode != "200" and errorCode != 200) %} + {% if (errorCode != "403" and errorCode != 403) and (plugin != "arcgis" or (errorCode != "200" and errorCode != 200)) %}

Please try again or contact support if the problem persists.

{% endif %}
-{% endif %} {% endblock %} {% block beforeContent %} From 5cef431eed4b5f7f43af841173990497b8258dab Mon Sep 17 00:00:00 2001 From: Gibah Joseph Date: Wed, 26 Aug 2026 16:17:07 +0100 Subject: [PATCH 5/5] fix: update error message for restricted 403 responses to clarify URL accessibility --- src/filters/validationMessageLookup.js | 2 +- test/unit/views/check/url.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/filters/validationMessageLookup.js b/src/filters/validationMessageLookup.js index f9ae7fcab..3415ed708 100644 --- a/src/filters/validationMessageLookup.js +++ b/src/filters/validationMessageLookup.js @@ -31,7 +31,7 @@ const validationMessages = { exists: 'The URL does not exist. Check the URL you\'ve entered is correct', filetype: 'The file referenced by URL must be a CSV, GeoJSON, GML or GeoPackage file', size: 'The file referenced by URL must be smaller than 100MB', - restricted403: 'We could not access this URL automatically. It may be protected by a bot-protection service, such as Cloudflare or Imperva.' + restricted403: 'The URL must be accessible' }, validationResult: { required: 'Sorry, there is a problem with the service. Try again later' diff --git a/test/unit/views/check/url.test.js b/test/unit/views/check/url.test.js index a1119d8d8..9c244bfd9 100644 --- a/test/unit/views/check/url.test.js +++ b/test/unit/views/check/url.test.js @@ -12,6 +12,6 @@ describe('check URL page', () => { }) expect(html).toContain('bot protection, such as Cloudflare or Imperva') - expect(html).toContain('We could not access this URL automatically. It may be protected by a bot-protection service, such as Cloudflare or Imperva.') + expect(html).toContain('The URL must be accessible') }) })