fix: enhance error handling and messaging for URL submissions and pro…

src/controllers/resultsController.js

@@ -103,15 +103,23 @@ export async function getRequestDataMiddleware (req, res, next) {
 export async function checkForErroredResponse (req, res, next) {
   // Sentry metrics will count repeatedly if page reloads - not complete solution.
   if (req.locals.requestData.response?.error) {
-    const { errMsg } = req.locals.requestData.response.error
+    const processingError = req.locals.requestData.response.error
+    const { errMsg } = processingError
+    const organisationId = req.locals.requestData.getParams()?.organisationName
+    const errorOptions = {
+      template: 'check/error-redirect.html',
+      errorDetail: processingError,
+      organisationId,
+      organisationName: organisationId ? orgIdToName(organisationId) : undefined
+    }
     if (errMsg && errMsg.length > 0) {
       Sentry.metrics.count('url_submission.async_processing_failure', 1, { attributes: { error_message: errMsg } })
       // Disable as this is not an error we want to track in Sentry, only need metrics
       Sentry.getCurrentScope().setTag('async_handled_processing_error', true)
-      return next(new MiddlewareError(errMsg, 500, { template: 'check/error-redirect.html' }))
+      return next(new MiddlewareError(errMsg, 500, errorOptions))
     } else {
       Sentry.metrics.count('url_submission.async_processing_failure', 1, { attributes: { error_message: 'unknown' } })
-      return next(new MiddlewareError('An unknown error occurred when processing your endpoint', 500, { template: 'check/error-redirect.html' }))
+      return next(new MiddlewareError('An unknown error occurred when processing your endpoint', 500, errorOptions))
     }
   }
   next()

src/controllers/submitUrlController.js

@@ -7,6 +7,8 @@ import { types } from '../utils/logging.js'
 import axios from 'axios'
 import { allowedFileTypes } from '../utils/utils.js'
 import config from '../../config/index.js'
+import { MiddlewareError } from '../utils/errors.js'
+import { orgIdToName } from '../utils/orgIdToName.js'
 
 const HTTP_STATUS_METHOD_NOT_ALLOWED = 405
 const HTTP_STATUS_BLOCKED = 403
@@ -17,6 +19,17 @@ class SubmitUrlController extends UploadController {
     const localValidationErrorType = await SubmitUrlController.localUrlValidation(req.body.url)
 
     if (localValidationErrorType) {
+      if (localValidationErrorType === 'restricted403') {
+        const organisationId = req.sessionModel.get('orgId')
+        const organisationName = req.sessionModel.get('lpa') ?? orgIdToName(organisationId)
+        return next(new MiddlewareError('We cannot access your endpoint URL', 403, {
+          template: 'check/error-redirect.html',
+          errorDetail: { errCode: '403' },
+          organisationId,
+          organisationName
+        }))
+      }
+
       const error = {
         key: 'url',
         type: localValidationErrorType

src/utils/errors.js

@@ -22,6 +22,8 @@ export class MiddlewareError extends Error {
    * @param {Object} [options] - Additional options
    * @param {string} [options.template] - Custom error template path
    * @param {Error} [options.cause] - Cause of the error
+   * @param {string} [options.organisationId] - Organisation ID for a return link
+   * @param {string} [options.organisationName] - Organisation name for a return link
    */
   constructor (message, statusCode, options = {}) {
     super(message, options)
@@ -31,5 +33,7 @@ export class MiddlewareError extends Error {
     this.statusCode = statusCode
     this.template = options?.template ?? 'errorPages/error.njk'
     this.errorDetail = options?.errorDetail
+    this.organisationId = options?.organisationId
+    this.organisationName = options?.organisationName
   }
 }

src/views/check/error-redirect.html

@@ -9,20 +9,27 @@
 
 {% block content %}
 {% set errorMessage = err.message | default("An error occurred, please try again later.", true) %}
+{% set errorCode = err.errorDetail.errCode %}
+{% set exceptionType = err.errorDetail.exceptionType %}
+{% set contentType = err.errorDetail.contentType %}
+{% set plugin = err.errorDetail.plugin %}
+
+{% if errorCode != "403" and errorCode != 403 %}
 {{ govukErrorSummary({
-          titleText: "There’s a problem",
-          errorList: [
-            {
-              text: errorMessage,
-              href: "#bad-upload"
-            }
-          ]
+  titleText: "There’s a problem",
+  errorList: [
+    {
+      text: errorMessage,
+      href: "#bad-upload"
+    }
+  ]
 }) }}
+{% endif %}
 
 <div class="govuk-grid-row">
   <div class="govuk-grid-column-two-thirds">
     {# Display specific error details for certain errors else show error message if available #}
-    {% if errorMessage == "SSL certificate verification failed" %}
+    {% if exceptionType == "SSLError" or exceptionType == "SSLCertVerificationError" %}
       <p class="govuk-body" id="bad-upload" tabindex="-1">
         The link we’re trying to access is not loading using a secure connection. We could not verify the Secure Sockets Layer (SSL) certificate.
       </p>
@@ -38,7 +45,22 @@
       </ul>
 
       <p class="govuk-body">Contact your IT team for support if you need it.</p>
-    {% elif errorMessage == "The selected file must be a CSV, GeoJSON, GML or GeoPackage file" %}
+    {% elif errorCode == "403" or errorCode == 403 %}
+      <h1 class="govuk-heading-xl">We cannot access your endpoint URL</h1>
+
+      <p class="govuk-body">You need to:</p>
+      <ul class="govuk-list govuk-list--bullet">
+        <li>host the URL on a server that does not block access with set permissions</li>
+        <li>remove any bot protection that blocks automated downloads</li>
+      </ul>
+
+      <p class="govuk-body">If you need help, contact your IT team for support and tell them that you have a ‘HTTP status code 403’ error.</p>
+      <p class="govuk-body">If your IT team needs the Planning Data platform IP addresses, email <a class="govuk-link" href="mailto:digitalland@communities.gov.uk">digitalland@communities.gov.uk</a>.</p>
+
+      {% if err.organisationId and err.organisationName %}
+        <p class="govuk-body"><a class="govuk-link" href="/organisations/{{ err.organisationId | urlencode }}">Return to {{ err.organisationName }} overview</a></p>
+      {% endif %}
+    {% elif contentType and "text/html" in contentType %}
       <p class="govuk-body" id="bad-upload" tabindex="-1">The URL returns a HTML webpage. This must be a data file.</p>
 
       <p class="govuk-body">Provide a direct link to the data file in one of the following formats:</p>
@@ -52,10 +74,7 @@
       </ul>
 
       <p class="govuk-body">Contact your IT team for support if you need it.</p>
-    {% elif errorMessage == "The URL must be accessible" %}
-      <p class="govuk-body" id="bad-upload" tabindex="-1">You must host the URL on a server which does not block access due to set permissions.</p>
-      <p class="govuk-body" id="bad-upload" tabindex="-1">Contact your IT team for support if you need it, referencing a 'HTTP status code 403' error.</p>
-    {% elif errorMessage == "URL must be the data layer" %}
+    {% elif plugin == "arcgis" and (errorCode == "200" or errorCode == 200) %}
       <p class="govuk-body" id="bad-upload" tabindex="-1">The URL you have provided is an ArcGIS link, which is not the data layer.</p>
       <p class="govuk-body" id="bad-upload" tabindex="-1">The link to the data layer ends with a forward slash, followed by a number. For example, /8.</p>
       <p class="govuk-body" id="bad-upload" tabindex="-1">If you believe you have provided the information correctly and the problem persists, contact us at <a
@@ -66,18 +85,19 @@
       </p>
     {% endif %}
     {# Specific case where we don't want duplication of contact reasons #}
-    {% if errorMessage != "URL must be the data layer" %}
+    {% if (errorCode != "403" and errorCode != 403) and (plugin != "arcgis" or (errorCode != "200" and errorCode != 200)) %}
       <p class="govuk-body">Please <a href="/check/upload-method">try again</a> or <a
           href="mailto:digitalland@communities.gov.uk">contact support</a> if the problem persists.</p>
     {% endif %}
   </div>
 </div>
-
 {% endblock %}
 
 {% block beforeContent %}
+{% if err.errorDetail.errCode != "403" and err.errorDetail.errCode != 403 %}
 {{ govukBackLink({
 text: "Back",
 href: "javascript:window.history.back()"
 }) }}
-{% endblock %}
+{% endif %}
+{% endblock %}

src/views/check/url.html

@@ -62,6 +62,7 @@
           <p class="govuk-body-s">If you are not sure about how to find your endpoint details, contact your GIS specialist, or <a href="mailto:DigitalLand@communities.gov.uk" class="govuk-link">DigitalLand@communities.gov.uk</a>.</p>
           {% if 'url' in errors and errors['url'].type == 'restricted403' %}
             <p class="govuk-body">You must host the URL on a server which does not block access due to set permissions.</p>
+            <p class="govuk-body">The website may also use bot protection, such as Cloudflare or Imperva, which blocks automated downloads.</p>
             <p class="govuk-body">Contact your IT team for support if you need it, referencing a 'HTTP status code 403' error.</p>
           {% endif %}
         {% endset %}
@@ -90,4 +91,4 @@
       </form>
     </div>
   </div>
-{% endblock %}
+{% endblock %}

test/unit/submitUrlController.test.js

@@ -126,6 +126,30 @@ describe('SubmitUrlController', async () => {
       expect(asyncRequestApi.postUrlRequest).not.toHaveBeenCalled()
       expect(next).toHaveBeenCalled()
     })
+
+    it('should render the dedicated page for a locally detected 403', async () => {
+      const localUrlValidation = vi.spyOn(SubmitUrlController, 'localUrlValidation').mockResolvedValue('restricted403')
+      const req = {
+        body: { url: 'http://example.com' },
+        sessionModel: {
+          get: vi.fn((key) => ({ orgId: 'local-authority:ABC', lpa: 'Example Council' })[key])
+        },
+        session: { id: '1234' }
+      }
+      const next = vi.fn()
+
+      await submitUrlController.post(req, {}, next)
+
+      expect(asyncRequestApi.postUrlRequest).not.toHaveBeenCalled()
+      expect(next).toHaveBeenCalledWith(expect.objectContaining({
+        statusCode: 403,
+        template: 'check/error-redirect.html',
+        errorDetail: { errCode: '403' },
+        organisationId: 'local-authority:ABC',
+        organisationName: 'Example Council'
+      }))
+      localUrlValidation.mockRestore()
+    })
   })
 
   describe('getHeadRequest', () => {

test/unit/views/check/errorRedirect.test.js

@@ -0,0 +1,49 @@
+import { describe, expect, it } from 'vitest'
+import { setupNunjucks } from '../../../../src/serverSetup/nunjucks.js'
+
+const nunjucks = setupNunjucks({ datasetNameMapping: new Map() })
+
+const renderError = (errorDetail, organisation = {}) => nunjucks.render('check/error-redirect.html', {
+  err: {
+    message: 'A user-facing message that may change',
+    errorDetail,
+    ...organisation
+  }
+})
+
+describe('check error redirect page', () => {
+  it('uses the exception type for SSL certificate errors', () => {
+    const html = renderError({ exceptionType: 'SSLCertVerificationError' })
+
+    expect(html).toContain('We could not verify the Secure Sockets Layer (SSL) certificate')
+  })
+
+  it('renders the dedicated 403 page and LPA overview link', () => {
+    const html = renderError({ errCode: '403', contentType: 'text/html' }, {
+      organisationId: 'local-authority:ABC',
+      organisationName: 'Example Council'
+    })
+
+    expect(html).toContain('We cannot access your endpoint URL')
+    expect(html).toContain('host the URL on a server that does not block access with set permissions')
+    expect(html).toContain('remove any bot protection that blocks automated downloads')
+    expect(html).toContain('‘HTTP status code 403’ error')
+    expect(html).toContain('digitalland@communities.gov.uk')
+    expect(html).toContain('href="/organisations/local-authority%3AABC"')
+    expect(html).toContain('Return to Example Council overview')
+    expect(html).not.toContain('There’s a problem')
+  })
+
+  it('uses the content type for HTML responses', () => {
+    const html = renderError({ contentType: 'text/html; charset=UTF-8' })
+
+    expect(html).toContain('The URL returns a HTML webpage')
+  })
+
+  it('uses the error code and plugin for ArcGIS layer errors', () => {
+    const html = renderError({ errCode: '200', plugin: 'arcgis' })
+
+    expect(html).toContain('The URL you have provided is an ArcGIS link, which is not the data layer')
+    expect(html).not.toContain('contact support')
+  })
+})

test/unit/views/check/url.test.js

@@ -0,0 +1,17 @@
+import { describe, expect, it } from 'vitest'
+import { setupNunjucks } from '../../../../src/serverSetup/nunjucks.js'
+
+const nunjucks = setupNunjucks({ datasetNameMapping: new Map() })
+
+describe('check URL page', () => {
+  it('explains that bot protection can cause a 403 error', () => {
+    const html = nunjucks.render('check/url.html', {
+      errors: { url: { type: 'restricted403' } },
+      options: {},
+      data: { check: { url: '' } }
+    })
+
+    expect(html).toContain('bot protection, such as Cloudflare or Imperva')
+    expect(html).toContain('The URL must be accessible')
+  })
+})