diff --git a/__tests__/av_tests.js b/__tests__/av_tests.js new file mode 100644 index 000000000..6d849f3a0 --- /dev/null +++ b/__tests__/av_tests.js @@ -0,0 +1,478 @@ +const puppeteer = require("puppeteer"); +const { BASE_URL } = require("../scripts/testBaseUrl"); + +// AV (audiovisual) manifest for AV-specific behaviour. A simple single-file +// AV manifest (no ranges) is rendered by the mediaelement extension. +const AV_VIDEO_MANIFEST = + "https://iiif.io/api/cookbook/recipe/0003-mvm-video/manifest.json"; + +// AV manifest WITH a table of contents (structures/ranges). When an AV manifest +// has ranges and preferMediaElementExtension is false (the default), the viewer +// uses uv-av-extension -> AVCenterPanel (iiif-av-component) instead of the +// mediaelement player. +const AV_TOC_MANIFEST = + "https://iiif.io/api/cookbook/recipe/0064-opera-one-canvas/manifest.json"; + +// AV manifests with a transcription. UV surfaces transcriptions/captions in +// the mediaelement player when the canvas provides them as a text/vtt (or +// text/srt) rendering, as an additional body on the painting annotation, or +// as a supplementing annotation per IIIF cookbook recipe 0219 (inline or in +// an externally referenced annotation page). The AV extension used for +// manifests with ranges has no caption support. +// +// The player fetches the VTT with XHR, so cross-origin transcriptions work +// only when every response hop (including any redirect) carries an +// Access-Control-Allow-Origin header. The same-origin fixture below is +// immune to that; the cross-origin fixture points at fixtures.iiif.io, +// which serves CORS headers on a direct https URL. +const AV_CAPTIONED_MANIFEST = `${BASE_URL}/test-fixtures/captioned-video-manifest.json`; +const AV_CROSS_ORIGIN_CAPTIONED_MANIFEST = `${BASE_URL}/test-fixtures/cross-origin-captioned-video-manifest.json`; + +// This fixture references the VTT through http://dlib.indiana.edu, whose 301 +// upgrade redirect carries no CORS headers, so the URL is unreadable as-is. +// UV resolves such captions to their https destination before wiring the +// track (MediaElementCenterPanel.resolveCaptionSource). +const AV_REDIRECTED_CAPTIONED_MANIFEST = `${BASE_URL}/test-fixtures/redirected-captioned-video-manifest.json`; + +// The IIIF cookbook's own example of a captioned video: the VTT is a +// supplementing annotation in an inline annotation page on the canvas. +const AV_COOKBOOK_CAPTION_MANIFEST = + "https://iiif.io/api/cookbook/recipe/0219-using-caption-file/manifest.json"; + +// Same supplementing pattern, but the canvas references the annotation page +// by id only, so the viewer has to fetch it. +const AV_SUPPLEMENTING_EXTERNAL_MANIFEST = `${BASE_URL}/test-fixtures/supplementing-external-captioned-video-manifest.json`; + +const viewerUrl = (manifestUrl) => { + //const separator = BASE_URL.includes("#?") ? "&" : "#?"; + return `${BASE_URL}#?manifest=${encodeURIComponent(manifestUrl)}`; +}; + +describe("Universal Viewer", () => { + let browser; + let page; + + beforeAll(async () => { + browser = await puppeteer.launch({ + headless: true, + args: ["--no-sandbox", "--disable-setuid-sandbox"], + }); + page = await browser.newPage(); + }); + + afterAll(async () => { + await browser.close(); + }); + + // AV MANIFEST TEST + describe("AV manifest", () => { + // Use a dedicated page so the AV tests are isolated from whatever state + // (e.g. in-flight iframe loads) earlier suites left on the shared page. + let avPage; + + beforeAll(async () => { + avPage = await browser.newPage(); + }); + + afterAll(async () => { + await avPage.close(); + }); + + beforeEach(async () => { + // The example page reads the manifest from the URL only on the initial + // document load, so force a full reload (a hash-only change would not + // re-initialise the viewer). + await avPage.goto("about:blank"); + await avPage.goto(viewerUrl(AV_VIDEO_MANIFEST), { + waitUntil: "domcontentloaded", + }); + }, 60000); + + it("loads the AV manifest into the mediaelement player", async () => { + expect(avPage.url()).toContain(encodeURIComponent(AV_VIDEO_MANIFEST)); + + await avPage.waitForSelector(".uv", { visible: true }); + + // The AV manifest is handled by the mediaelement extension, which adds + // this class to the extension host element. + await avPage.waitForSelector(".uv-mediaelement-extension", { + visible: true, + }); + + // The MediaElement.js player renders into a .mejs__container. For a + // video canvas it also carries the .mejs__video class. + await avPage.waitForSelector(".mejs__container.mejs__video", { + visible: true, + }); + + // The underlying media element points at the canvas' video resource. + const videoSrc = await avPage.$eval( + ".mejs__mediaelement video", + (el) => el.src || el.querySelector("source")?.src || "" + ); + expect(videoSrc).toMatch(/\.mp4($|\?)/); + + const pageText = await avPage.evaluate(() => document.body.innerText); + expect(pageText).not.toContain("Unable to load"); + expect(pageText).not.toContain("Error loading"); + }, 60000); + + it("renders AV playback controls", async () => { + await avPage.waitForSelector(".mejs__controls", { visible: true }); + + // Play / pause button. + const playButton = await avPage.$(".mejs__playpause-button"); + expect(playButton).toBeTruthy(); + + // Current time readout. + const currentTime = await avPage.$eval(".mejs__currenttime", (el) => + el.textContent.trim() + ); + expect(currentTime).toMatch(/^\d{2}:\d{2}/); + }, 60000); + }); + + // AV MANIFEST WITH TABLE OF CONTENTS TEST + // An AV manifest that defines ranges/structures is routed to the AV extension + // (AVCenterPanel / iiif-av-component) rather than the mediaelement player. + describe("AV manifest with table of contents", () => { + // Use a dedicated page so the AV tests are isolated from whatever state + // (e.g. in-flight iframe loads) earlier suites left on the shared page. + let avPage; + + beforeAll(async () => { + avPage = await browser.newPage(); + }); + + afterAll(async () => { + await avPage.close(); + }); + + beforeEach(async () => { + // Force a full reload so the viewer re-initialises on this manifest + // (a hash-only change would keep the previously loaded manifest). + await avPage.goto("about:blank"); + await avPage.goto(viewerUrl(AV_TOC_MANIFEST), { + waitUntil: "domcontentloaded", + }); + }, 60000); + + it("loads the AV manifest into the AV center panel", async () => { + expect(avPage.url()).toContain(encodeURIComponent(AV_TOC_MANIFEST)); + + await avPage.waitForSelector(".uv", { visible: true }); + + // The AV extension mounts the AVComponent into a .iiif-av-component + // wrapper inside the center panel. + await avPage.waitForSelector(".iiif-av-component .player", { + visible: true, + }); + + // This path must NOT fall back to the mediaelement player. + const mejsCount = await avPage.$$eval( + ".mejs__container", + (els) => els.length + ); + expect(mejsCount).toBe(0); + + // The media element is created as video.anno / audio.anno. + await avPage.waitForSelector( + ".iiif-av-component video.anno, .iiif-av-component audio.anno" + ); + + const pageText = await avPage.evaluate(() => document.body.innerText); + expect(pageText).not.toContain("Unable to load"); + expect(pageText).not.toContain("Error loading"); + }, 60000); + + it("renders AV component playback controls", async () => { + await avPage.waitForSelector(".iiif-av-component .controls-container", { + visible: true, + }); + + // Play/pause button. + const playButton = await avPage.$( + ".iiif-av-component .controls-container .av-icon-play" + ); + expect(playButton).toBeTruthy(); + + // Duration display. + const duration = await avPage.$( + ".iiif-av-component .time-display .canvas-duration" + ); + expect(duration).toBeTruthy(); + }, 60000); + }); + + // AV MANIFEST WITH A TRANSCRIPTION TEST + describe("AV manifest with a transcription", () => { + let avPage; + + beforeAll(async () => { + avPage = await browser.newPage(); + }); + + afterAll(async () => { + await avPage.close(); + }); + + beforeEach(async () => { + // Force a full reload so the viewer re-initialises on this manifest. + await avPage.goto("about:blank"); + await avPage.goto(viewerUrl(AV_CAPTIONED_MANIFEST), { + waitUntil: "domcontentloaded", + }); + }, 60000); + + it("surfaces the transcription as a caption track in the player", async () => { + await avPage.waitForSelector(".mejs__container.mejs__video", { + visible: true, + }); + + // The captions button only renders when a caption track was wired up. + await avPage.waitForSelector(".mejs__captions-button", { + visible: true, + }); + + // The points at the manifest's VTT transcription. + const track = await avPage.$eval("track[src*='captions.vtt']", (t) => ({ + kind: t.kind, + srclang: t.srclang, + label: t.label, + })); + expect(track.kind).toBe("subtitles"); + expect(track.srclang).toBe("en"); + expect(track.label).toBe("English captions"); + + // The transcription appears as a selectable option, and becomes + // enabled once the player has loaded the VTT file. + await avPage.waitForFunction(() => { + const input = document.querySelector( + ".mejs__captions-selector input:not([value='none'])" + ); + return input && !input.disabled; + }); + }, 60000); + + it("displays the transcription text during playback", async () => { + await avPage.waitForSelector(".mejs__captions-button", { + visible: true, + }); + + // Wait for the player to finish loading the VTT. + await avPage.waitForFunction(() => { + const input = document.querySelector( + ".mejs__captions-selector input:not([value='none'])" + ); + return input && !input.disabled; + }); + + // Turn captions on through the player UI. + await avPage.evaluate(() => { + document + .querySelector(".mejs__captions-selector input:not([value='none'])") + .click(); + }); + + // Play (muted, so headless autoplay is allowed) to reach the first cue. + await avPage.evaluate(() => { + const video = document.querySelector(".mejs__mediaelement video"); + video.muted = true; + return video.play(); + }); + + await avPage.waitForFunction(() => { + const el = document.querySelector(".mejs__captions-text"); + return el && el.textContent.trim().length > 0; + }); + + const captionText = await avPage.$eval(".mejs__captions-text", (el) => + el.textContent.trim() + ); + expect(captionText).toBe( + "Just before lunch one day, a puppet show was put on at school." + ); + }, 60000); + + it("displays a cross-origin transcription served with CORS headers", async () => { + await avPage.goto("about:blank"); + await avPage.goto(viewerUrl(AV_CROSS_ORIGIN_CAPTIONED_MANIFEST), { + waitUntil: "domcontentloaded", + }); + + await avPage.waitForSelector(".mejs__captions-button", { + visible: true, + }); + + // The points at the cross-origin VTT. + const trackSrc = await avPage.$eval( + "track[src*='lunchroom_manners.vtt']", + (t) => t.src + ); + expect(trackSrc).toBe( + "https://fixtures.iiif.io/video/indiana/lunchroom_manners/lunchroom_manners.vtt" + ); + + // The caption option only becomes enabled once the player has + // successfully fetched the cross-origin VTT. + await avPage.waitForFunction(() => { + const input = document.querySelector( + ".mejs__captions-selector input:not([value='none'])" + ); + return input && !input.disabled; + }); + + // Turn captions on and play; the first cue ("[music]") starts 1.2s in. + await avPage.evaluate(() => { + document + .querySelector(".mejs__captions-selector input:not([value='none'])") + .click(); + const video = document.querySelector(".mejs__mediaelement video"); + video.muted = true; + return video.play(); + }); + + await avPage.waitForFunction(() => { + const el = document.querySelector(".mejs__captions-text"); + return el && el.textContent.trim().length > 0; + }); + + const captionText = await avPage.$eval(".mejs__captions-text", (el) => + el.textContent.trim() + ); + expect(captionText).toBe("[music]"); + }, 60000); + + it("resolves a transcription behind an institutional redirect", async () => { + await avPage.goto("about:blank"); + await avPage.goto(viewerUrl(AV_REDIRECTED_CAPTIONED_MANIFEST), { + waitUntil: "domcontentloaded", + }); + + await avPage.waitForSelector(".mejs__captions-button", { + visible: true, + }); + + // The manifest's http:// URL is unreadable (its redirect hop has no + // CORS headers), so UV must have resolved the track to the readable + // https destination. + const trackSrc = await avPage.$eval( + "track[src*='lunchroom_manners.vtt']", + (t) => t.src + ); + expect(trackSrc).toBe( + "https://dlib.indiana.edu/iiif_av/lunchroom_manners/lunchroom_manners.vtt" + ); + + await avPage.waitForFunction(() => { + const input = document.querySelector( + ".mejs__captions-selector input:not([value='none'])" + ); + return input && !input.disabled; + }); + + await avPage.evaluate(() => { + document + .querySelector(".mejs__captions-selector input:not([value='none'])") + .click(); + const video = document.querySelector(".mejs__mediaelement video"); + video.muted = true; + return video.play(); + }); + + await avPage.waitForFunction(() => { + const el = document.querySelector(".mejs__captions-text"); + return el && el.textContent.trim().length > 0; + }); + + const captionText = await avPage.$eval(".mejs__captions-text", (el) => + el.textContent.trim() + ); + expect(captionText).toBe("[music]"); + }, 60000); + + it("displays a transcription supplied as a supplementing annotation (cookbook 0219)", async () => { + await avPage.goto("about:blank"); + await avPage.goto(viewerUrl(AV_COOKBOOK_CAPTION_MANIFEST), { + waitUntil: "domcontentloaded", + }); + + await avPage.waitForSelector(".mejs__captions-button", { + visible: true, + }); + + // The track carries the annotation body's label and language. + const track = await avPage.$eval( + "track[src*='lunchroom_manners.vtt']", + (t) => ({ label: t.label, srclang: t.srclang }) + ); + expect(track.label).toBe("Captions in WebVTT format"); + expect(track.srclang).toBe("en"); + + await avPage.waitForFunction(() => { + const input = document.querySelector( + ".mejs__captions-selector input:not([value='none'])" + ); + return input && !input.disabled; + }); + + await avPage.evaluate(() => { + document + .querySelector(".mejs__captions-selector input:not([value='none'])") + .click(); + const video = document.querySelector(".mejs__mediaelement video"); + video.muted = true; + return video.play(); + }); + + await avPage.waitForFunction(() => { + const el = document.querySelector(".mejs__captions-text"); + return el && el.textContent.trim().length > 0; + }); + + const captionText = await avPage.$eval(".mejs__captions-text", (el) => + el.textContent.trim() + ); + expect(captionText).toBe("[music]"); + }, 60000); + + it("displays a transcription from an externally referenced annotation page", async () => { + await avPage.goto("about:blank"); + await avPage.goto(viewerUrl(AV_SUPPLEMENTING_EXTERNAL_MANIFEST), { + waitUntil: "domcontentloaded", + }); + + await avPage.waitForSelector(".mejs__captions-button", { + visible: true, + }); + + await avPage.waitForFunction(() => { + const input = document.querySelector( + ".mejs__captions-selector input:not([value='none'])" + ); + return input && !input.disabled; + }); + + await avPage.evaluate(() => { + document + .querySelector(".mejs__captions-selector input:not([value='none'])") + .click(); + const video = document.querySelector(".mejs__mediaelement video"); + video.muted = true; + return video.play(); + }); + + await avPage.waitForFunction(() => { + const el = document.querySelector(".mejs__captions-text"); + return el && el.textContent.trim().length > 0; + }); + + const captionText = await avPage.$eval(".mejs__captions-text", (el) => + el.textContent.trim() + ); + expect(captionText).toBe( + "Just before lunch one day, a puppet show was put on at school." + ); + }, 60000); + }); +}); diff --git a/__tests__/test.js b/__tests__/image_tests.js similarity index 85% rename from __tests__/test.js rename to __tests__/image_tests.js index 1f050fcb1..f9d23bf10 100644 --- a/__tests__/test.js +++ b/__tests__/image_tests.js @@ -1,5 +1,3 @@ -test.skip("Configuration options", () => {}); - const puppeteer = require("puppeteer"); const { BASE_URL } = require("../scripts/testBaseUrl"); @@ -7,10 +5,6 @@ const { BASE_URL } = require("../scripts/testBaseUrl"); const COOKBOOK_BOUND_MULTIVOLUME_MANIFEST = "https://iiif.io/api/cookbook/recipe/0031-bound-multivolume/manifest.json"; -// PDF manifest for PDF-specific behaviour -const PDF_MULTI_FILE_MANIFEST = - "https://digital.library.villanova.edu/Item/vudl:294631/Manifest"; - const viewerUrl = (manifestUrl) => { //const separator = BASE_URL.includes("#?") ? "&" : "#?"; return `${BASE_URL}#?manifest=${encodeURIComponent(manifestUrl)}`; @@ -211,9 +205,8 @@ describe("Universal Viewer", () => { // COOKBOOK MANIFEST TEST describe("viewer controls", () => { beforeEach(async () => { - await page.goto(viewerUrl(COOKBOOK_BOUND_MULTIVOLUME_MANIFEST), - { - waitUntil: "domcontentloaded" + await page.goto(viewerUrl(COOKBOOK_BOUND_MULTIVOLUME_MANIFEST), { + waitUntil: "domcontentloaded", }); }); @@ -414,73 +407,4 @@ describe("Universal Viewer", () => { await page.waitForSelector(moreInfoHeader, { hidden: true }); }); }); - - // PDF MANIFEST TEST - describe("PDF manifest", () => { - beforeEach(async () => { - await page.goto(viewerUrl(PDF_MULTI_FILE_MANIFEST), { - waitUntil: "domcontentloaded", - }); - }); - - it("loads PDF manifest successfully", async () => { - expect(page.url()).toContain(encodeURIComponent(PDF_MULTI_FILE_MANIFEST)); - - await page.waitForSelector(".uv", { visible: true }); - - await page.waitForFunction(() => { - return document.querySelectorAll("iframe").length > 0; - }); - - const viewerFrame = page.frames().find((f) => { - const url = f.url(); - - return ( - url.includes("uv.html") || - url.includes("viewer") || - url.includes("manifest") - ); - }); - - expect(viewerFrame).toBeTruthy(); - - await viewerFrame.waitForSelector("canvas", { visible: true}); - - const canvasInfo = await viewerFrame.evaluate(() => { - const canvas = document.querySelector( - "canvas" - ); - - if (!canvas) return null; - return { - width: canvas.width, - height: canvas.height, - }; - }); - expect(canvasInfo).not.toBeNull(); - expect(canvasInfo.width).toBeGreaterThan(0); - expect(canvasInfo.height).toBeGreaterThan(0); - - const pageText = await viewerFrame.evaluate(() => document.body.innerText); - - expect(pageText).not.toContain("Unable to load"); - expect(pageText).not.toContain("Error loading"); - }); - - it("shows multiple PDF files in the sidebar and allows navigation", async () => { - await page.waitForSelector("button.expandButton", { visible: true }); - await page.click("button.expandButton"); - - await page.waitForSelector(".thumb", { visible: true }); - - const thumbs = await page.$$(".thumb"); - - expect(thumbs.length).toBeGreaterThan(1); - await thumbs[1].click(); - - await page.waitForFunction(() => window.location.href.includes("cv=1")); - - expect(page.url()).toContain("cv=1"); - }); - }); }); diff --git a/__tests__/pdf_tests.js b/__tests__/pdf_tests.js new file mode 100644 index 000000000..e18b9cf66 --- /dev/null +++ b/__tests__/pdf_tests.js @@ -0,0 +1,142 @@ +const puppeteer = require("puppeteer"); +const { BASE_URL } = require("../scripts/testBaseUrl"); + +// PDF manifest for PDF-specific behaviour +const PDF_MULTI_FILE_MANIFEST = + "https://digital.library.villanova.edu/Item/vudl:294631/Manifest"; + +const viewerUrl = (manifestUrl) => { + //const separator = BASE_URL.includes("#?") ? "&" : "#?"; + return `${BASE_URL}#?manifest=${encodeURIComponent(manifestUrl)}`; +}; + +describe("Universal Viewer", () => { + let browser; + let page; + + beforeAll(async () => { + browser = await puppeteer.launch({ + headless: true, + args: ["--no-sandbox", "--disable-setuid-sandbox"], + }); + page = await browser.newPage(); + }); + + afterAll(async () => { + await browser.close(); + }); + + // PDF MANIFEST TEST + describe("PDF manifest", () => { + beforeEach(async () => { + await page.goto(viewerUrl(PDF_MULTI_FILE_MANIFEST), { + waitUntil: "domcontentloaded", + }); + }); + + it("loads PDF manifest successfully", async () => { + expect(page.url()).toContain(encodeURIComponent(PDF_MULTI_FILE_MANIFEST)); + + await page.waitForSelector(".uv", { visible: true }); + + await page.waitForFunction(() => { + return document.querySelectorAll("iframe").length > 0; + }); + + const viewerFrame = page.frames().find((f) => { + const url = f.url(); + + return ( + url.includes("uv.html") || + url.includes("viewer") || + url.includes("manifest") + ); + }); + + expect(viewerFrame).toBeTruthy(); + + await viewerFrame.waitForSelector("canvas", { visible: true }); + + const canvasInfo = await viewerFrame.evaluate(() => { + const canvas = document.querySelector("canvas"); + + if (!canvas) return null; + return { + width: canvas.width, + height: canvas.height, + }; + }); + expect(canvasInfo).not.toBeNull(); + expect(canvasInfo.width).toBeGreaterThan(0); + expect(canvasInfo.height).toBeGreaterThan(0); + + const pageText = await viewerFrame.evaluate( + () => document.body.innerText + ); + + expect(pageText).not.toContain("Unable to load"); + expect(pageText).not.toContain("Error loading"); + }); + + it("shows multiple PDF files in the sidebar and allows navigation", async () => { + // In a fresh browser session the left panel opens automatically + // (panelOpen defaults to true), so wait for it rather than clicking + // the expand button, which is only visible while the panel is closed. + await page.waitForSelector(".leftPanel.open", { visible: true }); + + await page.waitForSelector(".thumb", { visible: true }); + + const thumbs = await page.$$(".thumb"); + + expect(thumbs.length).toBeGreaterThan(1); + await thumbs[1].click(); + + await page.waitForFunction(() => window.location.href.includes("cv=1")); + + expect(page.url()).toContain("cv=1"); + }); + + it("can collapse and re-expand the sidebar with the expand button", async () => { + // The sidebar opens automatically, so collapse it first to make the + // expand button available. The open-finished class is toggled once + // the panel animation completes. + await page.waitForSelector(".leftPanel.open-finished", { + visible: true, + }); + await page.click(".leftPanel button.collapseButton"); + + // Collapsed: the panel content hides and the expand button appears. + await page.waitForFunction(() => { + const panel = document.querySelector(".leftPanel"); + return panel && !panel.classList.contains("open-finished"); + }); + await page.waitForSelector(".leftPanel button.expandButton", { + visible: true, + }); + await page.waitForSelector(".leftPanel .tabs", { hidden: true }); + + expect( + await page.$eval(".leftPanel button.expandButton", (btn) => + btn.getAttribute("aria-expanded") + ) + ).toBe("false"); + + await page.click(".leftPanel button.expandButton"); + + // Expanded again: the panel reopens and the thumbnails are visible. + await page.waitForSelector(".leftPanel.open-finished", { + visible: true, + }); + await page.waitForSelector(".thumb", { visible: true }); + + expect( + await page.$eval(".leftPanel", (el) => el.classList.contains("open")) + ).toBe(true); + expect( + await page.$eval(".leftPanel button.expandButton", (btn) => + btn.getAttribute("aria-expanded") + ) + ).toBe("true"); + }); + }); +}); diff --git a/scripts/testBaseUrl.js b/scripts/testBaseUrl.js index 402c12958..c4065a8ca 100644 --- a/scripts/testBaseUrl.js +++ b/scripts/testBaseUrl.js @@ -8,23 +8,19 @@ let BASE_URL = "http://localhost:4444"; // environment variable if (process.env.BASE_URL) { - BASE_URL = process.env.BASE_URL + BASE_URL = process.env.BASE_URL; } // local config file if (fs.existsSync(localConfigPath)) { try { - const localConfig = JSON.parse( - fs.readFileSync(localConfigPath, "utf8") - ); + const localConfig = JSON.parse(fs.readFileSync(localConfigPath, "utf8")); if (localConfig && localConfig.BASE_URL) { BASE_URL = localConfig.BASE_URL; - } + } } catch (err) { - console.warn( - "Invalid testBaseUrl.local.json, using default BASE_URL" - ); + console.warn("Invalid testBaseUrl.local.json, using default BASE_URL"); } } -module.exports = { BASE_URL }; \ No newline at end of file +module.exports = { BASE_URL }; diff --git a/src/content-handlers/iiif/modules/uv-mediaelementcenterpanel-module/MediaElementCenterPanel.ts b/src/content-handlers/iiif/modules/uv-mediaelementcenterpanel-module/MediaElementCenterPanel.ts index 6cea0dfb3..154c587c3 100644 --- a/src/content-handlers/iiif/modules/uv-mediaelementcenterpanel-module/MediaElementCenterPanel.ts +++ b/src/content-handlers/iiif/modules/uv-mediaelementcenterpanel-module/MediaElementCenterPanel.ts @@ -27,6 +27,17 @@ type TextTrackDescriptor = { id: string; }; +const captionTypes = new Set(["text/vtt", "text/srt"]); + +// A label in raw annotation JSON may be a plain string or a language map. +const captionLabel = (label: any): string | undefined => { + if (!label || typeof label === "string") { + return label || undefined; + } + const values = label[Object.keys(label)[0]]; + return Array.isArray(values) ? values[0] : undefined; +}; + type MediaSourceDescriptor = { label: string; type: string; @@ -206,7 +217,23 @@ export class MediaElementCenterPanel extends CenterPanel< } } + // Captions may also be supplied as supplementing annotations on the + // canvas (IIIF cookbook recipe 0219). + const supplementing = await this.getSupplementingCaptions(canvas); + for (const caption of supplementing) { + if (!subtitles.some((subtitle) => subtitle.id === caption.id)) { + subtitles.push(caption); + } + } + if (subtitles.length > 0) { + // Resolve caption URLs to ones the player's XHR will be able to read. + for (const subtitle of subtitles) { + if (subtitle.id) { + subtitle.id = await this.resolveCaptionSource(subtitle.id); + } + } + // Show captions options popover for better interface feedback subtitles.unshift({ id: "none" }); } @@ -392,6 +419,96 @@ export class MediaElementCenterPanel extends CenterPanel< this.extensionHost.publish(Events.LOAD); } + // Captions/transcriptions supplied as supplementing annotations in the + // canvas' annotations pages (IIIF cookbook recipe 0219). Inline + // annotation pages are read directly; pages referenced by id alone are + // fetched. + async getSupplementingCaptions( + canvas: Canvas + ): Promise { + const captions: TextTrackDescriptor[] = []; + const pages: any[] = canvas.getProperty("annotations") || []; + + for (const page of pages) { + let items: any[] = page.items; + + if (!items && page.id) { + try { + const response = await fetch(page.id); + if (response.ok) { + items = (await response.json()).items; + } + } catch { + console.warn( + `Annotation page ${page.id} could not be read (CORS headers are required); any captions it contains will be unavailable.` + ); + } + } + + if (!items) { + continue; + } + + for (const annotation of items) { + const motivations = Array.isArray(annotation.motivation) + ? annotation.motivation + : [annotation.motivation]; + + if (!motivations.includes("supplementing")) { + continue; + } + + const bodies = Array.isArray(annotation.body) + ? annotation.body + : [annotation.body]; + + for (const body of bodies) { + if (body && body.id && captionTypes.has(body.format)) { + captions.push({ + id: body.id, + label: captionLabel(body.label), + language: body.language, + }); + } + } + } + } + + return captions; + } + + // Captions are fetched with XHR by the player, so a cross-origin URL is + // only readable when every response hop carries CORS headers. Follow any + // CORS-friendly redirect to its final URL, and retry plain-http sources + // over https — an http -> https upgrade redirect whose 301 response lacks + // CORS headers is blocked by the browser even though its destination is + // readable. + async resolveCaptionSource(src: string): Promise { + const attempts: string[] = [src]; + + if (src.startsWith("http://")) { + attempts.push(src.replace(/^http:\/\//, "https://")); + } + + for (const attempt of attempts) { + try { + const response = await fetch(attempt); + if (response.ok) { + return response.url; + } + } catch { + // expected when the attempt is blocked by CORS or mixed content; + // fall through to the next candidate + } + } + + console.warn( + `Captions at ${src} could not be read (CORS headers are required on every response, including redirects); the player will omit this track.` + ); + + return src; + } + appendTextTracks(subtitles: Array) { for (const subtitle of subtitles) { this.$media.append( @@ -429,7 +546,7 @@ export class MediaElementCenterPanel extends CenterPanel< return typeGroup === "audio" || typeGroup === "video"; } - // vtt, srt, csv + // vtt, srt isTypeCaption(element: Rendering | AnnotationBody) { const type: RenderingFormat | MediaType | null = element.getFormat(); @@ -437,8 +554,6 @@ export class MediaElementCenterPanel extends CenterPanel< return false; } - const captionTypes = new Set(["text/vtt", "text/srt"]); - return captionTypes.has(type.toString()); } diff --git a/src/test-fixtures/captioned-video-manifest.json b/src/test-fixtures/captioned-video-manifest.json new file mode 100644 index 000000000..97c8b62ec --- /dev/null +++ b/src/test-fixtures/captioned-video-manifest.json @@ -0,0 +1,48 @@ +{ + "@context": "http://iiif.io/api/presentation/3/context.json", + "id": "http://localhost:4444/test-fixtures/captioned-video-manifest.json", + "type": "Manifest", + "label": { + "en": ["Video with captions (e2e test fixture)"] + }, + "items": [ + { + "id": "http://localhost:4444/test-fixtures/captioned-video-manifest.json/canvas", + "type": "Canvas", + "height": 360, + "width": 480, + "duration": 572.034, + "items": [ + { + "id": "http://localhost:4444/test-fixtures/captioned-video-manifest.json/canvas/page", + "type": "AnnotationPage", + "items": [ + { + "id": "http://localhost:4444/test-fixtures/captioned-video-manifest.json/canvas/page/annotation", + "type": "Annotation", + "motivation": "painting", + "target": "http://localhost:4444/test-fixtures/captioned-video-manifest.json/canvas", + "body": [ + { + "id": "https://fixtures.iiif.io/video/indiana/lunchroom_manners/high/lunchroom_manners_1024kb.mp4", + "type": "Video", + "height": 360, + "width": 480, + "duration": 572.034, + "format": "video/mp4" + }, + { + "id": "http://localhost:4444/test-fixtures/captions.vtt", + "type": "Text", + "format": "text/vtt", + "label": "English captions", + "language": "en" + } + ] + } + ] + } + ] + } + ] +} diff --git a/src/test-fixtures/captions.vtt b/src/test-fixtures/captions.vtt new file mode 100644 index 000000000..f0903de25 --- /dev/null +++ b/src/test-fixtures/captions.vtt @@ -0,0 +1,9 @@ +WEBVTT + +1 +00:00:00.000 --> 00:00:20.000 +Just before lunch one day, a puppet show was put on at school. + +2 +00:00:20.100 --> 00:00:40.000 +It was called "Mister Bungle Goes to Lunch". diff --git a/src/test-fixtures/cross-origin-captioned-video-manifest.json b/src/test-fixtures/cross-origin-captioned-video-manifest.json new file mode 100644 index 000000000..d9859b06e --- /dev/null +++ b/src/test-fixtures/cross-origin-captioned-video-manifest.json @@ -0,0 +1,48 @@ +{ + "@context": "http://iiif.io/api/presentation/3/context.json", + "id": "http://localhost:4444/test-fixtures/cross-origin-captioned-video-manifest.json", + "type": "Manifest", + "label": { + "en": ["Video with cross-origin captions (e2e test fixture)"] + }, + "items": [ + { + "id": "http://localhost:4444/test-fixtures/cross-origin-captioned-video-manifest.json/canvas", + "type": "Canvas", + "height": 360, + "width": 480, + "duration": 572.034, + "items": [ + { + "id": "http://localhost:4444/test-fixtures/cross-origin-captioned-video-manifest.json/canvas/page", + "type": "AnnotationPage", + "items": [ + { + "id": "http://localhost:4444/test-fixtures/cross-origin-captioned-video-manifest.json/canvas/page/annotation", + "type": "Annotation", + "motivation": "painting", + "target": "http://localhost:4444/test-fixtures/cross-origin-captioned-video-manifest.json/canvas", + "body": [ + { + "id": "https://fixtures.iiif.io/video/indiana/lunchroom_manners/high/lunchroom_manners_1024kb.mp4", + "type": "Video", + "height": 360, + "width": 480, + "duration": 572.034, + "format": "video/mp4" + }, + { + "id": "https://fixtures.iiif.io/video/indiana/lunchroom_manners/lunchroom_manners.vtt", + "type": "Text", + "format": "text/vtt", + "label": "English captions", + "language": "en" + } + ] + } + ] + } + ] + } + ] +} diff --git a/src/test-fixtures/redirected-captioned-video-manifest.json b/src/test-fixtures/redirected-captioned-video-manifest.json new file mode 100644 index 000000000..eecad2c41 --- /dev/null +++ b/src/test-fixtures/redirected-captioned-video-manifest.json @@ -0,0 +1,48 @@ +{ + "@context": "http://iiif.io/api/presentation/3/context.json", + "id": "http://localhost:4444/test-fixtures/redirected-captioned-video-manifest.json", + "type": "Manifest", + "label": { + "en": ["Video with captions behind a redirect (e2e test fixture)"] + }, + "items": [ + { + "id": "http://localhost:4444/test-fixtures/redirected-captioned-video-manifest.json/canvas", + "type": "Canvas", + "height": 360, + "width": 480, + "duration": 572.034, + "items": [ + { + "id": "http://localhost:4444/test-fixtures/redirected-captioned-video-manifest.json/canvas/page", + "type": "AnnotationPage", + "items": [ + { + "id": "http://localhost:4444/test-fixtures/redirected-captioned-video-manifest.json/canvas/page/annotation", + "type": "Annotation", + "motivation": "painting", + "target": "http://localhost:4444/test-fixtures/redirected-captioned-video-manifest.json/canvas", + "body": [ + { + "id": "https://fixtures.iiif.io/video/indiana/lunchroom_manners/high/lunchroom_manners_1024kb.mp4", + "type": "Video", + "height": 360, + "width": 480, + "duration": 572.034, + "format": "video/mp4" + }, + { + "id": "http://dlib.indiana.edu/iiif_av/lunchroom_manners/lunchroom_manners.vtt", + "type": "Text", + "format": "text/vtt", + "label": "English captions", + "language": "en" + } + ] + } + ] + } + ] + } + ] +} diff --git a/src/test-fixtures/supplementing-annotation-page.json b/src/test-fixtures/supplementing-annotation-page.json new file mode 100644 index 000000000..7b6a7ad2d --- /dev/null +++ b/src/test-fixtures/supplementing-annotation-page.json @@ -0,0 +1,22 @@ +{ + "@context": "http://iiif.io/api/presentation/3/context.json", + "id": "http://localhost:4444/test-fixtures/supplementing-annotation-page.json", + "type": "AnnotationPage", + "items": [ + { + "id": "http://localhost:4444/test-fixtures/supplementing-annotation-page.json/annotation", + "type": "Annotation", + "motivation": "supplementing", + "body": { + "id": "http://localhost:4444/test-fixtures/captions.vtt", + "type": "Text", + "format": "text/vtt", + "label": { + "en": ["English captions"] + }, + "language": "en" + }, + "target": "http://localhost:4444/test-fixtures/supplementing-external-captioned-video-manifest.json/canvas" + } + ] +} diff --git a/src/test-fixtures/supplementing-external-captioned-video-manifest.json b/src/test-fixtures/supplementing-external-captioned-video-manifest.json new file mode 100644 index 000000000..a4923a7b6 --- /dev/null +++ b/src/test-fixtures/supplementing-external-captioned-video-manifest.json @@ -0,0 +1,47 @@ +{ + "@context": "http://iiif.io/api/presentation/3/context.json", + "id": "http://localhost:4444/test-fixtures/supplementing-external-captioned-video-manifest.json", + "type": "Manifest", + "label": { + "en": [ + "Video with captions in an external annotation page (e2e test fixture)" + ] + }, + "items": [ + { + "id": "http://localhost:4444/test-fixtures/supplementing-external-captioned-video-manifest.json/canvas", + "type": "Canvas", + "height": 360, + "width": 480, + "duration": 572.034, + "items": [ + { + "id": "http://localhost:4444/test-fixtures/supplementing-external-captioned-video-manifest.json/canvas/page", + "type": "AnnotationPage", + "items": [ + { + "id": "http://localhost:4444/test-fixtures/supplementing-external-captioned-video-manifest.json/canvas/page/annotation", + "type": "Annotation", + "motivation": "painting", + "target": "http://localhost:4444/test-fixtures/supplementing-external-captioned-video-manifest.json/canvas", + "body": { + "id": "https://fixtures.iiif.io/video/indiana/lunchroom_manners/high/lunchroom_manners_1024kb.mp4", + "type": "Video", + "height": 360, + "width": 480, + "duration": 572.034, + "format": "video/mp4" + } + } + ] + } + ], + "annotations": [ + { + "id": "http://localhost:4444/test-fixtures/supplementing-annotation-page.json", + "type": "AnnotationPage" + } + ] + } + ] +} diff --git a/webpack.config.js b/webpack.config.js index 19a6606d5..a20c78922 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -136,6 +136,10 @@ const config = [ { from: resolvePath("./node_modules/mediaelement/build/mejs-controls.svg"), to: resolvePath("./dist"), + }, + { + from: resolvePath("./src/test-fixtures"), + to: resolvePath("./dist/test-fixtures"), } ], }),