fix: Show heatmap and image trace PNGs through blob URLs

draftlogs/8114_fix.md

@@ -0,0 +1 @@
+- Fix the growth of renderer memory in Chromium when a `heatmap` or `image` trace redraws many times, for example with live data [[#8114](https://github.com/plotly/plotly.js/pull/8114)], with thanks to @JeS24 for the contribution!

src/lib/image_href.ts

@@ -0,0 +1,90 @@
+'use strict';
+
+import { xlink } from '../constants/xmlns_namespaces';
+import helpers from '../snapshot/helpers';
+import type { GraphDiv } from '../types/core/graph-div.internal';
+
+interface ImageNode extends SVGImageElement {
+    _blobUrl?: string;
+    _dataUrl?: string;
+}
+
+function releaseBlobUrl(node: ImageNode) {
+    if (!node._blobUrl) return;
+    helpers.revokeObjectURL(node._blobUrl);
+    delete node._blobUrl;
+    delete node._dataUrl;
+}
+
+/**
+ * Set `xlink:href` on each selected `<image>` and revoke any blob URL previously assigned by this helper.
+ *
+ * Canvas sources use PNG blob URLs, with a stored data URL for SVG export.
+ * Export plots and empty canvases keep the canvas data URL.
+ * String sources use the supplied URL unchanged.
+ * The graph div tracks image nodes that need blob URL cleanup.
+ * Call `releaseImageHrefs` when images leave the plot or the plot is purged.
+ *
+ * @param gd - the graph div that owns the images
+ * @param image - a d3 selection of `<image>` elements
+ * @param source - a canvas to encode as a PNG, or a URL to use as is
+ */
+export function setImageHref(gd: GraphDiv, image: any, source: HTMLCanvasElement | string) {
+    const href = typeof source === 'string' ? source : source.toDataURL('image/png');
+    const useBlob = typeof source !== 'string' && !gd._context?._exportedPlot && helpers.IMAGE_URL_PREFIX.test(href);
+    const blob = useBlob ? helpers.createBlob(href.replace(helpers.IMAGE_URL_PREFIX, ''), 'png') : null;
+    const nodes: ImageNode[] = gd._imageBlobNodes || (gd._imageBlobNodes = []);
+
+    image.each(function (this: ImageNode) {
+        const previous = this._blobUrl;
+        if (blob) {
+            this._blobUrl = helpers.createObjectURL(blob);
+            this._dataUrl = href;
+            if (nodes.indexOf(this) === -1) nodes.push(this);
+        } else {
+            delete this._blobUrl;
+            delete this._dataUrl;
+        }
+
+        this.setAttributeNS(xlink, 'href', this._blobUrl || href);
+        if (previous) helpers.revokeObjectURL(previous);
+    });
+}
+
+/**
+ * Revoke tracked image blob URLs for the graph div.
+ * Clear the stored URLs and remove the released nodes from the graph div's tracking list.
+ *
+ * @param gd - the graph div that owns the images
+ * @param removedOnly - true to revoke only URLs for images no longer inside `gd`. Defaults to false (revoke all).
+ */
+export function releaseImageHrefs(gd: GraphDiv, removedOnly?: boolean) {
+    const nodes: ImageNode[] | undefined = gd._imageBlobNodes;
+    if (!nodes) return;
+
+    const kept = nodes.filter((node) => {
+        if (removedOnly && node._blobUrl && gd.contains(node)) return true;
+        releaseBlobUrl(node);
+        return false;
+    });
+
+    if (kept.length) gd._imageBlobNodes = kept;
+    else delete gd._imageBlobNodes;
+}
+
+/**
+ * Replace image blob URLs in serialized SVG with their stored PNG data URLs.
+ * The source SVG must retain the image nodes and their URL metadata from `setImageHref`.
+ *
+ * @param svgNode - the source SVG element
+ * @param svgString - the serialized SVG to update
+ * @returns the updated SVG string. Leaves the SVG DOM unchanged.
+ */
+export function restoreImageDataUrls(svgNode: SVGElement, svgString: string): string {
+    const images = svgNode.querySelectorAll('image') as NodeListOf<ImageNode>;
+    for (let i = 0; i < images.length; i++) {
+        const { _blobUrl, _dataUrl } = images[i];
+        if (_blobUrl && _dataUrl) svgString = svgString.replace(_blobUrl, () => _dataUrl);
+    }
+    return svgString;
+}

src/plot_api/subroutines.js

@@ -20,6 +20,8 @@ var enforceAxisConstraints = axisConstraints.enforce;
 var cleanAxisConstraints = axisConstraints.clean;
 var doAutoRange = require('../plots/cartesian/autorange').doAutoRange;
 
+const { releaseImageHrefs } = require('../lib/image_href');
+
 var SVG_TEXT_ANCHOR_START = 'start';
 var SVG_TEXT_ANCHOR_MIDDLE = 'middle';
 var SVG_TEXT_ANCHOR_END = 'end';
@@ -734,6 +736,9 @@ exports.drawData = function(gd) {
         basePlotModules[i].plot(gd);
     }
 
+    // Release blob URLs for image nodes removed during this redraw.
+    releaseImageHrefs(gd, true);
+
     exports.redrawReglTraces(gd);
 
     // styling separate from drawing

src/plots/plots.js

@@ -23,6 +23,8 @@ var frameAttrs = require('./frame_attributes');
 
 var getModuleCalcData = require('../plots/get_data').getModuleCalcData;
 
+const { releaseImageHrefs } = require('../lib/image_href');
+
 var relinkPrivateKeys = Lib.relinkPrivateKeys;
 var _ = Lib._;
 
@@ -1473,6 +1475,9 @@ plots.purge = function(gd) {
     // Cancel the pending redraw for web fonts
     delete gd._fontLoadToken;
 
+    // Release all image blob URLs before the graph div loses its plot state.
+    releaseImageHrefs(gd);
+
     // data and layout
     delete gd.data;
     delete gd.layout;

src/snapshot/tosvg.js

@@ -7,6 +7,9 @@ var Drawing = require('../components/drawing');
 var Color = require('../components/color');
 
 var xmlnsNamespaces = require('../constants/xmlns_namespaces');
+
+const { restoreImageDataUrls } = require('../lib/image_href');
+
 var DOUBLEQUOTE_REGEX = /"/g;
 var DUMMY_SUB = 'TOBESTRIPPED';
 // Match TOBESTRIPPED adjacent to either a literal " or its entity form ".
@@ -179,5 +182,8 @@ module.exports = function toSVG(gd, format, scale) {
     // Fix quotations around font strings and gradient URLs
     s = s.replace(DUMMY_REGEX, "'");
 
+    // Embed the image data so the exported SVG does not depend on temporary blob URLs.
+    s = restoreImageDataUrls(svg.node(), s);
+
     return s;
 };

src/traces/heatmap/plot.js

@@ -17,6 +17,8 @@ var LINE_SPACING = alignmentConstants.LINE_SPACING;
 var supportsPixelatedImage = require('../../lib/supports_pixelated_image');
 var PIXELATED_IMAGE_STYLE = require('../../constants/pixelated_image').STYLE;
 
+const { setImageHref } = require('../../lib/image_href');
+
 var labelClass = 'heatmap-label';
 
 function selectLabels(plotGroup) {
@@ -365,9 +367,9 @@ module.exports = function (gd, plotinfo, cdheatmaps, heatmapLayer) {
             height: imageHeight,
             width: imageWidth,
             x: left,
-            y: top,
-            'xlink:href': canvas.toDataURL('image/png')
+            y: top
         });
+        setImageHref(gd, image3, canvas);
 
         if (drawingMethod === 'fast' && !zsmooth) {
             image3.attr('style', PIXELATED_IMAGE_STYLE);

src/traces/image/plot.js

@@ -8,6 +8,8 @@ var constants = require('./constants');
 var supportsPixelatedImage = require('../../lib/supports_pixelated_image');
 var PIXELATED_IMAGE_STYLE = require('../../constants/pixelated_image').STYLE;
 
+const { setImageHref } = require('../../lib/image_href');
+
 module.exports = function plot(gd, plotinfo, cdimage, imageLayer) {
     var xa = plotinfo.xaxis;
     var ya = plotinfo.yaxis;
@@ -184,18 +186,15 @@ module.exports = function plot(gd, plotinfo, cdimage, imageLayer) {
             }
         })
         .then(function() {
-            var href, canvas;
+            let canvas;
             if(trace._hasZ) {
                 canvas = drawMagnifiedPixelsOnCanvas(function(i, j) {
                     var _z = z[j][i];
                     if(Lib.isTypedArray(_z)) _z = Array.from(_z);
                     return _z;
                 });
-                href = canvas.toDataURL('image/png');
             } else if(trace._hasSource) {
-                if(realImage) {
-                    href = trace.source;
-                } else {
+                if(!realImage) {
                     var context = trace._canvas.el.getContext('2d', {willReadFrequently: true});
                     var data = context.getImageData(0, 0, w, h).data;
                     canvas = drawMagnifiedPixelsOnCanvas(function(i, j) {
@@ -207,12 +206,11 @@ module.exports = function plot(gd, plotinfo, cdimage, imageLayer) {
                             data[index + 3]
                         ];
                     });
-                    href = canvas.toDataURL('image/png');
                 }
             }
 
+            setImageHref(gd, image3, realImage ? trace.source : canvas);
             image3.attr({
-                'xlink:href': href,
                 height: imageHeight,
                 width: imageWidth,
                 x: left,

src/types/core/graph-div.internal.d.ts

@@ -42,6 +42,8 @@ export interface GraphDiv extends HTMLDivElement {
     _fullLayout?: FullLayout;
     /** Most recent hover payloads (used to deduplicate hover events). */
     _hoverdata?: any[];
+    /** Image nodes with blob URLs to release after removal or on purge. */
+    _imageBlobNodes?: SVGImageElement[];
     /** True once `Plotly.plot`/`newPlot` has run at least once. */
     _initialized?: boolean;
     /** Outstanding async operations the next call must await. */

test/jasmine/tests/heatmap_test.js

@@ -726,7 +726,7 @@ describe('heatmap plot', function() {
         var mockCopy = Lib.extendDeep({}, mock);
 
         function getImageURL() {
-            return d3Select('.hm > image').attr('href');
+            return d3Select('.hm > image').node()._dataUrl;
         }
 
         var imageURLs = [];
@@ -755,6 +755,68 @@ describe('heatmap plot', function() {
         .then(done, done.fail);
     });
 
+    it('should show a blob URL and revoke the blob URLs that it no longer shows', function(done) {
+        const getImage = () => d3Select('.hm > image');
+        const revoked = [];
+        const revokeObjectURL = URL.revokeObjectURL;
+        spyOn(URL, 'revokeObjectURL').and.callFake((url) => {
+            revoked.push(url);
+            revokeObjectURL.call(URL, url);
+        });
+
+        const hrefs = [];
+        const expectRevoked = () => {
+            hrefs.forEach((href) => expect(revoked).toContain(href));
+        };
+
+        Plotly.newPlot(gd, [{ type: 'heatmap', z: [[1, 2], [3, 4]] }])
+            .then(() => {
+                hrefs.push(getImage().attr('href'));
+                expect(hrefs[0]).toMatch(/^blob:/);
+                expect(getImage().node()._dataUrl).toMatch(/^data:image\/png;base64,/);
+
+                return Plotly.restyle(gd, 'colorscale', 'Greens');
+            })
+            .then(() => {
+                const href = getImage().attr('href');
+                expect(href).toMatch(/^blob:/);
+                expect(revoked).toContain(hrefs[0]);
+                expect(revoked).not.toContain(href);
+                hrefs.push(href);
+
+                return Plotly.react(gd, [{ type: 'scatter', y: [1, 2] }]);
+            })
+            .then(() => {
+                expectRevoked();
+                expect(gd._imageBlobNodes).toBeUndefined();
+
+                return Plotly.react(gd, [{ type: 'heatmap', z: [[1, 2], [3, 4]] }]);
+            })
+            .then(() => {
+                hrefs.push(getImage().attr('href'));
+                Plotly.purge(gd);
+                expectRevoked();
+            })
+            .then(done, done.fail);
+    });
+
+    it('should put a data URL in an exported SVG', function(done) {
+        const expectDataUrl = (svg) => {
+            expect(svg).toContain('data:image/png;base64,');
+            expect(svg).not.toContain('blob:');
+        };
+
+        Plotly.newPlot(gd, [{ type: 'heatmap', z: [[1, 2], [3, 4]] }])
+            .then(() => Plotly.toImage(gd, { format: 'svg' }))
+            .then((url) => {
+                expectDataUrl(decodeURIComponent(url));
+
+                // Call toSVG last because it modifies the plot DOM.
+                expectDataUrl(Plotly.Snapshot.toSVG(gd));
+            })
+            .then(done, done.fail);
+    });
+
     it('draws canvas with correct margins', function(done) {
         var mockWithPadding = require('../../image/mocks/heatmap_brick_padding.json');
         var mockWithoutPadding = Lib.extendDeep({}, mockWithPadding);

test/jasmine/tests/image_test.js

@@ -286,7 +286,8 @@ describe('image plot', function() {
     });
 
     function getImageURL() {
-        return d3Select(sel).attr('href');
+        const node = d3Select(sel).node();
+        return node._dataUrl || d3Select(node).attr('href');
     }
 
     [