Add HTML-in-Canvas support with texElement2D API

examples/src/examples/misc/html-texture.example.mjs

@@ -0,0 +1,202 @@
+// @config DESCRIPTION This example demonstrates the HTML-in-Canvas proposal using texElement2D to render HTML content directly as a WebGL texture. Features graceful fallback to canvas rendering when not supported.
+import { deviceType, rootPath } from 'examples/utils';
+import * as pc from 'playcanvas';
+
+const canvas = /** @type {HTMLCanvasElement} */ (document.getElementById('application-canvas'));
+
+// Enable layoutsubtree for HTML-in-Canvas support
+canvas.setAttribute('layoutsubtree', '');
+// Alternative attribute names that might be used in different implementations
+canvas.setAttribute('data-layoutsubtree', '');
+canvas.style.contain = 'layout style paint';
+
+window.focus();
+
+const gfxOptions = {
+    deviceTypes: [deviceType],
+    glslangUrl: `${rootPath}/static/lib/glslang/glslang.js`,
+    twgslUrl: `${rootPath}/static/lib/twgsl/twgsl.js`
+};
+
+const device = await pc.createGraphicsDevice(canvas, gfxOptions);
+device.maxPixelRatio = Math.min(window.devicePixelRatio, 2);
+
+const createOptions = new pc.AppOptions();
+createOptions.graphicsDevice = device;
+
+createOptions.componentSystems = [pc.RenderComponentSystem, pc.CameraComponentSystem, pc.LightComponentSystem];
+createOptions.resourceHandlers = [pc.TextureHandler, pc.ContainerHandler];
+
+const app = new pc.AppBase(canvas);
+app.init(createOptions);
+app.start();
+
+// Set the canvas to fill the window and automatically change resolution to be the same as the canvas size
+app.setCanvasFillMode(pc.FILLMODE_FILL_WINDOW);
+app.setCanvasResolution(pc.RESOLUTION_AUTO);
+
+// Ensure canvas is resized when window changes size
+const resize = () => app.resizeCanvas();
+window.addEventListener('resize', resize);
+app.on('destroy', () => {
+    window.removeEventListener('resize', resize);
+});
+
+// Create an HTML element to use as texture source
+// According to the HTML-in-Canvas proposal, the element must be a direct child of the canvas
+const htmlElement = document.createElement('div');
+htmlElement.style.width = '512px';
+htmlElement.style.height = '512px';
+htmlElement.style.position = 'absolute';
+htmlElement.style.top = '0';
+htmlElement.style.left = '0';
+htmlElement.style.pointerEvents = 'none'; // Prevent interaction
+htmlElement.style.zIndex = '-1'; // Place behind canvas content
+htmlElement.style.backgroundColor = '#ff6b6b';
+htmlElement.style.background = 'linear-gradient(45deg, #ff6b6b, #4ecdc4, #45b7d1, #f9ca24)';
+htmlElement.style.borderRadius = '20px';
+htmlElement.style.padding = '20px';
+htmlElement.style.fontFamily = 'Arial, sans-serif';
+htmlElement.style.fontSize = '24px';
+htmlElement.style.color = 'white';
+htmlElement.style.textAlign = 'center';
+htmlElement.style.display = 'flex';
+htmlElement.style.flexDirection = 'column';
+htmlElement.style.justifyContent = 'center';
+htmlElement.style.alignItems = 'center';
+htmlElement.innerHTML = `
+    <h1 style="margin: 0 0 20px 0; text-shadow: 2px 2px 4px rgba(0,0,0,0.5);">HTML in Canvas!</h1>
+    <p style="margin: 0; text-shadow: 1px 1px 2px rgba(0,0,0,0.5);">This texture is rendered from HTML using texElement2D</p>
+    <div id="animated-element" style="margin-top: 20px; width: 50px; height: 50px; background: white; border-radius: 50%; animation: pulse 2s infinite;">
+    </div>
+`;
+
+// Add CSS animation
+const style = document.createElement('style');
+style.textContent = `
+    @keyframes pulse {
+        0% { transform: scale(1); opacity: 1; }
+        50% { transform: scale(1.2); opacity: 0.7; }
+        100% { transform: scale(1); opacity: 1; }
+    }
+`;
+document.head.appendChild(style);
+
+// Add the HTML element as a direct child of the canvas
+canvas.appendChild(htmlElement);
+
+// Create texture from HTML element
+const htmlTexture = new pc.Texture(device, {
+    width: 512,
+    height: 512,
+    format: pc.PIXELFORMAT_RGBA8,
+    name: 'htmlTexture'
+});
+
+// Helper function to create fallback canvas texture
+const createFallbackTexture = () => {
+    const fallbackCanvas = document.createElement('canvas');
+    fallbackCanvas.width = 512;
+    fallbackCanvas.height = 512;
+    const ctx = fallbackCanvas.getContext('2d');
+
+    if (!ctx) {
+        console.error('Failed to get 2D context');
+        return null;
+    }
+
+    const gradient = ctx.createLinearGradient(0, 0, 512, 512);
+    gradient.addColorStop(0, '#ff6b6b');
+    gradient.addColorStop(0.33, '#4ecdc4');
+    gradient.addColorStop(0.66, '#45b7d1');
+    gradient.addColorStop(1, '#f9ca24');
+    ctx.fillStyle = gradient;
+    ctx.fillRect(0, 0, 512, 512);
+
+    ctx.fillStyle = 'white';
+    ctx.font = 'bold 36px Arial';
+    ctx.textAlign = 'center';
+    ctx.shadowColor = 'rgba(0,0,0,0.5)';
+    ctx.shadowBlur = 4;
+    ctx.shadowOffsetX = 2;
+    ctx.shadowOffsetY = 2;
+    ctx.fillText('HTML in Canvas!', 256, 180);
+
+    ctx.font = '20px Arial';
+    ctx.fillText('(Canvas Fallback)', 256, 220);
+    ctx.fillText('texElement2D not available', 256, 260);
+
+    ctx.beginPath();
+    ctx.arc(256, 320, 25, 0, 2 * Math.PI);
+    ctx.fillStyle = 'white';
+    ctx.fill();
+
+    return fallbackCanvas;
+};
+
+// Set the HTML element as the texture source
+if (device.supportsTexElement2D) {
+    console.log('texElement2D is supported - attempting to use HTML element as texture');
+    try {
+        htmlTexture.setSource(/** @type {any} */ (htmlElement));
+        console.log('Successfully set HTML element as texture source');
+    } catch (error) {
+        console.warn('Failed to use texElement2D:', error.message);
+        console.log('Falling back to canvas rendering');
+
+        const fallbackCanvas = createFallbackTexture();
+        if (fallbackCanvas) {
+            htmlTexture.setSource(fallbackCanvas);
+        }
+    }
+} else {
+    console.warn('texElement2D is not supported - falling back to canvas rendering');
+    const fallbackCanvas = createFallbackTexture();
+    if (fallbackCanvas) {
+        htmlTexture.setSource(fallbackCanvas);
+    }
+}
+
+// Create material with the HTML texture
+const material = new pc.StandardMaterial();
+material.diffuseMap = htmlTexture;
+material.update();
+
+// create box entity
+const box = new pc.Entity('cube');
+box.addComponent('render', {
+    type: 'box',
+    material: material
+});
+app.root.addChild(box);
+
+// create camera entity
+const camera = new pc.Entity('camera');
+camera.addComponent('camera', {
+    clearColor: new pc.Color(0.1, 0.1, 0.1)
+});
+app.root.addChild(camera);
+camera.setPosition(0, 0, 3);
+
+// create directional light entity
+const light = new pc.Entity('light');
+light.addComponent('light');
+app.root.addChild(light);
+light.setEulerAngles(45, 0, 0);
+
+// Update the HTML texture periodically to capture animations
+let updateCounter = 0;
+app.on('update', (/** @type {number} */ dt) => {
+    box.rotate(10 * dt, 20 * dt, 30 * dt);
+
+    // Update texture every few frames to capture HTML animations
+    updateCounter += dt;
+    if (updateCounter > 0.1) { // Update 10 times per second
+        updateCounter = 0;
+        if (device.supportsTexElement2D) {
+            htmlTexture.upload();
+        }
+    }
+});
+
+export { app };

src/platform/graphics/graphics-device.js

@@ -840,17 +840,18 @@ class GraphicsDevice extends EventHandler {
     }
 
     /**
-     * Reports whether a texture source is a canvas, image, video or ImageBitmap.
+     * Reports whether a texture source is a canvas, image, video, ImageBitmap, or HTML element.
      *
      * @param {*} texture - Texture source data.
-     * @returns {boolean} True if the texture is a canvas, image, video or ImageBitmap and false
+     * @returns {boolean} True if the texture is a canvas, image, video, ImageBitmap, or HTML element and false
      * otherwise.
      * @ignore
      */
     _isBrowserInterface(texture) {
         return this._isImageBrowserInterface(texture) ||
                 this._isImageCanvasInterface(texture) ||
-                this._isImageVideoInterface(texture);
+                this._isImageVideoInterface(texture) ||
+                this._isHTMLElementInterface(texture);
     }
 
     _isImageBrowserInterface(texture) {
@@ -866,6 +867,13 @@ class GraphicsDevice extends EventHandler {
         return (typeof HTMLVideoElement !== 'undefined' && texture instanceof HTMLVideoElement);
     }
 
+    _isHTMLElementInterface(texture) {
+        return (typeof HTMLElement !== 'undefined' && texture instanceof HTMLElement &&
+                !(texture instanceof HTMLImageElement) &&
+                !(texture instanceof HTMLCanvasElement) &&
+                !(texture instanceof HTMLVideoElement));
+    }
+
     /**
      * Sets the width and height of the canvas, then fires the `resizecanvas` event. Note that the
      * specified width and height values will be multiplied by the value of

src/platform/graphics/texture.js

@@ -1008,11 +1008,11 @@ class Texture {
     }
 
     /**
-     * Set the pixel data of the texture from a canvas, image, video DOM element. If the texture is
-     * a cubemap, the supplied source must be an array of 6 canvases, images or videos.
+     * Set the pixel data of the texture from a canvas, image, video, or HTML DOM element. If the texture is
+     * a cubemap, the supplied source must be an array of 6 canvases, images, videos, or HTML elements.
      *
-     * @param {HTMLCanvasElement|HTMLImageElement|HTMLVideoElement|HTMLCanvasElement[]|HTMLImageElement[]|HTMLVideoElement[]} source - A
-     * canvas, image or video element, or an array of 6 canvas, image or video elements.
+     * @param {HTMLCanvasElement|HTMLImageElement|HTMLVideoElement|HTMLElement|HTMLCanvasElement[]|HTMLImageElement[]|HTMLVideoElement[]|HTMLElement[]} source - A
+     * canvas, image, video, or HTML element, or an array of 6 canvas, image, video, or HTML elements.
      * @param {number} [mipLevel] - A non-negative integer specifying the image level of detail.
      * Defaults to 0, which represents the base image source. A level value of N, that is greater
      * than 0, represents the image source for the Nth mipmap reduction level.
@@ -1066,7 +1066,13 @@ class Texture {
                 if (source instanceof HTMLVideoElement) {
                     width = source.videoWidth;
                     height = source.videoHeight;
+                } else if (this.device._isHTMLElementInterface(source)) {
+                    // For HTML elements, use getBoundingClientRect for dimensions
+                    const rect = source.getBoundingClientRect();
+                    width = Math.floor(rect.width) || 1;
+                    height = Math.floor(rect.height) || 1;
                 } else {
+                    // For canvas and image elements
                     width = source.width;
                     height = source.height;
                 }

src/platform/graphics/webgl/webgl-graphics-device.js

@@ -797,6 +797,10 @@ class WebglGraphicsDevice extends GraphicsDevice {
         this.extCompressedTextureATC = this.getExtension('WEBGL_compressed_texture_atc');
         this.extCompressedTextureASTC = this.getExtension('WEBGL_compressed_texture_astc');
         this.extTextureCompressionBPTC = this.getExtension('EXT_texture_compression_bptc');
+
+        // Check for HTML-in-Canvas support (texElement2D)
+        // This is a proposed API that may not be available yet
+        this.supportsTexElement2D = typeof gl.texElement2D === 'function';
     }
 
     /**

src/platform/graphics/webgl/webgl-texture.js

@@ -659,37 +659,20 @@ class WebglTexture {
             } else {
                 // ----- 2D -----
                 if (device._isBrowserInterface(mipObject)) {
-                    // Downsize images that are too large to be used as textures
-                    if (device._isImageBrowserInterface(mipObject)) {
-                        if (mipObject.width > device.maxTextureSize || mipObject.height > device.maxTextureSize) {
-                            mipObject = downsampleImage(mipObject, device.maxTextureSize);
-                            if (mipLevel === 0) {
-                                texture._width = mipObject.width;
-                                texture._height = mipObject.height;
-                            }
-                        }
-                    }
-
-                    const w = mipObject.width || mipObject.videoWidth;
-                    const h = mipObject.height || mipObject.videoHeight;
+                    // Handle HTML elements using texElement2D if supported
+                    if (device._isHTMLElementInterface(mipObject) && device.supportsTexElement2D) {
+                        // Use texElement2D for HTML elements
+                        device.setUnpackFlipY(texture._flipY);
+                        device.setUnpackPremultiplyAlpha(texture._premultiplyAlpha);
 
-                    // Upload the image, canvas or video
-                    device.setUnpackFlipY(texture._flipY);
-                    device.setUnpackPremultiplyAlpha(texture._premultiplyAlpha);
+                        // Get dimensions from the HTML element
+                        const rect = mipObject.getBoundingClientRect();
+                        const w = Math.floor(rect.width) || texture._width;
+                        const h = Math.floor(rect.height) || texture._height;
 
-                    // TEMP: disable fast path for video updates until
-                    // https://bugs.chromium.org/p/chromium/issues/detail?id=1511207 is resolved
-                    if (this._glCreated && texture._width === w && texture._height === h && !device._isImageVideoInterface(mipObject)) {
-                        gl.texSubImage2D(
-                            gl.TEXTURE_2D,
-                            mipLevel,
-                            0, 0,
-                            this._glFormat,
-                            this._glPixelType,
-                            mipObject
-                        );
-                    } else {
-                        gl.texImage2D(
+                        // texElement2D has a different signature than texImage2D
+                        // According to the proposal: texElement2D(target, level, internalformat, format, type, element)
+                        gl.texElement2D(
                             gl.TEXTURE_2D,
                             mipLevel,
                             this._glInternalFormat,
@@ -702,6 +685,51 @@ class WebglTexture {
                             texture._width = w;
                             texture._height = h;
                         }
+                    } else {
+                        // Downsize images that are too large to be used as textures
+                        if (device._isImageBrowserInterface(mipObject)) {
+                            if (mipObject.width > device.maxTextureSize || mipObject.height > device.maxTextureSize) {
+                                mipObject = downsampleImage(mipObject, device.maxTextureSize);
+                                if (mipLevel === 0) {
+                                    texture._width = mipObject.width;
+                                    texture._height = mipObject.height;
+                                }
+                            }
+                        }
+
+                        const w = mipObject.width || mipObject.videoWidth;
+                        const h = mipObject.height || mipObject.videoHeight;
+
+                        // Upload the image, canvas or video
+                        device.setUnpackFlipY(texture._flipY);
+                        device.setUnpackPremultiplyAlpha(texture._premultiplyAlpha);
+
+                        // TEMP: disable fast path for video updates until
+                        // https://bugs.chromium.org/p/chromium/issues/detail?id=1511207 is resolved
+                        if (this._glCreated && texture._width === w && texture._height === h && !device._isImageVideoInterface(mipObject)) {
+                            gl.texSubImage2D(
+                                gl.TEXTURE_2D,
+                                mipLevel,
+                                0, 0,
+                                this._glFormat,
+                                this._glPixelType,
+                                mipObject
+                            );
+                        } else {
+                            gl.texImage2D(
+                                gl.TEXTURE_2D,
+                                mipLevel,
+                                this._glInternalFormat,
+                                this._glFormat,
+                                this._glPixelType,
+                                mipObject
+                            );
+
+                            if (mipLevel === 0) {
+                                texture._width = w;
+                                texture._height = h;
+                            }
+                        }
                     }
                 } else {
                     // Upload the byte array