diff --git a/examples/index.ts b/examples/index.ts index cf455302..9ead0828 100644 --- a/examples/index.ts +++ b/examples/index.ts @@ -90,6 +90,8 @@ const defaultPhysicalPixelRatio = 1; const resolution = Number(urlParams.get('resolution')) || 720; const enableInspector = urlParams.get('inspector') === 'true'; const forceWebGL2 = urlParams.get('webgl2') === 'true'; + const textureProcessingLimit = + Number(urlParams.get('textureProcessingLimit')) || 0; const physicalPixelRatio = Number(urlParams.get('ppr')) || defaultPhysicalPixelRatio; @@ -114,6 +116,7 @@ const defaultPhysicalPixelRatio = 1; perfMultiplier, enableInspector, forceWebGL2, + textureProcessingLimit, ); return; } @@ -136,6 +139,7 @@ async function runTest( perfMultiplier: number, enableInspector: boolean, forceWebGL2: boolean, + textureProcessingLimit: number, ) { const testModule = testModules[getTestPath(test)]; if (!testModule) { @@ -157,6 +161,7 @@ async function runTest( physicalPixelRatio, enableInspector, forceWebGL2, + textureProcessingLimit, customSettings, ); @@ -170,9 +175,13 @@ async function runTest( parent: renderer.root, fontSize: 50, }); - overlayText.once( + overlayText.on( 'loaded', - (target: any, { dimensions }: NodeLoadedPayload) => { + (target: any, { type, dimensions }: NodeLoadedPayload) => { + if (type !== 'text') { + return; + } + overlayText.x = renderer.settings.appWidth - dimensions.width - 20; overlayText.y = renderer.settings.appHeight - dimensions.height - 20; }, @@ -227,6 +236,7 @@ async function initRenderer( physicalPixelRatio: number, enableInspector: boolean, forceWebGL2?: boolean, + textureProcessingLimit?: number, customSettings?: Partial, ) { let inspector: typeof Inspector | undefined; @@ -246,6 +256,7 @@ async function initRenderer( renderEngine: renderMode === 'webgl' ? WebGlCoreRenderer : CanvasCoreRenderer, fontEngines: [SdfTextRenderer, CanvasTextRenderer], + textureProcessingLimit: textureProcessingLimit, ...customSettings, }, 'app', @@ -425,7 +436,7 @@ async function runAutomation( // Allow some time for all images to load and the RaF to unpause // and render if needed. - await delay(200); + await new Promise((resolve) => setTimeout(resolve, 200)); if (snapshot) { console.log(`Calling snapshot(${testName})`); await snapshot(testName, adjustedOptions); @@ -454,6 +465,20 @@ async function runAutomation( } } -function delay(ms: number) { - return new Promise((resolve) => setTimeout(resolve, ms)); +function waitForRendererIdle(renderer: RendererMain) { + return new Promise((resolve) => { + let timeout: NodeJS.Timeout | undefined; + const startTimeout = () => { + timeout = setTimeout(() => { + resolve(); + }, 200); + }; + + renderer.once('idle', () => { + if (timeout) { + clearTimeout(timeout); + } + startTimeout(); + }); + }); } diff --git a/examples/tests/alpha.ts b/examples/tests/alpha.ts index e177025a..8f5ae06d 100644 --- a/examples/tests/alpha.ts +++ b/examples/tests/alpha.ts @@ -26,12 +26,6 @@ export async function automation(settings: ExampleSettings) { } export default async function test({ renderer, testRoot }: ExampleSettings) { - /* - * redRect will persist and change color every frame - * greenRect will persist and be detached and reattached to the root every second - * blueRect will be created and destroyed every 500 ms - */ - const parent = renderer.createNode({ x: 200, y: 240, @@ -55,8 +49,5 @@ export default async function test({ renderer, testRoot }: ExampleSettings) { alpha: 1, }); - /* - * End: Sprite Map Demo - */ console.log('ready!'); } diff --git a/examples/tests/stress-images.ts b/examples/tests/stress-images.ts new file mode 100644 index 00000000..9580a230 --- /dev/null +++ b/examples/tests/stress-images.ts @@ -0,0 +1,37 @@ +import type { ExampleSettings } from '../common/ExampleSettings.js'; + +export default async function ({ renderer, testRoot }: ExampleSettings) { + const screenWidth = 1920; + const screenHeight = 1080; + const totalImages = 1000; + + // Calculate the grid dimensions for square images + const gridSize = Math.ceil(Math.sqrt(totalImages)); // Approximate grid size + const imageSize = Math.floor( + Math.min(screenWidth / gridSize, screenHeight / gridSize), + ); // Square size + + // Create a root node for the grid + const gridNode = renderer.createNode({ + x: 0, + y: 0, + width: screenWidth, + height: screenHeight, + parent: testRoot, + }); + + // Create and position images in the grid + new Array(totalImages).fill(0).forEach((_, i) => { + const x = (i % gridSize) * imageSize; + const y = Math.floor(i / gridSize) * imageSize; + + renderer.createNode({ + parent: gridNode, + x, + y, + width: imageSize, + height: imageSize, + src: `https://picsum.photos/id/${i}/${imageSize}/${imageSize}`, // Random images + }); + }); +} diff --git a/examples/tests/stress-mix.ts b/examples/tests/stress-mix.ts new file mode 100644 index 00000000..efc483f0 --- /dev/null +++ b/examples/tests/stress-mix.ts @@ -0,0 +1,100 @@ +import type { INode, ITextNode } from '../../dist/exports/index.js'; +import type { ExampleSettings } from '../common/ExampleSettings.js'; + +export const Colors = { + Black: 0x000000ff, + Red: 0xff0000ff, + Green: 0x00ff00ff, + Blue: 0x0000ffff, + Magenta: 0xff00ffff, + Gray: 0x7f7f7fff, + White: 0xffffffff, +}; + +const textureType = ['Image', 'Color', 'Text', 'Gradient']; + +const gradients = [ + 'colorTl', + 'colorTr', + 'colorBl', + 'colorBr', + 'colorTop', + 'colorBottom', + 'colorLeft', + 'colorRight', + 'color', +]; + +export default async function ({ renderer, testRoot }: ExampleSettings) { + const screenWidth = 1920; + const screenHeight = 1080; + const totalImages = 1000; + + // Calculate the grid dimensions for square images + const gridSize = Math.ceil(Math.sqrt(totalImages)); // Approximate grid size + const imageSize = Math.floor( + Math.min(screenWidth / gridSize, screenHeight / gridSize), + ); // Square size + + // Create a root node for the grid + const gridNode = renderer.createNode({ + x: 0, + y: 0, + width: screenWidth, + height: screenHeight, + parent: testRoot, + }); + + // Create and position images in the grid + new Array(totalImages).fill(0).forEach((_, i) => { + const x = (i % gridSize) * imageSize; + const y = Math.floor(i / gridSize) * imageSize; + + // pick a random texture type + const texture = textureType[Math.floor(Math.random() * textureType.length)]; + + // pick a random color from Colors + const clr = + Object.values(Colors)[ + Math.floor(Math.random() * Object.keys(Colors).length) + ]; + + const node = { + parent: gridNode, + x, + y, + width: imageSize, + height: imageSize, + } as Partial | Partial; + + if (texture === 'Image') { + node.src = `https://picsum.photos/id/${i}/${imageSize}/${imageSize}`; + } else if (texture === 'Text') { + (node as Partial).text = `Text ${i}`; + (node as Partial).fontSize = 18; + node.color = clr; + } else if (texture === 'Gradient') { + const gradient = gradients[Math.floor(Math.random() * gradients.length)]; + // @ts-ignore + node[gradient] = clr; + + const secondGradient = + gradients[Math.floor(Math.random() * gradients.length)]; + const secondColor = + Object.values(Colors)[ + Math.floor(Math.random() * Object.keys(Colors).length) + ]; + + // @ts-ignore + node[secondGradient] = secondColor; + } else { + node.color = clr; + } + + if (texture === 'Text') { + renderer.createTextNode(node as ITextNode); + } else { + renderer.createNode(node); + } + }); +} diff --git a/examples/tests/stress-textures.ts b/examples/tests/stress-textures.ts new file mode 100644 index 00000000..133935b4 --- /dev/null +++ b/examples/tests/stress-textures.ts @@ -0,0 +1,53 @@ +import type { ExampleSettings } from '../common/ExampleSettings.js'; + +export const Colors = { + Black: 0x000000ff, + Red: 0xff0000ff, + Green: 0x00ff00ff, + Blue: 0x0000ffff, + Magenta: 0xff00ffff, + Gray: 0x7f7f7fff, + White: 0xffffffff, +}; + +export default async function ({ renderer, testRoot }: ExampleSettings) { + const screenWidth = 1920; + const screenHeight = 1080; + const totalImages = 1000; + + // Calculate the grid dimensions for square images + const gridSize = Math.ceil(Math.sqrt(totalImages)); // Approximate grid size + const imageSize = Math.floor( + Math.min(screenWidth / gridSize, screenHeight / gridSize), + ); // Square size + + // Create a root node for the grid + const gridNode = renderer.createNode({ + x: 0, + y: 0, + width: screenWidth, + height: screenHeight, + parent: testRoot, + }); + + // Create and position images in the grid + new Array(totalImages).fill(0).forEach((_, i) => { + const x = (i % gridSize) * imageSize; + const y = Math.floor(i / gridSize) * imageSize; + + // pick a random color from Colors + const clr = + Object.values(Colors)[ + Math.floor(Math.random() * Object.keys(Colors).length) + ]; + + renderer.createNode({ + parent: gridNode, + x, + y, + width: imageSize, + height: imageSize, + color: clr, + }); + }); +} diff --git a/examples/tests/text-mixed.ts b/examples/tests/text-mixed.ts new file mode 100644 index 00000000..684efde1 --- /dev/null +++ b/examples/tests/text-mixed.ts @@ -0,0 +1,108 @@ +import type { INode, ITextNode } from '../../dist/exports/index.js'; +import type { ExampleSettings } from '../common/ExampleSettings.js'; + +export async function automation(settings: ExampleSettings) { + await test(settings); + await settings.snapshot(); +} + +/** + * Tests that Single-Channel Signed Distance Field (SSDF) fonts are rendered + * correctly. + * + * Text that is thinner than the certified snapshot may indicate that the + * SSDF font atlas texture was premultiplied before being uploaded to the GPU. + * + * @param settings + * @returns + */ +export default async function test(settings: ExampleSettings) { + const { renderer, testRoot } = settings; + + let ssdf: ITextNode | undefined, + canvas: ITextNode | undefined, + factory: INode | undefined; + + const textFactory = () => { + const canvas = document.createElement('canvas'); + canvas.width = 300; + canvas.height = 200; + const ctx = canvas.getContext('2d'); + if (!ctx) throw new Error('Unable to create canvas 2d context'); + ctx.fillStyle = 'red'; + ctx.font = '50px sans-serif'; + ctx.fillText('Factory', 0, 50); + return ctx.getImageData(0, 0, 300, 200); + }; + + const drawText = (x = 0) => { + // Set a smaller snapshot area + ssdf = renderer.createTextNode({ + x, + text: 'SSDF', + color: 0x00ff00ff, + fontFamily: 'Ubuntu-ssdf', + parent: testRoot, + fontSize: 80, + lineHeight: 80 * 1.2, + }); + + canvas = renderer.createTextNode({ + x, + color: 0xff0000ff, + y: 100, + text: `Canvas`, + parent: testRoot, + fontSize: 50, + }); + + factory = renderer.createNode({ + x, + y: 150, + width: 300, + height: 200, + parent: testRoot, + texture: renderer.createTexture('ImageTexture', { + src: textFactory, + }), + }); + }; + + let offset = 0; + window.addEventListener('keydown', (e) => { + if (e.key === 'Enter') { + if (ssdf || canvas || factory) { + ssdf?.destroy(); + ssdf = undefined; + + canvas?.destroy(); + canvas = undefined; + + factory?.destroy(); + factory = undefined; + } + + setTimeout(() => { + drawText(); + }, 200); + } + + if (e.key === 'ArrowRight') { + offset += 10; + + if (ssdf) ssdf.x = offset; + if (canvas) canvas.x = offset; + if (factory) factory.x = offset; + } + + if (e.key === 'ArrowLeft') { + offset -= 10; + + if (ssdf) ssdf.x = offset; + if (canvas) canvas.x = offset; + if (factory) factory.x = offset; + } + }); + + drawText(); +} diff --git a/examples/tests/texture-factory.ts b/examples/tests/texture-factory.ts index 14e90b7d..5fd0a8b5 100644 --- a/examples/tests/texture-factory.ts +++ b/examples/tests/texture-factory.ts @@ -100,7 +100,7 @@ export default async function test({ renderer, testRoot }: ExampleSettings) { }); return new Promise((resolve, reject) => { - setTimeout(() => { + renderer.once('idle', () => { let result = ''; if ((setKey && factoryRuns === 1) || (!setKey && factoryRuns === 2)) { textNode.color = 0x00ff00ff; @@ -112,7 +112,7 @@ export default async function test({ renderer, testRoot }: ExampleSettings) { textNode.text += `: ${result}`; if (result === 'Pass') resolve(true); else reject({ setKey, factoryRuns }); - }, 50); + }); }); } diff --git a/examples/tests/texture-spritemap.ts b/examples/tests/texture-spritemap.ts new file mode 100644 index 00000000..59efdd38 --- /dev/null +++ b/examples/tests/texture-spritemap.ts @@ -0,0 +1,82 @@ +/* + * If not stated otherwise in this file or this component's LICENSE file the + * following copyright and licenses apply: + * + * Copyright 2023 Comcast Cable Communications Management, LLC. + * + * Licensed under the Apache License, Version 2.0 (the License); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { ExampleSettings } from '../common/ExampleSettings.js'; +import spritemap from '../assets/spritemap.png'; + +export async function automation(settings: ExampleSettings) { + // Snapshot single page + await test(settings); + await settings.snapshot(); +} + +export default async function test({ renderer, testRoot }: ExampleSettings) { + const FONT_SIZE = 45; + + renderer.createTextNode({ + text: `Texture Spritemap Test`, + fontSize: FONT_SIZE, + offsetY: -5, + parent: testRoot, + }); + + const spriteMapTexture = renderer.createTexture('ImageTexture', { + src: spritemap, + }); + + spriteMapTexture.on('load', (dimensions) => { + console.log('Spritemap Texture loaded', dimensions); + }); + + function execTest(y: number, x: number, title: string): Promise { + renderer.createTextNode({ + text: title, + fontSize: FONT_SIZE, + y: y, + parent: testRoot, + }); + + const character = renderer.createTexture('SubTexture', { + texture: spriteMapTexture, + x: x, + y: 0, + width: 100, + height: 150, + }); + + renderer.createNode({ + x: 20, + y: y + 80, + width: 100, + height: 150, + texture: character, + parent: testRoot, + }); + + return new Promise((resolve, reject) => { + renderer.once('idle', () => { + resolve(true); + }); + }); + } + + await execTest(80, 0, 'Character 1'); + await execTest(300, 100, 'Character 2'); + await execTest(520, 200, 'Character 3'); +} diff --git a/examples/tests/textures.ts b/examples/tests/textures.ts index 60009545..32bfc1e2 100644 --- a/examples/tests/textures.ts +++ b/examples/tests/textures.ts @@ -78,7 +78,7 @@ export default async function test({ renderer, testRoot }: ExampleSettings) { await execLoadingTest(elevator, 200, 268); - // Test: Check that we capture a texture load failure + // // Test: Check that we capture a texture load failure const failure = renderer.createNode({ x: curX, y: curY, @@ -88,7 +88,7 @@ export default async function test({ renderer, testRoot }: ExampleSettings) { await execFailureTest(failure); - // Test: Check that we capture a texture load failure + // // Test: Check that we capture a texture load failure const failure2 = renderer.createNode({ x: curX, y: curY, @@ -98,7 +98,7 @@ export default async function test({ renderer, testRoot }: ExampleSettings) { await execFailureTest(failure2); - // Test: NoiseTexture + // // Test: NoiseTexture curX = renderer.settings.appWidth / 2; curY = BEGIN_Y; diff --git a/package.json b/package.json index 54bc061a..2bc2fe31 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@lightningjs/renderer", - "version": "2.9.0-beta1", + "version": "2.9.0-beta4", "description": "Lightning 3 Renderer", "type": "module", "main": "./dist/exports/index.js", diff --git a/src/core/CoreNode.test.ts b/src/core/CoreNode.test.ts index 1d53881a..ce590a86 100644 --- a/src/core/CoreNode.test.ts +++ b/src/core/CoreNode.test.ts @@ -17,55 +17,73 @@ * limitations under the License. */ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { CoreNode, type CoreNodeProps, UpdateType } from './CoreNode.js'; import { Stage } from './Stage.js'; import { mock } from 'vitest-mock-extended'; import { type TextureOptions } from './CoreTextureManager.js'; import { type BaseShaderController } from '../main-api/ShaderController'; +import { createBound } from './lib/utils.js'; +import { ImageTexture } from './textures/ImageTexture.js'; -describe('set color()', () => { - const defaultProps: CoreNodeProps = { - alpha: 0, - autosize: false, - clipping: false, - color: 0, - colorBl: 0, - colorBottom: 0, - colorBr: 0, - colorLeft: 0, - colorRight: 0, - colorTl: 0, - colorTop: 0, - colorTr: 0, - height: 0, - mount: 0, - mountX: 0, - mountY: 0, - parent: null, - pivot: 0, - pivotX: 0, - pivotY: 0, - rotation: 0, - rtt: false, - scale: 0, - scaleX: 0, - scaleY: 0, - shader: mock(), - src: '', - texture: null, - textureOptions: {} as TextureOptions, - width: 0, - x: 0, - y: 0, - zIndex: 0, - zIndexLocked: 0, - preventCleanup: false, - strictBounds: false, - }; +const defaultProps: CoreNodeProps = { + alpha: 0, + autosize: false, + clipping: false, + color: 0, + colorBl: 0, + colorBottom: 0, + colorBr: 0, + colorLeft: 0, + colorRight: 0, + colorTl: 0, + colorTop: 0, + colorTr: 0, + height: 0, + mount: 0, + mountX: 0, + mountY: 0, + parent: null, + pivot: 0, + pivotX: 0, + pivotY: 0, + rotation: 0, + rtt: false, + scale: 0, + scaleX: 0, + scaleY: 0, + shader: mock(), + src: '', + texture: null, + textureOptions: {} as TextureOptions, + width: 0, + x: 0, + y: 0, + zIndex: 0, + zIndexLocked: 0, + preventCleanup: false, + strictBounds: false, +}; + +const clippingRect = { + x: 0, + y: 0, + width: 200, + height: 200, + valid: false, +}; + +const stage = mock({ + strictBound: createBound(0, 0, 200, 200), + preloadBound: createBound(0, 0, 200, 200), + defaultTexture: { + state: 'loaded', + }, +}); +describe('set color()', () => { it('should set all color subcomponents.', () => { - const node = new CoreNode(mock(), defaultProps); + const node = new CoreNode(stage, defaultProps); node.colorBl = 0x99aabbff; node.colorBr = 0xaabbccff; node.colorTl = 0xbbcceeff; @@ -85,7 +103,7 @@ describe('set color()', () => { }); it('should set update type.', () => { - const node = new CoreNode(mock(), defaultProps); + const node = new CoreNode(stage, defaultProps); node.updateType = 0; node.color = 0xffffffff; @@ -93,3 +111,89 @@ describe('set color()', () => { expect(node.updateType).toBe(UpdateType.PremultipliedColors); }); }); + +describe('isRenderable checks', () => { + it('should return false if node is not renderable', () => { + const node = new CoreNode(stage, defaultProps); + expect(node.isRenderable).toBe(false); + }); + + it('visible node that is a color texture', () => { + const node = new CoreNode(stage, defaultProps); + node.alpha = 1; + node.x = 0; + node.y = 0; + node.width = 100; + node.height = 100; + node.color = 0xffffffff; + + node.update(0, clippingRect); + expect(node.isRenderable).toBe(true); + }); + + it('visible node that is a texture', () => { + const node = new CoreNode(stage, defaultProps); + node.alpha = 1; + node.x = 0; + node.y = 0; + node.width = 100; + node.height = 100; + node.texture = mock({ + state: 'initial', + }); + + node.update(0, clippingRect); + expect(node.isRenderable).toBe(false); + + node.texture.state = 'loaded'; + node.setUpdateType(UpdateType.IsRenderable); + node.update(1, clippingRect); + + expect(node.isRenderable).toBe(true); + }); + + it('a node with a texture with alpha 0 should not be renderable', () => { + const node = new CoreNode(stage, defaultProps); + node.alpha = 0; + node.x = 0; + node.y = 0; + node.width = 100; + node.height = 100; + node.texture = mock({ + state: 'loaded', + }); + + node.update(0, clippingRect); + expect(node.isRenderable).toBe(false); + }); + + it('a node with a texture that is OutOfBounds should not be renderable', () => { + const node = new CoreNode(stage, defaultProps); + node.alpha = 1; + node.x = 300; + node.y = 300; + node.width = 100; + node.height = 100; + node.texture = mock({ + state: 'loaded', + }); + + node.update(0, clippingRect); + expect(node.isRenderable).toBe(false); + }); + + it('a node with a freed texture should not be renderable', () => { + const node = new CoreNode(stage, defaultProps); + node.alpha = 1; + node.x = 0; + node.y = 0; + node.width = 100; + node.height = 100; + node.texture = mock({ + state: 'freed', + }); + + node.update(0, clippingRect); + expect(node.isRenderable).toBe(false); + }); +}); diff --git a/src/core/CoreNode.ts b/src/core/CoreNode.ts index 90ec3843..c543f072 100644 --- a/src/core/CoreNode.ts +++ b/src/core/CoreNode.ts @@ -25,11 +25,11 @@ import { import type { TextureOptions } from './CoreTextureManager.js'; import type { CoreRenderer } from './renderers/CoreRenderer.js'; import type { Stage } from './Stage.js'; -import type { - Texture, - TextureFailedEventHandler, - TextureFreedEventHandler, - TextureLoadedEventHandler, +import { + type Texture, + type TextureFailedEventHandler, + type TextureFreedEventHandler, + type TextureLoadedEventHandler, } from './textures/Texture.js'; import type { Dimensions, @@ -767,6 +767,17 @@ export class CoreNode extends EventEmitter { UpdateType.RenderBounds | UpdateType.RenderState, ); + + // if the default texture isn't loaded yet, wait for it to load + // this only happens when the node is created before the stage is ready + if ( + this.stage.defaultTexture && + this.stage.defaultTexture.state !== 'loaded' + ) { + this.stage.defaultTexture.once('loaded', () => { + this.setUpdateType(UpdateType.IsRenderable); + }); + } } //#region Textures @@ -780,11 +791,6 @@ export class CoreNode extends EventEmitter { // synchronous task after calling loadTexture() queueMicrotask(() => { texture.preventCleanup = this.props.preventCleanup; - // Preload texture if required - if (this.textureOptions.preload) { - texture.ctxTexture.load(); - } - texture.on('loaded', this.onTextureLoaded); texture.on('failed', this.onTextureFailed); texture.on('freed', this.onTextureFreed); @@ -829,6 +835,7 @@ export class CoreNode extends EventEmitter { private onTextureLoaded: TextureLoadedEventHandler = (_, dimensions) => { this.autosizeNode(dimensions); + this.setUpdateType(UpdateType.IsRenderable); // Texture was loaded. In case the RAF loop has already stopped, we request // a render to ensure the texture is rendered. @@ -839,10 +846,13 @@ export class CoreNode extends EventEmitter { this.notifyParentRTTOfUpdate(); } - this.emit('loaded', { - type: 'texture', - dimensions, - } satisfies NodeTextureLoadedPayload); + // ignore 1x1 pixel textures + if (dimensions.width > 1 && dimensions.height > 1) { + this.emit('loaded', { + type: 'texture', + dimensions, + } satisfies NodeTextureLoadedPayload); + } // Trigger a local update if the texture is loaded and the resizeMode is 'contain' if (this.props.textureOptions?.resizeMode?.type === 'contain') { @@ -851,6 +861,8 @@ export class CoreNode extends EventEmitter { }; private onTextureFailed: TextureFailedEventHandler = (_, error) => { + this.setUpdateType(UpdateType.IsRenderable); + // If parent has a render texture, flag that we need to update if (this.parentHasRenderTexture) { this.notifyParentRTTOfUpdate(); @@ -863,6 +875,8 @@ export class CoreNode extends EventEmitter { }; private onTextureFreed: TextureFreedEventHandler = () => { + this.setUpdateType(UpdateType.IsRenderable); + // If parent has a render texture, flag that we need to update if (this.parentHasRenderTexture) { this.notifyParentRTTOfUpdate(); @@ -1000,7 +1014,7 @@ export class CoreNode extends EventEmitter { if (this.updateType & UpdateType.RenderTexture && this.rtt) { // Only the RTT node itself triggers `renderToTexture` this.hasRTTupdates = true; - this.stage.renderer?.renderToTexture(this); + this.loadRenderTexture(); } if (this.updateType & UpdateType.Global) { @@ -1205,64 +1219,6 @@ export class CoreNode extends EventEmitter { } } - //check if CoreNode is renderable based on props - hasRenderableProperties(): boolean { - if (this.props.texture) { - return true; - } - - if (!this.props.width || !this.props.height) { - return false; - } - - if (this.props.shader !== this.stage.defShaderCtr) { - return true; - } - - if (this.props.clipping) { - return true; - } - - if (this.props.color !== 0) { - return true; - } - - // Consider removing these checks and just using the color property check above. - // Maybe add a forceRender prop for nodes that should always render. - if (this.props.colorTop !== 0) { - return true; - } - - if (this.props.colorBottom !== 0) { - return true; - } - - if (this.props.colorLeft !== 0) { - return true; - } - - if (this.props.colorRight !== 0) { - return true; - } - - if (this.props.colorTl !== 0) { - return true; - } - - if (this.props.colorTr !== 0) { - return true; - } - - if (this.props.colorBl !== 0) { - return true; - } - - if (this.props.colorBr !== 0) { - return true; - } - return false; - } - checkRenderBounds(): CoreNodeRenderState { assertTruthy(this.renderBound); assertTruthy(this.strictBound); @@ -1392,27 +1348,102 @@ export class CoreNode extends EventEmitter { } /** - * This function updates the `isRenderable` property based on certain conditions. - * - * @returns + * Updates the `isRenderable` property based on various conditions. */ updateIsRenderable() { - let newIsRenderable; - if (this.worldAlpha === 0 || !this.hasRenderableProperties()) { - newIsRenderable = false; - } else { - newIsRenderable = this.renderState > CoreNodeRenderState.OutOfBounds; + let newIsRenderable = false; + let needsTextureOwnership = false; + + // If the node is out of bounds or has an alpha of 0, it is not renderable + if (this.checkBasicRenderability() === false) { + this.updateTextureOwnership(false); + this.setRenderable(false); + return; } - if (this.isRenderable !== newIsRenderable) { - this.isRenderable = newIsRenderable; - this.onChangeIsRenderable(newIsRenderable); + + if (this.texture !== null) { + needsTextureOwnership = true; + + // we're only renderable if the texture state is loaded + newIsRenderable = this.texture.state === 'loaded'; + } else if ( + (this.hasShader() || this.hasColorProperties() === true) && + this.hasDimensions() === true + ) { + // This mean we have dimensions and a color set, so we can render a ColorTexture + if ( + this.stage.defaultTexture && + this.stage.defaultTexture.state === 'loaded' + ) { + newIsRenderable = true; + } } + + this.updateTextureOwnership(needsTextureOwnership); + this.setRenderable(newIsRenderable); } - onChangeIsRenderable(isRenderable: boolean) { + /** + * Checks if the node is renderable based on world alpha, dimensions and out of bounds status. + */ + checkBasicRenderability(): boolean { + if (this.worldAlpha === 0 || this.isOutOfBounds() === true) { + return false; + } else { + return true; + } + } + + /** + * Sets the renderable state and triggers changes if necessary. + * @param isRenderable - The new renderable state + */ + setRenderable(isRenderable: boolean) { + this.isRenderable = isRenderable; + } + + /** + * Changes the renderable state of the node. + */ + updateTextureOwnership(isRenderable: boolean) { this.texture?.setRenderableOwner(this, isRenderable); } + /** + * Checks if the node is out of the viewport bounds. + */ + isOutOfBounds(): boolean { + return this.renderState <= CoreNodeRenderState.OutOfBounds; + } + + /** + * Checks if the node has dimensions (width/height) + */ + hasDimensions(): boolean { + return this.props.width !== 0 && this.props.height !== 0; + } + + /** + * Checks if the node has any color properties set. + */ + hasColorProperties(): boolean { + return ( + this.props.color !== 0 || + this.props.colorTop !== 0 || + this.props.colorBottom !== 0 || + this.props.colorLeft !== 0 || + this.props.colorRight !== 0 || + this.props.colorTl !== 0 || + this.props.colorTr !== 0 || + this.props.colorBl !== 0 || + this.props.colorBr !== 0 + ); + } + + hasShader(): boolean { + return this.props.shader !== null; + } + calculateRenderCoords() { const { width, height, globalTransform: transform } = this; assertTruthy(transform); @@ -1562,7 +1593,9 @@ export class CoreNode extends EventEmitter { colorTr: this.premultipliedColorTr, colorBl: this.premultipliedColorBl, colorBr: this.premultipliedColorBr, - texture: this.texture, + // if we do not have a texture, use the default texture + // this assumes any renderable node is either a distinct texture or a ColorTexture + texture: this.texture || this.stage.defaultTexture, textureOptions: this.textureOptions, zIndex: this.zIndex, shader: this.shader.shader, @@ -1643,11 +1676,11 @@ export class CoreNode extends EventEmitter { this.setUpdateType(UpdateType.Local); if (this.props.rtt) { - this.texture = this.stage.txManager.loadTexture('RenderTexture', { + this.texture = this.stage.txManager.createTexture('RenderTexture', { width: this.width, height: this.height, }); - this.textureOptions.preload = true; + this.setUpdateType(UpdateType.RenderTexture); } } @@ -1663,11 +1696,11 @@ export class CoreNode extends EventEmitter { this.setUpdateType(UpdateType.Local); if (this.props.rtt) { - this.texture = this.stage.txManager.loadTexture('RenderTexture', { + this.texture = this.stage.txManager.createTexture('RenderTexture', { width: this.width, height: this.height, }); - this.textureOptions.preload = true; + this.setUpdateType(UpdateType.RenderTexture); } } @@ -2025,12 +2058,31 @@ export class CoreNode extends EventEmitter { } } private initRenderTexture() { - this.texture = this.stage.txManager.loadTexture('RenderTexture', { + this.texture = this.stage.txManager.createTexture('RenderTexture', { width: this.width, height: this.height, }); - this.textureOptions.preload = true; - this.stage.renderer?.renderToTexture(this); // Only this RTT node + + this.loadRenderTexture(); + } + + private loadRenderTexture() { + if (this.texture === null) { + return; + } + + // If the texture is already loaded, render to it immediately + if (this.texture.state === 'loaded') { + this.stage.renderer?.renderToTexture(this); + return; + } + + // call load immediately to ensure the texture is created + this.stage.txManager.loadTexture(this.texture, true); + this.texture.once('loaded', () => { + this.stage.renderer?.renderToTexture(this); // Only this RTT node + this.setUpdateType(UpdateType.IsRenderable); + }); } private cleanupRenderTexture() { @@ -2110,7 +2162,7 @@ export class CoreNode extends EventEmitter { return; } - this.texture = this.stage.txManager.loadTexture('ImageTexture', { + this.texture = this.stage.txManager.createTexture('ImageTexture', { src: imageUrl, width: this.props.width, height: this.props.height, @@ -2201,16 +2253,19 @@ export class CoreNode extends EventEmitter { if (this.props.texture === value) { return; } + const oldTexture = this.props.texture; if (oldTexture) { oldTexture.setRenderableOwner(this, false); this.unloadTexture(); } + this.props.texture = value; - if (value) { - value.setRenderableOwner(this, this.isRenderable); + if (value !== null) { + value.setRenderableOwner(this, this.isRenderable); // WVB TODO: check if this is correct this.loadTexture(); } + this.setUpdateType(UpdateType.IsRenderable); } @@ -2241,12 +2296,12 @@ export class CoreNode extends EventEmitter { settings: Partial, ): IAnimationController { const animation = new CoreAnimation(this, props, settings); - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call + const controller = new CoreAnimationController( this.stage.animationManager, animation, ); - // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return controller; } diff --git a/src/core/CoreTextNode.ts b/src/core/CoreTextNode.ts index db6368a8..3c7f0a15 100644 --- a/src/core/CoreTextNode.ts +++ b/src/core/CoreTextNode.ts @@ -368,15 +368,20 @@ export class CoreTextNode extends CoreNode implements CoreTextNodeProps { this.textRenderer.set.y(this.trState, this.globalTransform.ty); } - override hasRenderableProperties(): boolean { + override checkBasicRenderability() { + if (this.worldAlpha === 0 || this.isOutOfBounds() === true) { + return false; + } + if (this.trState && this.trState.props.text !== '') { return true; } - return super.hasRenderableProperties(); + + return false; } - override onChangeIsRenderable(isRenderable: boolean) { - super.onChangeIsRenderable(isRenderable); + override setRenderable(isRenderable: boolean) { + super.setRenderable(isRenderable); this.textRenderer.setIsRenderable(this.trState, isRenderable); } diff --git a/src/core/CoreTextureManager.ts b/src/core/CoreTextureManager.ts index 1fda233f..31e6ba22 100644 --- a/src/core/CoreTextureManager.ts +++ b/src/core/CoreTextureManager.ts @@ -107,20 +107,6 @@ export type ResizeModeOptions = * multiple Nodes each using a different set of options. */ export interface TextureOptions { - /** - * Preload the texture immediately even if it's not being rendered to the - * screen. - * - * @remarks - * This allows the texture to be used immediately without any delay when it - * is first needed for rendering. Otherwise the loading process will start - * when the texture is first rendered, which may cause a delay in that texture - * being shown properly. - * - * @defaultValue `false` - */ - preload?: boolean; - /** * Flip the texture horizontally when rendering * @@ -161,6 +147,11 @@ export class CoreTextureManager extends EventEmitter { */ txConstructors: Partial = {}; + private downloadTextureSourceQueue: Array = []; + private priorityQueue: Array = []; + private uploadTextureQueue: Array = []; + private initialized = false; + imageWorkerManager: ImageWorkerManager | null = null; hasCreateImageBitmap = !!self.createImageBitmap; imageBitmapSupported = { @@ -219,6 +210,7 @@ export class CoreTextureManager extends EventEmitter { ); } + this.initialized = true; this.emit('initialized'); }) .catch((e) => { @@ -227,6 +219,7 @@ export class CoreTextureManager extends EventEmitter { ); // initialized without image worker manager and createImageBitmap + this.initialized = true; this.emit('initialized'); }); @@ -239,103 +232,37 @@ export class CoreTextureManager extends EventEmitter { private async validateCreateImageBitmap(): Promise { // Test if createImageBitmap is supported using a simple 1x1 PNG image - // prettier-ignore (this is a binary PNG image) + // prettier-ignore const pngBinaryData = new Uint8Array([ - 0x89, - 0x50, - 0x4e, - 0x47, - 0x0d, - 0x0a, - 0x1a, - 0x0a, // PNG signature - 0x00, - 0x00, - 0x00, - 0x0d, // IHDR chunk length - 0x49, - 0x48, - 0x44, - 0x52, // "IHDR" chunk type - 0x00, - 0x00, - 0x00, - 0x01, // Width: 1 - 0x00, - 0x00, - 0x00, - 0x01, // Height: 1 - 0x01, // Bit depth: 1 - 0x03, // Color type: Indexed - 0x00, // Compression method: Deflate - 0x00, // Filter method: None - 0x00, // Interlace method: None - 0x25, - 0xdb, - 0x56, - 0xca, // CRC for IHDR - 0x00, - 0x00, - 0x00, - 0x03, // PLTE chunk length - 0x50, - 0x4c, - 0x54, - 0x45, // "PLTE" chunk type - 0x00, - 0x00, - 0x00, // Palette entry: Black - 0xa7, - 0x7a, - 0x3d, - 0xda, // CRC for PLTE - 0x00, - 0x00, - 0x00, - 0x01, // tRNS chunk length - 0x74, - 0x52, - 0x4e, - 0x53, // "tRNS" chunk type - 0x00, // Transparency for black: Fully transparent - 0x40, - 0xe6, - 0xd8, - 0x66, // CRC for tRNS - 0x00, - 0x00, - 0x00, - 0x0a, // IDAT chunk length - 0x49, - 0x44, - 0x41, - 0x54, // "IDAT" chunk type - 0x08, - 0xd7, // Deflate header - 0x63, - 0x60, - 0x00, - 0x00, - 0x00, - 0x02, - 0x00, - 0x01, // Zlib-compressed data - 0xe2, - 0x21, - 0xbc, - 0x33, // CRC for IDAT - 0x00, - 0x00, - 0x00, - 0x00, // IEND chunk length - 0x49, - 0x45, - 0x4e, - 0x44, // "IEND" chunk type - 0xae, - 0x42, - 0x60, - 0x82, // CRC for IEND + 0x89, 0x50, 0x4e, 0x47, + 0x0d, 0x0a, 0x1a, 0x0a, // PNG signature + 0x00, 0x00, 0x00, 0x0d, // IHDR chunk length + 0x49, 0x48, 0x44, 0x52, // "IHDR" chunk type + 0x00, 0x00, 0x00, 0x01, // Width: 1 + 0x00, 0x00, 0x00, 0x01, // Height: 1 + 0x01, // Bit depth: 1 + 0x03, // Color type: Indexed + 0x00, // Compression method: Deflate + 0x00, // Filter method: None + 0x00, // Interlace method: None + 0x25, 0xdb, 0x56, 0xca, // CRC for IHDR + 0x00, 0x00, 0x00, 0x03, // PLTE chunk length + 0x50, 0x4c, 0x54, 0x45, // "PLTE" chunk type + 0x00, 0x00, 0x00, // Palette entry: Black + 0xa7, 0x7a, 0x3d, 0xda, // CRC for PLTE + 0x00, 0x00, 0x00, 0x01, // tRNS chunk length + 0x74, 0x52, 0x4e, 0x53, // "tRNS" chunk type + 0x00, // Transparency for black: Fully transparent + 0x40, 0xe6, 0xd8, 0x66, // CRC for tRNS + 0x00, 0x00, 0x00, 0x0a, // IDAT chunk length + 0x49, 0x44, 0x41, 0x54, // "IDAT" chunk type + 0x08, 0xd7, // Deflate header + 0x63, 0x60, 0x00, 0x00, + 0x00, 0x02, 0x00, 0x01, // Zlib-compressed data + 0xe2, 0x21, 0xbc, 0x33, // CRC for IDAT + 0x00, 0x00, 0x00, 0x00, // IEND chunk length + 0x49, 0x45, 0x4e, 0x44, // "IEND" chunk type + 0xae, 0x42, 0x60, 0x82, // CRC for IEND ]); const support: CreateImageBitmapSupport = { @@ -381,7 +308,33 @@ export class CoreTextureManager extends EventEmitter { this.txConstructors[textureType] = textureClass; } - loadTexture( + /** + * Enqueue a texture for downloading its source image. + */ + enqueueDownloadTextureSource(texture: Texture): void { + if (!this.downloadTextureSourceQueue.includes(texture)) { + this.downloadTextureSourceQueue.push(texture); + } + } + + /** + * Enqueue a texture for uploading to the GPU. + * + * @param texture - The texture to upload + */ + enqueueUploadTexture(texture: Texture): void { + if (this.uploadTextureQueue.includes(texture) === false) { + this.uploadTextureQueue.push(texture); + } + } + + /** + * Create a texture + * + * @param textureType - The type of texture to create + * @param props - The properties to use for the texture + */ + createTexture( textureType: Type, props: ExtractProps, ): InstanceType { @@ -391,29 +344,148 @@ export class CoreTextureManager extends EventEmitter { throw new Error(`Texture type "${textureType}" is not registered`); } - if (!texture) { - const cacheKey = TextureClass.makeCacheKey(props as any); - if (cacheKey && this.keyCache.has(cacheKey)) { - // console.log('Getting texture by cache key', cacheKey); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - texture = this.keyCache.get(cacheKey)!; - } else { - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-explicit-any - texture = new TextureClass(this, props as any); - if (cacheKey) { - this.initTextureToCache(texture, cacheKey); - } + const cacheKey = TextureClass.makeCacheKey(props as any); + if (cacheKey && this.keyCache.has(cacheKey)) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + texture = this.keyCache.get(cacheKey)!; + } else { + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-explicit-any + texture = new TextureClass(this, props as any); + + if (cacheKey) { + this.initTextureToCache(texture, cacheKey); } } + return texture as InstanceType; } - private initTextureToCache(texture: Texture, cacheKey: string) { + /** + * Override loadTexture to use the batched approach. + * + * @param texture - The texture to load + * @param immediate - Whether to prioritize the texture for immediate loading + */ + loadTexture(texture: Texture, priority?: boolean): void { + if (texture.state === 'loaded' || texture.state === 'loading') { + return; + } + + texture.setSourceState('loading'); + texture.setCoreCtxState('loading'); + + // if we're not initialized, just queue the texture into the priority queue + if (this.initialized === false) { + this.priorityQueue.push(texture); + return; + } + + // prioritize the texture for immediate loading + if (priority === true) { + texture + .getTextureData() + .then(() => { + this.uploadTexture(texture); + }) + .catch((err) => { + console.error(err); + }); + } + + // enqueue the texture for download and upload + this.enqueueDownloadTextureSource(texture); + } + + /** + * Upload a texture to the GPU + * + * @param texture Texture to upload + */ + uploadTexture(texture: Texture): void { + const coreContext = texture.loadCtxTexture(); + coreContext.load(); + } + + /** + * Process a limited number of downloads and uploads. + * + * @param maxItems - The maximum number of items to process + */ + processSome(maxItems = 0): void { + if (this.initialized === false) { + return; + } + + let itemsProcessed = 0; + + // Process priority queue + while ( + this.priorityQueue.length > 0 && + (maxItems === 0 || itemsProcessed < maxItems) + ) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const texture = this.priorityQueue.shift()!; + texture.getTextureData().then(() => { + this.uploadTexture(texture); + }); + itemsProcessed++; + } + + // Process uploads + while ( + this.uploadTextureQueue.length > 0 && + (maxItems === 0 || itemsProcessed < maxItems) + ) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + this.uploadTexture(this.uploadTextureQueue.shift()!); + itemsProcessed++; + } + + // Process downloads + while ( + this.downloadTextureSourceQueue.length > 0 && + (maxItems === 0 || itemsProcessed < maxItems) + ) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const texture = this.downloadTextureSourceQueue.shift()!; + queueMicrotask(() => { + texture.getTextureData().then(() => { + this.enqueueUploadTexture(texture); + }); + }); + + itemsProcessed++; + } + } + + public hasUpdates(): boolean { + return ( + this.downloadTextureSourceQueue.length > 0 || + this.uploadTextureQueue.length > 0 + ); + } + + /** + * Initialize a texture to the cache + * + * @param texture Texture to cache + * @param cacheKey Cache key for the texture + */ + initTextureToCache(texture: Texture, cacheKey: string) { const { keyCache, inverseKeyCache } = this; keyCache.set(cacheKey, texture); inverseKeyCache.set(texture, cacheKey); } + /** + * Get a texture from the cache + * + * @param cacheKey + */ + getTextureFromCache(cacheKey: string): Texture | undefined { + return this.keyCache.get(cacheKey); + } + /** * Remove a texture from the cache * @@ -429,4 +501,22 @@ export class CoreTextureManager extends EventEmitter { keyCache.delete(cacheKey); } } + + /** + * Resolve a parent texture from the cache or fallback to the provided texture. + * + * @param texture - The provided texture to resolve. + * @returns The cached or provided texture. + */ + resolveParentTexture(texture: ImageTexture): Texture { + if (!texture?.props) { + return texture; + } + + const cacheKey = ImageTexture.makeCacheKey(texture.props); + const cachedTexture = cacheKey + ? this.getTextureFromCache(cacheKey) + : undefined; + return cachedTexture ?? texture; + } } diff --git a/src/core/Stage.ts b/src/core/Stage.ts index ee72e858..6eb5693d 100644 --- a/src/core/Stage.ts +++ b/src/core/Stage.ts @@ -53,6 +53,8 @@ import { santizeCustomDataMap } from '../main-api/utils.js'; import type { SdfTextRenderer } from './text-rendering/renderers/SdfTextRenderer/SdfTextRenderer.js'; import type { CanvasTextRenderer } from './text-rendering/renderers/CanvasTextRenderer.js'; import { createBound, createPreloadBounds, type Bound } from './lib/utils.js'; +import type { Texture } from './textures/Texture.js'; +import { ColorTexture } from './textures/ColorTexture.js'; export interface StageOptions { appWidth: number; @@ -73,6 +75,7 @@ export interface StageOptions { fontEngines: (typeof CanvasTextRenderer | typeof SdfTextRenderer)[]; inspector: boolean; strictBounds: boolean; + textureProcessingLimit: number; } export type StageFpsUpdateHandler = ( @@ -103,6 +106,7 @@ export class Stage { public readonly strictBound: Bound; public readonly preloadBound: Bound; public readonly strictBounds: boolean; + public readonly defaultTexture: Texture | null = null; /** * Renderer Event Bus for the Stage to emit events onto @@ -148,6 +152,13 @@ export class Stage { this.eventBus = options.eventBus; this.txManager = new CoreTextureManager(numImageWorkers); + + // Wait for the Texture Manager to initialize + // once it does, request a render + this.txManager.on('initialized', () => { + this.requestRender(); + }); + this.txMemManager = new TextureMemoryManager(this, textureMemory); this.shManager = new CoreShaderManager(); this.animationManager = new AnimationManager(); @@ -183,6 +194,7 @@ export class Stage { this.renderer = new renderEngine(rendererOptions); const renderMode = this.renderer.mode || 'webgl'; + this.createDefaultTexture(); this.defShaderCtr = this.renderer.getDefShaderCtr(); setPremultiplyMode(renderMode); @@ -289,6 +301,33 @@ export class Stage { }); } + /** + * Create default PixelTexture + */ + createDefaultTexture() { + (this.defaultTexture as ColorTexture) = this.txManager.createTexture( + 'ColorTexture', + { + color: 0xffffffff, + }, + ); + + assertTruthy(this.defaultTexture instanceof ColorTexture); + + this.txManager.loadTexture(this.defaultTexture, true); + + // Mark the default texture as ALWAYS renderable + // This prevents it from ever being cleaned up. + // Fixes https://github.com/lightning-js/renderer/issues/262 + this.defaultTexture.setRenderableOwner(this, true); + + // When the default texture is loaded, request a render in case the + // RAF is paused. Fixes: https://github.com/lightning-js/renderer/issues/123 + this.defaultTexture.once('loaded', () => { + this.requestRender(); + }); + } + /** * Update animations */ @@ -305,7 +344,11 @@ export class Stage { * Check if the scene has updates */ hasSceneUpdates() { - return !!this.root.updateType || this.renderRequested; + return ( + !!this.root.updateType || + this.renderRequested || + this.txManager.hasUpdates() + ); } /** @@ -320,6 +363,10 @@ export class Stage { this.root.update(this.deltaTime, this.root.clippingRect); } + // Process some textures + // TODO this should have a configurable amount + this.txManager.processSome(this.options.textureProcessingLimit); + // Reset render operations and clear the canvas renderer.reset(); @@ -407,6 +454,7 @@ export class Stage { addQuads(node: CoreNode) { assertTruthy(this.renderer); + // If the node is renderable and has a loaded texture, render it if (node.isRenderable === true) { node.renderQuads(this.renderer); } diff --git a/src/core/TextureMemoryManager.ts b/src/core/TextureMemoryManager.ts index 625c8147..37ec652a 100644 --- a/src/core/TextureMemoryManager.ts +++ b/src/core/TextureMemoryManager.ts @@ -143,7 +143,6 @@ export class TextureMemoryManager { // If the threshold is 0, we disable the memory manager by replacing the // setTextureMemUse method with a no-op function. if (criticalThreshold === 0) { - // eslint-disable-next-line @typescript-eslint/no-empty-function this.setTextureMemUse = () => {}; } } @@ -228,7 +227,7 @@ export class TextureMemoryManager { break; } if (texture.preventCleanup === false) { - texture.ctxTexture.free(); + texture.free(); txManager.removeTextureFromCache(texture); } if (this.memUsed <= memTarget) { @@ -258,7 +257,6 @@ export class TextureMemoryManager { getMemoryInfo(): MemoryInfo { let renderableTexturesLoaded = 0; const renderableMemUsed = [...this.loadedTextures.keys()].reduce( - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion (acc, texture) => { renderableTexturesLoaded += texture.renderable ? 1 : 0; return ( diff --git a/src/core/lib/ImageWorker.ts b/src/core/lib/ImageWorker.ts index 6e8e73da..85d7d7f3 100644 --- a/src/core/lib/ImageWorker.ts +++ b/src/core/lib/ImageWorker.ts @@ -48,9 +48,6 @@ interface ImageWorkerMessage { /* eslint-disable */ function createImageWorker() { - var supportsOptionsCreateImageBitmap = false; - var supportsFullCreateImageBitmap = false; - function hasAlphaChannel(mimeType: string) { return mimeType.indexOf('image/png') !== -1; } @@ -62,8 +59,15 @@ function createImageWorker() { y: number | null, width: number | null, height: number | null, + options: { + supportsOptionsCreateImageBitmap: boolean; + supportsFullCreateImageBitmap: boolean; + }, ): Promise { return new Promise(function (resolve, reject) { + var supportsOptionsCreateImageBitmap = + options.supportsOptionsCreateImageBitmap; + var supportsFullCreateImageBitmap = options.supportsFullCreateImageBitmap; var xhr = new XMLHttpRequest(); xhr.open('GET', src, true); xhr.responseType = 'blob'; @@ -97,10 +101,7 @@ function createImageWorker() { reject(error); }); return; - } - - // createImageBitmap without crop but with options - if (supportsOptionsCreateImageBitmap === true) { + } else if (supportsOptionsCreateImageBitmap === true) { createImageBitmap(blob, { premultiplyAlpha: withAlphaChannel ? 'premultiply' : 'none', colorSpaceConversion: 'none', @@ -112,18 +113,17 @@ function createImageWorker() { .catch(function (error) { reject(error); }); - return; + } else { + // Fallback for browsers that do not support createImageBitmap with options + // this is supported for Chrome v50 to v52/54 that doesn't support options + createImageBitmap(blob) + .then(function (data) { + resolve({ data, premultiplyAlpha: premultiplyAlpha }); + }) + .catch(function (error) { + reject(error); + }); } - - // Fallback for browsers that do not support createImageBitmap with options - // this is supported for Chrome v50 to v52/54 that doesn't support options - createImageBitmap(blob) - .then(function (data) { - resolve({ data, premultiplyAlpha: premultiplyAlpha }); - }) - .catch(function (error) { - reject(error); - }); }; xhr.onerror = function () { @@ -145,7 +145,14 @@ function createImageWorker() { var width = event.data.sw; var height = event.data.sh; - getImage(src, premultiplyAlpha, x, y, width, height) + // these will be set to true if the browser supports the createImageBitmap options or full + var supportsOptionsCreateImageBitmap = false; + var supportsFullCreateImageBitmap = false; + + getImage(src, premultiplyAlpha, x, y, width, height, { + supportsOptionsCreateImageBitmap, + supportsFullCreateImageBitmap, + }) .then(function (data) { self.postMessage({ id: id, src: src, data: data }); }) @@ -197,32 +204,28 @@ export class ImageWorkerManager { let workerCode = `(${createImageWorker.toString()})()`; // Replace placeholders with actual initialization values - const supportsOptions = createImageBitmapSupport.options ? 'true' : 'false'; - const supportsFull = createImageBitmapSupport.full ? 'true' : 'false'; - workerCode = workerCode.replace( - 'var supportsOptionsCreateImageBitmap = false;', - `var supportsOptionsCreateImageBitmap = ${supportsOptions};`, - ); - workerCode = workerCode.replace( - 'var supportsFullCreateImageBitmap = false;', - `var supportsFullCreateImageBitmap = ${supportsFull};`, - ); + if (createImageBitmapSupport.options) { + workerCode = workerCode.replace( + 'var supportsOptionsCreateImageBitmap = false;', + 'var supportsOptionsCreateImageBitmap = true;', + ); + } - const blob: Blob = new Blob([workerCode.replace('"use strict";', '')], { + if (createImageBitmapSupport.full) { + workerCode = workerCode.replace( + 'var supportsFullCreateImageBitmap = false;', + 'var supportsFullCreateImageBitmap = true;', + ); + } + + workerCode = workerCode.replace('"use strict";', ''); + const blob: Blob = new Blob([workerCode], { type: 'application/javascript', }); const blobURL: string = (self.URL ? URL : webkitURL).createObjectURL(blob); const workers: Worker[] = []; for (let i = 0; i < numWorkers; i++) { - const worker = new Worker(blobURL); - - // Pass `createImageBitmap` support level during worker initialization - worker.postMessage({ - type: 'init', - support: createImageBitmapSupport, - }); - - workers.push(worker); + workers.push(new Worker(blobURL)); } return workers; } diff --git a/src/core/lib/WebGlContextWrapper.ts b/src/core/lib/WebGlContextWrapper.ts index dd1e00e5..99564ef0 100644 --- a/src/core/lib/WebGlContextWrapper.ts +++ b/src/core/lib/WebGlContextWrapper.ts @@ -68,6 +68,7 @@ export class WebGlContextWrapper { public readonly TEXTURE_WRAP_T; public readonly LINEAR; public readonly CLAMP_TO_EDGE; + public readonly RGB; public readonly RGBA; public readonly UNSIGNED_BYTE; public readonly UNPACK_PREMULTIPLY_ALPHA_WEBGL; @@ -158,6 +159,7 @@ export class WebGlContextWrapper { this.TEXTURE_WRAP_T = gl.TEXTURE_WRAP_T; this.LINEAR = gl.LINEAR; this.CLAMP_TO_EDGE = gl.CLAMP_TO_EDGE; + this.RGB = gl.RGB; this.RGBA = gl.RGBA; this.UNSIGNED_BYTE = gl.UNSIGNED_BYTE; this.UNPACK_PREMULTIPLY_ALPHA_WEBGL = gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL; @@ -1269,7 +1271,7 @@ export class WebGlContextWrapper { // prettier-ignore type IsUniformMethod = MethodName extends `uniform${string}` - ? // eslint-disable-next-line @typescript-eslint/no-explicit-any + ? MethodType extends (location: WebGLUniformLocation | null, ...args: any[]) => void ? true : false diff --git a/src/core/renderers/CoreContextTexture.ts b/src/core/renderers/CoreContextTexture.ts index 94ecde3a..d300844d 100644 --- a/src/core/renderers/CoreContextTexture.ts +++ b/src/core/renderers/CoreContextTexture.ts @@ -23,6 +23,7 @@ import type { Texture } from '../textures/Texture.js'; export abstract class CoreContextTexture { readonly textureSource: Texture; private memManager: TextureMemoryManager; + public state: 'freed' | 'loading' | 'loaded' | 'failed' = 'freed'; constructor(memManager: TextureMemoryManager, textureSource: Texture) { this.memManager = memManager; diff --git a/src/core/renderers/canvas/CanvasCoreRenderer.ts b/src/core/renderers/canvas/CanvasCoreRenderer.ts index d5bb799f..8d6f3a6f 100644 --- a/src/core/renderers/canvas/CanvasCoreRenderer.ts +++ b/src/core/renderers/canvas/CanvasCoreRenderer.ts @@ -22,7 +22,7 @@ import type { CoreNode } from '../../CoreNode.js'; import type { CoreShaderManager } from '../../CoreShaderManager.js'; import { getRgbaComponents, type RGBA } from '../../lib/utils.js'; import { SubTexture } from '../../textures/SubTexture.js'; -import type { Texture } from '../../textures/Texture.js'; +import { TextureType, type Texture } from '../../textures/Texture.js'; import type { CoreContextTexture } from '../CoreContextTexture.js'; import { CoreRenderer, @@ -43,6 +43,7 @@ import { type IParsedColor, } from './internal/ColorUtils.js'; import { UnsupportedShader } from './shaders/UnsupportedShader.js'; +import { assertTruthy } from '../../../utils.js'; export class CanvasCoreRenderer extends CoreRenderer { private context: CanvasRenderingContext2D; @@ -78,7 +79,6 @@ export class CanvasCoreRenderer extends CoreRenderer { } reset(): void { - // eslint-disable-next-line no-self-assign this.canvas.width = this.canvas.width; // quick reset canvas const ctx = this.context; @@ -119,6 +119,17 @@ export class CanvasCoreRenderer extends CoreRenderer { | { x: number; y: number; width: number; height: number } | undefined; + const textureType = texture?.type; + assertTruthy(textureType, 'Texture type is not defined'); + + // The Canvas2D renderer only supports image and color textures + if ( + textureType !== TextureType.image && + textureType !== TextureType.color + ) { + return; + } + if (texture) { if (texture instanceof SubTexture) { frame = texture.props; @@ -127,10 +138,9 @@ export class CanvasCoreRenderer extends CoreRenderer { ctxTexture = texture.ctxTexture as CanvasCoreTexture; if (texture.state === 'freed') { - ctxTexture.load(); return; } - if (texture.state !== 'loaded' || !ctxTexture.hasImage()) { + if (texture.state !== 'loaded') { return; } } @@ -175,7 +185,7 @@ export class CanvasCoreRenderer extends CoreRenderer { ctx.clip(path); } - if (ctxTexture) { + if (textureType === TextureType.image && ctxTexture) { const image = ctxTexture.getImage(color); ctx.globalAlpha = color.a ?? alpha; if (frame) { @@ -191,10 +201,15 @@ export class CanvasCoreRenderer extends CoreRenderer { height, ); } else { - ctx.drawImage(image, tx, ty, width, height); + try { + ctx.drawImage(image, tx, ty, width, height); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + } catch (error) { + // noop + } } ctx.globalAlpha = 1; - } else if (hasGradient) { + } else if (textureType === TextureType.color && hasGradient) { let endX: number = tx; let endY: number = ty; let endColor: IParsedColor; @@ -214,7 +229,7 @@ export class CanvasCoreRenderer extends CoreRenderer { gradient.addColorStop(1, formatRgba(endColor)); ctx.fillStyle = gradient; ctx.fillRect(tx, ty, width, height); - } else { + } else if (textureType === TextureType.color) { ctx.fillStyle = formatRgba(color); ctx.fillRect(tx, ty, width, height); } diff --git a/src/core/renderers/canvas/CanvasCoreTexture.ts b/src/core/renderers/canvas/CanvasCoreTexture.ts index 2831b678..19c41125 100644 --- a/src/core/renderers/canvas/CanvasCoreTexture.ts +++ b/src/core/renderers/canvas/CanvasCoreTexture.ts @@ -23,7 +23,11 @@ import { CoreContextTexture } from '../CoreContextTexture.js'; import { formatRgba, type IParsedColor } from './internal/ColorUtils.js'; export class CanvasCoreTexture extends CoreContextTexture { - protected image: ImageBitmap | HTMLCanvasElement | undefined; + protected image: + | ImageBitmap + | HTMLCanvasElement + | HTMLImageElement + | undefined; protected tintCache: | { key: string; @@ -32,24 +36,22 @@ export class CanvasCoreTexture extends CoreContextTexture { | undefined; load(): void { - if (this.textureSource.state !== 'freed') { - return; - } - this.textureSource.setState('loading'); + this.textureSource.setCoreCtxState('loading'); + this.onLoadRequest() .then((size) => { - this.textureSource.setState('loaded', size); + this.textureSource.setCoreCtxState('loaded', size); this.updateMemSize(); }) .catch((err) => { - this.textureSource.setState('failed', err as Error); + this.textureSource.setCoreCtxState('failed', err as Error); }); } free(): void { this.image = undefined; this.tintCache = undefined; - this.textureSource.setState('freed'); + this.textureSource.setCoreCtxState('freed'); this.setTextureMemUse(0); } @@ -68,7 +70,9 @@ export class CanvasCoreTexture extends CoreContextTexture { return this.image !== undefined; } - getImage(color: IParsedColor): ImageBitmap | HTMLCanvasElement { + getImage( + color: IParsedColor, + ): ImageBitmap | HTMLCanvasElement | HTMLImageElement { const image = this.image; assertTruthy(image, 'Attempt to get unloaded image texture'); @@ -94,7 +98,7 @@ export class CanvasCoreTexture extends CoreContextTexture { } protected tintTexture( - source: ImageBitmap | HTMLCanvasElement, + source: ImageBitmap | HTMLCanvasElement | HTMLImageElement, color: string, ) { const { width, height } = source; @@ -120,7 +124,9 @@ export class CanvasCoreTexture extends CoreContextTexture { } private async onLoadRequest(): Promise { - const { data } = await this.textureSource.getTextureData(); + assertTruthy(this.textureSource?.textureData?.data, 'Texture data is null'); + const { data } = this.textureSource.textureData; + // TODO: canvas from text renderer should be able to provide the canvas directly // instead of having to re-draw it into a new canvas... if (data instanceof ImageData) { @@ -132,12 +138,13 @@ export class CanvasCoreTexture extends CoreContextTexture { this.image = canvas; return { width: data.width, height: data.height }; } else if ( - typeof ImageBitmap !== 'undefined' && - data instanceof ImageBitmap + (typeof ImageBitmap !== 'undefined' && data instanceof ImageBitmap) || + data instanceof HTMLImageElement ) { this.image = data; return { width: data.width, height: data.height }; } + return { width: 0, height: 0 }; } } diff --git a/src/core/renderers/webgl/WebGlCoreCtxSubTexture.ts b/src/core/renderers/webgl/WebGlCoreCtxSubTexture.ts index a2dfb5cd..a632e90e 100644 --- a/src/core/renderers/webgl/WebGlCoreCtxSubTexture.ts +++ b/src/core/renderers/webgl/WebGlCoreCtxSubTexture.ts @@ -18,6 +18,7 @@ */ import type { Dimensions } from '../../../common/CommonTypes.js'; +import { assertTruthy } from '../../../utils.js'; import type { TextureMemoryManager } from '../../TextureMemoryManager.js'; import type { WebGlContextWrapper } from '../../lib/WebGlContextWrapper.js'; import type { SubTexture } from '../../textures/SubTexture.js'; @@ -33,7 +34,8 @@ export class WebGlCoreCtxSubTexture extends WebGlCoreCtxTexture { } override async onLoadRequest(): Promise { - const props = await (this.textureSource as SubTexture).getTextureData(); + const props = (this.textureSource as SubTexture).textureData; + assertTruthy(props, 'SubTexture must have texture data'); if (props.data instanceof Uint8Array) { // its a 1x1 Color Texture diff --git a/src/core/renderers/webgl/WebGlCoreCtxTexture.ts b/src/core/renderers/webgl/WebGlCoreCtxTexture.ts index 7308894e..fa9910dd 100644 --- a/src/core/renderers/webgl/WebGlCoreCtxTexture.ts +++ b/src/core/renderers/webgl/WebGlCoreCtxTexture.ts @@ -41,7 +41,6 @@ const TRANSPARENT_TEXTURE_DATA = new Uint8Array([0, 0, 0, 0]); */ export class WebGlCoreCtxTexture extends CoreContextTexture { protected _nativeCtxTexture: WebGLTexture | null = null; - private _state: 'freed' | 'loading' | 'loaded' | 'failed' = 'freed'; private _w = 0; private _h = 0; @@ -53,9 +52,10 @@ export class WebGlCoreCtxTexture extends CoreContextTexture { super(memManager, textureSource); } - get ctxTexture(): WebGLTexture { - if (this._state === 'freed') { + get ctxTexture(): WebGLTexture | null { + if (this.state === 'freed') { this.load(); + return null; } assertTruthy(this._nativeCtxTexture); return this._nativeCtxTexture; @@ -80,41 +80,45 @@ export class WebGlCoreCtxTexture extends CoreContextTexture { */ load() { // If the texture is already loading or loaded, don't load it again. - if (this._state === 'loading' || this._state === 'loaded') { + if (this.state === 'loading' || this.state === 'loaded') { return; } - this._state = 'loading'; - this.textureSource.setState('loading'); + + this.state = 'loading'; + this.textureSource.setCoreCtxState('loading'); this._nativeCtxTexture = this.createNativeCtxTexture(); + if (this._nativeCtxTexture === null) { - this._state = 'failed'; - this.textureSource.setState( + this.state = 'failed'; + this.textureSource.setCoreCtxState( 'failed', new Error('Could not create WebGL Texture'), ); console.error('Could not create WebGL Texture'); return; } + this.onLoadRequest() .then(({ width, height }) => { // If the texture has been freed while loading, return early. - if (this._state === 'freed') { + if (this.state === 'freed') { return; } - this._state = 'loaded'; + + this.state = 'loaded'; this._w = width; this._h = height; // Update the texture source's width and height so that it can be used // for rendering. - this.textureSource.setState('loaded', { width, height }); + this.textureSource.setCoreCtxState('loaded', { width, height }); }) .catch((err) => { // If the texture has been freed while loading, return early. - if (this._state === 'freed') { + if (this.state === 'freed') { return; } - this._state = 'failed'; - this.textureSource.setState('failed', err); + this.state = 'failed'; + this.textureSource.setCoreCtxState('failed', err); console.error(err); }); } @@ -124,48 +128,51 @@ export class WebGlCoreCtxTexture extends CoreContextTexture { */ async onLoadRequest(): Promise { const { glw } = this; + const textureData = this.textureSource.textureData; + assertTruthy(textureData, 'Texture data is null'); // Set to a 1x1 transparent texture glw.texImage2D(0, glw.RGBA, 1, 1, 0, glw.RGBA, glw.UNSIGNED_BYTE, null); this.setTextureMemUse(TRANSPARENT_TEXTURE_DATA.byteLength); - const textureData = await this.textureSource?.getTextureData(); // If the texture has been freed while loading, return early. if (!this._nativeCtxTexture) { - assertTruthy(this._state === 'freed'); + assertTruthy(this.state === 'freed'); return { width: 0, height: 0 }; } let width = 0; let height = 0; - assertTruthy(this._nativeCtxTexture); glw.activeTexture(0); + + const tdata = textureData.data; + const format = textureData.premultiplyAlpha ? glw.RGBA : glw.RGB; + const formatBytes = format === glw.RGBA ? 4 : 3; + // If textureData is null, the texture is empty (0, 0) and we don't need to // upload any data to the GPU. if ( - (typeof ImageBitmap !== 'undefined' && - textureData.data instanceof ImageBitmap) || - textureData.data instanceof ImageData || + (typeof ImageBitmap !== 'undefined' && tdata instanceof ImageBitmap) || + tdata instanceof ImageData || // not using typeof HTMLImageElement due to web worker - isHTMLImageElement(textureData.data) + isHTMLImageElement(tdata) ) { - const data = textureData.data; - width = data.width; - height = data.height; + width = tdata.width; + height = tdata.height; glw.bindTexture(this._nativeCtxTexture); glw.pixelStorei( glw.UNPACK_PREMULTIPLY_ALPHA_WEBGL, !!textureData.premultiplyAlpha, ); - glw.texImage2D(0, glw.RGBA, glw.RGBA, glw.UNSIGNED_BYTE, data); - this.setTextureMemUse(width * height * 4); + glw.texImage2D(0, format, format, glw.UNSIGNED_BYTE, tdata); + this.setTextureMemUse(width * height * formatBytes); // generate mipmaps for power-of-2 textures or in WebGL2RenderingContext if (glw.isWebGl2() || (isPowerOfTwo(width) && isPowerOfTwo(height))) { glw.generateMipmap(); } - } else if (textureData.data === null) { + } else if (tdata === null) { width = 0; height = 0; // Reset to a 1x1 transparent texture @@ -173,23 +180,17 @@ export class WebGlCoreCtxTexture extends CoreContextTexture { glw.texImage2D( 0, - glw.RGBA, + format, 1, 1, 0, - glw.RGBA, + format, glw.UNSIGNED_BYTE, TRANSPARENT_TEXTURE_DATA, ); this.setTextureMemUse(TRANSPARENT_TEXTURE_DATA.byteLength); - } else if ('mipmaps' in textureData.data && textureData.data.mipmaps) { - const { - mipmaps, - width = 0, - height = 0, - type, - glInternalFormat, - } = textureData.data; + } else if ('mipmaps' in tdata && tdata.mipmaps) { + const { mipmaps, width = 0, height = 0, type, glInternalFormat } = tdata; const view = type === 'ktx' ? new DataView(mipmaps[0] ?? new ArrayBuffer(0)) @@ -204,7 +205,7 @@ export class WebGlCoreCtxTexture extends CoreContextTexture { glw.texParameteri(glw.TEXTURE_MIN_FILTER, glw.LINEAR); this.setTextureMemUse(view.byteLength); - } else if (textureData.data && textureData.data instanceof Uint8Array) { + } else if (tdata && tdata instanceof Uint8Array) { // Color Texture width = 1; height = 1; @@ -217,16 +218,16 @@ export class WebGlCoreCtxTexture extends CoreContextTexture { glw.texImage2D( 0, - glw.RGBA, + format, width, height, 0, - glw.RGBA, + format, glw.UNSIGNED_BYTE, - textureData.data, + tdata, ); - this.setTextureMemUse(width * height * 4); + this.setTextureMemUse(width * height * formatBytes); } else { console.error( `WebGlCoreCtxTexture.onLoadRequest: Unexpected textureData returned`, @@ -246,11 +247,11 @@ export class WebGlCoreCtxTexture extends CoreContextTexture { * @returns */ free() { - if (this._state === 'freed') { + if (this.state === 'freed') { return; } - this._state = 'freed'; - this.textureSource.setState('freed'); + this.state = 'freed'; + this.textureSource.setCoreCtxState('freed'); this._w = 0; this._h = 0; if (!this._nativeCtxTexture) { diff --git a/src/core/renderers/webgl/WebGlCoreRenderer.ts b/src/core/renderers/webgl/WebGlCoreRenderer.ts index 4b505a34..816010c7 100644 --- a/src/core/renderers/webgl/WebGlCoreRenderer.ts +++ b/src/core/renderers/webgl/WebGlCoreRenderer.ts @@ -36,7 +36,6 @@ import { } from './internal/RendererUtils.js'; import { WebGlCoreCtxTexture } from './WebGlCoreCtxTexture.js'; import { Texture, TextureType } from '../../textures/Texture.js'; -import { ColorTexture } from '../../textures/ColorTexture.js'; import { SubTexture } from '../../textures/SubTexture.js'; import { WebGlCoreCtxSubTexture } from './WebGlCoreCtxSubTexture.js'; import { CoreShaderManager } from '../../CoreShaderManager.js'; @@ -53,7 +52,6 @@ import { RenderTexture } from '../../textures/RenderTexture.js'; import type { CoreNode } from '../../CoreNode.js'; import { WebGlCoreCtxRenderTexture } from './WebGlCoreCtxRenderTexture.js'; import type { BaseShaderController } from '../../../main-api/ShaderController.js'; -import { ImageTexture } from '../../textures/ImageTexture.js'; const WORDS_PER_QUAD = 24; // const BYTES_PER_QUAD = WORDS_PER_QUAD * 4; @@ -95,7 +93,6 @@ export class WebGlCoreRenderer extends CoreRenderer { /** * White pixel texture used by default when no texture is specified. */ - defaultTexture: Texture; quadBufferUsage = 0; /** @@ -114,19 +111,6 @@ export class WebGlCoreRenderer extends CoreRenderer { const { canvas, clearColor, bufferMemory } = options; - this.defaultTexture = new ColorTexture(this.txManager); - - // Mark the default texture as ALWAYS renderable - // This prevents it from ever being cleaned up. - // Fixes https://github.com/lightning-js/renderer/issues/262 - this.defaultTexture.setRenderableOwner(this, true); - - // When the default texture is loaded, request a render in case the - // RAF is paused. Fixes: https://github.com/lightning-js/renderer/issues/123 - this.defaultTexture.once('loaded', () => { - this.stage.requestRender(); - }); - const gl = createWebGLContext( canvas, options.forceWebGL2, @@ -237,7 +221,9 @@ export class WebGlCoreRenderer extends CoreRenderer { */ addQuad(params: QuadOptions) { const { fQuadBuffer, uiQuadBuffer } = this; - let texture = params.texture || this.defaultTexture; + let texture = params.texture; + + assertTruthy(texture !== null, 'Texture is required'); /** * If the shader props contain any automatic properties, update it with the @@ -255,8 +241,6 @@ export class WebGlCoreRenderer extends CoreRenderer { } } - assertTruthy(texture.ctxTexture !== undefined, 'Invalid texture type'); - let { curBufferIdx: bufferIdx, curRenderOp } = this; const targetDims = { width: -1, height: -1 }; targetDims.width = params.width; @@ -270,7 +254,6 @@ export class WebGlCoreRenderer extends CoreRenderer { ); if (this.reuseRenderOp(params) === false) { - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument this.newRenderOp( targetShader, params.shaderProps as Record, @@ -353,7 +336,7 @@ export class WebGlCoreRenderer extends CoreRenderer { } const ctxTexture = texture.ctxTexture as WebGlCoreCtxTexture; - assertTruthy(ctxTexture.ctxTexture !== undefined); + assertTruthy(ctxTexture instanceof WebGlCoreCtxTexture); const textureIdx = this.addTexture(ctxTexture, bufferIdx); assertTruthy(this.curRenderOp !== null); @@ -723,6 +706,11 @@ export class WebGlCoreRenderer extends CoreRenderer { continue; } + if (!node.texture || !node.texture.ctxTexture) { + console.warn('Texture not loaded for RTT node', node); + continue; + } + // Set the active RTT node to the current node // So we can prevent rendering children of nested RTT nodes this.activeRttNode = node; diff --git a/src/core/renderers/webgl/internal/RendererUtils.ts b/src/core/renderers/webgl/internal/RendererUtils.ts index 41dabcce..b74a656c 100644 --- a/src/core/renderers/webgl/internal/RendererUtils.ts +++ b/src/core/renderers/webgl/internal/RendererUtils.ts @@ -141,9 +141,11 @@ export function createIndexBuffer(glw: WebGlContextWrapper, size: number) { export function isHTMLImageElement(obj: unknown): obj is HTMLImageElement { return ( obj !== null && - typeof obj === 'object' && - obj.constructor && - obj.constructor.name === 'HTMLImageElement' + ((typeof obj === 'object' && + obj.constructor && + obj.constructor.name === 'HTMLImageElement') || + (typeof HTMLImageElement !== 'undefined' && + obj instanceof HTMLImageElement)) ); } diff --git a/src/core/renderers/webgl/shaders/effects/RadialGradientEffect.ts b/src/core/renderers/webgl/shaders/effects/RadialGradientEffect.ts index 5568080a..0850d324 100644 --- a/src/core/renderers/webgl/shaders/effects/RadialGradientEffect.ts +++ b/src/core/renderers/webgl/shaders/effects/RadialGradientEffect.ts @@ -64,12 +64,12 @@ export class RadialGradientEffect extends ShaderEffect { static override getEffectKey(props: RadialGradientEffectProps): string { if ((props.colors as unknown as ShaderEffectValueMap).value as number[]) { - return `linearGradient${ + return `radialGradient${ ((props.colors as unknown as ShaderEffectValueMap).value as number[]) .length }`; } - return `linearGradient${props.colors!.length}`; + return `radialGradient${props.colors!.length}`; } static override resolveDefaults( diff --git a/src/core/text-rendering/font-face-types/SdfTrFontFace/SdfTrFontFace.ts b/src/core/text-rendering/font-face-types/SdfTrFontFace/SdfTrFontFace.ts index 7cbd5200..7b145b8f 100644 --- a/src/core/text-rendering/font-face-types/SdfTrFontFace/SdfTrFontFace.ts +++ b/src/core/text-rendering/font-face-types/SdfTrFontFace/SdfTrFontFace.ts @@ -77,7 +77,7 @@ export class SdfTrFontFace< ); // Load image - this.texture = stage.txManager.loadTexture('ImageTexture', { + this.texture = stage.txManager.createTexture('ImageTexture', { src: atlasUrl, // IMPORTANT: The SDF shader requires the alpha channel to NOT be // premultiplied on the atlas texture. If it is premultiplied, the @@ -86,17 +86,22 @@ export class SdfTrFontFace< premultiplyAlpha: false, }); + // Load the texture + stage.txManager.loadTexture(this.texture, true); + + // FIXME This is a stop-gap solution to avoid Font Face textures to be cleaned up + // Ideally we do want to clean up the textures if they're not being used to save as much memory as possible + // However, we need to make sure that the font face is reloaded if the texture is cleaned up and needed again + // and make sure the SdfFontRenderer is properly guarded against textures being reloaded + // for now this will do the trick and the increase on memory is not that big + this.texture.preventCleanup = true; + this.texture.on('loaded', () => { this.checkLoaded(); // Make sure we mark the stage for a re-render (in case the font's texture was freed and reloaded) stage.requestRender(); }); - // Pre-load it - stage.txManager.once('initialized', () => { - this.texture.ctxTexture.load(); - }); - // Set this.data to the fetched data from dataUrl fetch(atlasDataUrl) .then(async (response) => { diff --git a/src/core/text-rendering/renderers/CanvasTextRenderer.ts b/src/core/text-rendering/renderers/CanvasTextRenderer.ts index 6b0a14fa..1d42c5f8 100644 --- a/src/core/text-rendering/renderers/CanvasTextRenderer.ts +++ b/src/core/text-rendering/renderers/CanvasTextRenderer.ts @@ -26,8 +26,7 @@ import { getNormalizedRgbaComponents, getNormalizedAlphaComponent, } from '../../lib/utils.js'; -import type { ImageTexture } from '../../textures/ImageTexture.js'; -import { TrFontManager, type FontFamilyMap } from '../TrFontManager.js'; +import { type FontFamilyMap } from '../TrFontManager.js'; import type { TrFontFace } from '../font-face-types/TrFontFace.js'; import { WebTrFontFace } from '../font-face-types/WebTrFontFace.js'; import { @@ -47,7 +46,7 @@ const resolvedGlobal = typeof self === 'undefined' ? globalThis : self; /** * Global font set regardless of if run in the main thread or a web worker */ -const globalFontSet = ((resolvedGlobal.document as any)?.fonts || +const globalFontSet: FontFaceSet = (resolvedGlobal.document?.fonts || (resolvedGlobal as any).fonts) as FontFaceSet; declare module './TextRenderer.js' { @@ -99,7 +98,7 @@ export class CanvasTextRenderer extends TextRenderer { } else { this.canvas = document.createElement('canvas'); } - // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion + let context = this.canvas.getContext('2d', { willReadFrequently: true, }) as OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D | null; @@ -338,7 +337,8 @@ export class CanvasTextRenderer extends TextRenderer { assertTruthy(state.renderInfo); const node = state.node; - const texture = this.stage.txManager.loadTexture('ImageTexture', { + const texture = this.stage.txManager.createTexture('ImageTexture', { + premultiplyAlpha: true, src: function ( this: CanvasTextRenderer, lightning2TextRenderer: LightningTextTextureRenderer, @@ -361,6 +361,7 @@ export class CanvasTextRenderer extends TextRenderer { ); }.bind(this, state.lightning2TextRenderer, state.renderInfo), }); + if (state.textureNode) { // Use the existing texture node state.textureNode.texture = texture; diff --git a/src/core/textures/ColorTexture.ts b/src/core/textures/ColorTexture.ts index 8e1bbfb4..21f926ee 100644 --- a/src/core/textures/ColorTexture.ts +++ b/src/core/textures/ColorTexture.ts @@ -62,7 +62,7 @@ export class ColorTexture extends Texture { this.props.color = color; } - override async getTextureData(): Promise { + override async getTextureSource(): Promise { const pixelData = new Uint8Array(4); if (this.color === 0xffffffff) { @@ -77,6 +77,8 @@ export class ColorTexture extends Texture { pixelData[3] = (this.color >>> 24) & 0xff; // Alpha } + this.setSourceState('loaded', { width: 1, height: 1 }); + return { data: pixelData, premultiplyAlpha: true, diff --git a/src/core/textures/ImageTexture.ts b/src/core/textures/ImageTexture.ts index 3196cca1..67a01de8 100644 --- a/src/core/textures/ImageTexture.ts +++ b/src/core/textures/ImageTexture.ts @@ -119,7 +119,7 @@ export interface ImageTextureProps { * {@link ImageTextureProps.premultiplyAlpha} prop to `false`. */ export class ImageTexture extends Texture { - props: Required; + public props: Required; public override type: TextureType = TextureType.image; @@ -135,6 +135,10 @@ export class ImageTexture extends Texture { async loadImageFallback(src: string, hasAlpha: boolean) { const img = new Image(); + if (!src.startsWith('data:')) { + img.crossOrigin = 'Anonymous'; + } + return new Promise<{ data: HTMLImageElement; premultiplyAlpha: boolean }>( (resolve) => { img.onload = () => { @@ -222,7 +226,48 @@ export class ImageTexture extends Texture { return this.loadImageFallback(src, premultiplyAlpha ?? true); } - override async getTextureData(): Promise { + override async getTextureSource(): Promise { + let resp; + try { + resp = await this.determineImageTypeAndLoadImage(); + } catch (e) { + this.setSourceState('failed', e as Error); + return { + data: null, + }; + } + + if (resp.data === null) { + this.setSourceState('failed', Error('ImageTexture: No image data')); + return { + data: null, + }; + } + + let width, height; + // check if resp.data is typeof Uint8ClampedArray else + // use resp.data.width and resp.data.height + if (resp.data instanceof Uint8Array) { + width = this.props.width ?? 0; + height = this.props.height ?? 0; + } else { + width = resp.data?.width ?? (this.props.width || 0); + height = resp.data?.height ?? (this.props.height || 0); + } + + // we're loaded! + this.setSourceState('loaded', { + width, + height, + }); + + return { + data: resp.data, + premultiplyAlpha: this.props.premultiplyAlpha ?? true, + }; + } + + determineImageTypeAndLoadImage() { const { src, premultiplyAlpha, type } = this.props; if (src === null) { return { diff --git a/src/core/textures/NoiseTexture.ts b/src/core/textures/NoiseTexture.ts index 0a7522ba..943bf6cc 100644 --- a/src/core/textures/NoiseTexture.ts +++ b/src/core/textures/NoiseTexture.ts @@ -63,7 +63,7 @@ export class NoiseTexture extends Texture { this.props = NoiseTexture.resolveDefaults(props); } - override async getTextureData(): Promise { + override async getTextureSource(): Promise { const { width, height } = this.props; const size = width * height * 4; const pixelData8 = new Uint8ClampedArray(size); @@ -74,6 +74,9 @@ export class NoiseTexture extends Texture { pixelData8[i + 2] = v; pixelData8[i + 3] = 255; } + + this.setSourceState('loaded'); + return { data: new ImageData(pixelData8, width, height), }; diff --git a/src/core/textures/RenderTexture.ts b/src/core/textures/RenderTexture.ts index 874fbc00..b3704b0b 100644 --- a/src/core/textures/RenderTexture.ts +++ b/src/core/textures/RenderTexture.ts @@ -63,7 +63,9 @@ export class RenderTexture extends Texture { this.props.height = value; } - override async getTextureData(): Promise { + override async getTextureSource(): Promise { + this.setSourceState('loaded'); + return { data: null, premultiplyAlpha: null, diff --git a/src/core/textures/SubTexture.ts b/src/core/textures/SubTexture.ts index 20eab7b0..c28ef98d 100644 --- a/src/core/textures/SubTexture.ts +++ b/src/core/textures/SubTexture.ts @@ -17,7 +17,9 @@ * limitations under the License. */ +import { assertTruthy } from '../../utils.js'; import type { CoreTextureManager } from '../CoreTextureManager.js'; +import { ImageTexture } from './ImageTexture.js'; import { Texture, TextureType, @@ -83,7 +85,21 @@ export class SubTexture extends Texture { constructor(txManager: CoreTextureManager, props: SubTextureProps) { super(txManager); this.props = SubTexture.resolveDefaults(props || {}); - this.parentTexture = this.props.texture; + + assertTruthy(this.props.texture, 'SubTexture requires a parent texture'); + assertTruthy( + this.props.texture instanceof ImageTexture, + 'SubTexture requires an ImageTexture parent', + ); + + // Resolve parent texture from cache or fallback to provided texture + this.parentTexture = txManager.resolveParentTexture(this.props.texture); + + if (this.parentTexture.state === 'freed') { + this.txManager.loadTexture(this.parentTexture); + } + + this.parentTexture.setRenderableOwner(this, true); // If parent texture is already loaded / failed, trigger loaded event manually // so that users get a consistent event experience. @@ -104,14 +120,22 @@ export class SubTexture extends Texture { private onParentTxLoaded: TextureLoadedEventHandler = () => { // We ignore the parent's passed dimensions, and simply use the SubTexture's // configured dimensions (because that's all that matters here) - this.setState('loaded', { + this.setSourceState('loaded', { width: this.props.width, height: this.props.height, }); + + // If the parent already has a ctxTexture, we can set the core ctx state + if (this.parentTexture.ctxTexture !== undefined) { + this.setCoreCtxState('loaded', { + width: this.props.width, + height: this.props.height, + }); + } }; private onParentTxFailed: TextureFailedEventHandler = (target, error) => { - this.setState('failed', error); + this.setSourceState('failed', error); }; override onChangeIsRenderable(isRenderable: boolean): void { @@ -119,7 +143,8 @@ export class SubTexture extends Texture { this.parentTexture.setRenderableOwner(this, isRenderable); } - override async getTextureData(): Promise { + override async getTextureSource(): Promise { + // Check if parent texture is loaded return { data: this.props, }; diff --git a/src/core/textures/Texture.ts b/src/core/textures/Texture.ts index 8a445372..627819af 100644 --- a/src/core/textures/Texture.ts +++ b/src/core/textures/Texture.ts @@ -101,7 +101,12 @@ export interface TextureData { premultiplyAlpha?: boolean | null; } -export type TextureState = 'freed' | 'loading' | 'loaded' | 'failed'; +export type TextureState = + | 'initial' + | 'freed' + | 'loading' + | 'loaded' + | 'failed'; export enum TextureType { 'generic' = 0, @@ -119,6 +124,8 @@ export interface TextureStateEventMap { failed: TextureFailedEventHandler; } +export type UpdateType = 'source' | 'coreCtx'; + /** * Like the built-in Parameters<> type but skips the first parameter (which is * `target` currently) @@ -152,7 +159,12 @@ export abstract class Texture extends EventEmitter { readonly error: Error | null = null; - readonly state: TextureState = 'freed'; + // aggregate state + public state: TextureState = 'initial'; + // texture source state + private sourceState: TextureState = 'initial'; + // texture (gpu) state + private coreCtxState: TextureState = 'initial'; readonly renderableOwners = new Set(); @@ -164,6 +176,10 @@ export abstract class Texture extends EventEmitter { public preventCleanup = false; + public ctxTexture: CoreContextTexture | undefined; + + public textureData: TextureData | null = null; + constructor(protected txManager: CoreTextureManager) { super(); } @@ -184,20 +200,28 @@ export abstract class Texture extends EventEmitter { */ setRenderableOwner(owner: unknown, renderable: boolean): void { const oldSize = this.renderableOwners.size; - if (renderable) { - this.renderableOwners.add(owner); + + if (renderable === true) { + if (this.renderableOwners.has(owner) === false) { + // Add the owner to the set + this.renderableOwners.add(owner); + } + const newSize = this.renderableOwners.size; if (newSize > oldSize && newSize === 1) { - // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion (this.renderable as boolean) = true; (this.lastRenderableChangeTime as number) = this.txManager.frameTime; this.onChangeIsRenderable?.(true); + + // Check if the texture needs to be added to the loading queue + if (this.state === 'freed' || this.state === 'initial') { + this.txManager.loadTexture(this); + } } } else { this.renderableOwners.delete(owner); const newSize = this.renderableOwners.size; if (newSize < oldSize && newSize === 0) { - // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion (this.renderable as boolean) = false; (this.lastRenderableChangeTime as number) = this.txManager.frameTime; this.onChangeIsRenderable?.(false); @@ -217,54 +241,120 @@ export abstract class Texture extends EventEmitter { onChangeIsRenderable?(isRenderable: boolean): void; /** - * Get the CoreContextTexture for this Texture + * Load the core context texture for this Texture. + * The ctxTexture is created by the renderer and lives on the GPU. + * + * @returns + */ + loadCtxTexture(): CoreContextTexture { + if (this.ctxTexture === undefined) { + this.ctxTexture = this.txManager.renderer.createCtxTexture(this); + } + + return this.ctxTexture; + } + + /** + * Free the core context texture for this Texture. * * @remarks - * Each Texture has a corresponding CoreContextTexture that is used to - * manage the texture's native data depending on the renderer's mode - * (WebGL, Canvas, etc). + * The ctxTexture is created by the renderer and lives on the GPU. + */ + free(): void { + this.ctxTexture?.free(); + if (this.textureData !== null) { + this.textureData = null; + this.setSourceState('freed'); + } + } + + private setState( + state: TextureState, + type: UpdateType, + errorOrDimensions?: Error | Dimensions, + ): void { + const stateObj = type === 'source' ? 'sourceState' : 'coreCtxState'; + + if (this[stateObj] === state) { + return; + } + + this[stateObj] = state; + + if (state === 'loaded') { + (this.dimensions as Dimensions) = errorOrDimensions as Dimensions; + } else if (state === 'failed') { + (this.error as Error) = errorOrDimensions as Error; + } + + this.updateState(); + } + + /** + * Set the source state of the texture * - * The Texture and CoreContextTexture are always linked together in a 1:1 - * relationship. + * @remarks + * The source of the texture can either be generated by the texture itself or + * loaded from an external source. + * + * @param state State of the texture + * @param errorOrDimensions Error or dimensions of the texture */ - get ctxTexture() { - // The first time this is called, create the ctxTexture - const ctxTexture = this.txManager.renderer.createCtxTexture(this); - // And replace this getter with the value for future calls - Object.defineProperty(this, 'ctxTexture', { value: ctxTexture }); - return ctxTexture; + public setSourceState( + state: TextureState, + errorOrDimensions?: Error | Dimensions, + ): void { + this.setState(state, 'source', errorOrDimensions); } /** - * Set the state of the texture + * Set the core context state of the texture * - * @remark - * Intended for internal-use only but declared public so that it can be set - * by it's associated {@link CoreContextTexture} + * @remarks + * The core context state of the texture is the state of the texture on the GPU. * - * @param state - * @param args + * @param state State of the texture + * @param errorOrDimensions Error or dimensions of the texture */ - setState( - state: State, - ...args: ParametersSkipTarget + public setCoreCtxState( + state: TextureState, + errorOrDimensions?: Error | Dimensions, ): void { - if (this.state !== state) { - // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion - (this.state as TextureState) = state; - if (state === 'loaded') { - const loadedArgs = args as ParametersSkipTarget< - TextureStateEventMap['loaded'] - >; - (this.dimensions as Dimensions) = loadedArgs[0]; - } else if (state === 'failed') { - const failedArgs = args as ParametersSkipTarget< - TextureStateEventMap['failed'] - >; - (this.error as Error) = failedArgs[0]; - } - this.emit(state, ...args); + this.setState(state, 'coreCtx', errorOrDimensions); + } + + private updateState(): void { + const ctxState = this.coreCtxState; + const sourceState = this.sourceState; + + let newState: TextureState = 'freed'; + let payload: Error | Dimensions | null = null; + + if (sourceState === 'failed' || ctxState === 'failed') { + newState = 'failed'; + payload = this.error; // Error set by the source + } else if (sourceState === 'loading' || ctxState === 'loading') { + newState = 'loading'; + } else if (sourceState === 'loaded' && ctxState === 'loaded') { + newState = 'loaded'; + payload = this.dimensions; // Dimensions set by the source + } else if ( + (sourceState === 'loaded' && ctxState === 'freed') || + (ctxState === 'loaded' && sourceState === 'freed') + ) { + // If one is loaded and the other is freed, then we are in a loading state + newState = 'loading'; + } else { + newState = 'freed'; } + + if (this.state === newState) { + return; + } + + // emit the new state + this.state = newState; + this.emit(newState, payload); } /** @@ -277,7 +367,22 @@ export abstract class Texture extends EventEmitter { * @returns * The texture data for this texture. */ - abstract getTextureData(): Promise; + async getTextureData(): Promise { + if (this.textureData === null) { + this.textureData = await this.getTextureSource(); + } + + return this.textureData; + } + + /** + * Get the texture source for this texture. + * + * @remarks + * This method is called by the CoreContextTexture when the texture is loaded. + * The texture source is then used to populate the CoreContextTexture. + */ + abstract getTextureSource(): Promise; /** * Make a cache key for this texture. diff --git a/src/main-api/Renderer.ts b/src/main-api/Renderer.ts index 0e557917..c94ab1ae 100644 --- a/src/main-api/Renderer.ts +++ b/src/main-api/Renderer.ts @@ -267,6 +267,26 @@ export interface RendererMainSettings { * @defaultValue `true` */ strictBounds?: boolean; + + /** + * Texture Processing Limit + * + * @remarks + * The maximum number of textures to process in a single frame. This is used to + * prevent the renderer from processing too many textures in a single frame. + * + * @defaultValue `0` + */ + textureProcessingLimit?: number; + + /** + * Canvas object to use for rendering + * + * @remarks + * This is used to render the scene graph. If not provided, a new canvas + * element will be created and appended to the target element. + */ + canvas?: HTMLCanvasElement; } /** @@ -358,6 +378,8 @@ export class RendererMain extends EventEmitter { quadBufferSize: settings.quadBufferSize ?? 4 * 1024 * 1024, fontEngines: settings.fontEngines, strictBounds: settings.strictBounds ?? true, + textureProcessingLimit: settings.textureProcessingLimit || 0, + canvas: settings.canvas || document.createElement('canvas'), }; this.settings = resolvedSettings; @@ -367,12 +389,12 @@ export class RendererMain extends EventEmitter { deviceLogicalPixelRatio, devicePhysicalPixelRatio, inspector, + canvas, } = resolvedSettings; const deviceLogicalWidth = appWidth * deviceLogicalPixelRatio; const deviceLogicalHeight = appHeight * deviceLogicalPixelRatio; - const canvas = document.createElement('canvas'); this.canvas = canvas; canvas.width = deviceLogicalWidth * devicePhysicalPixelRatio; canvas.height = deviceLogicalHeight * devicePhysicalPixelRatio; @@ -400,6 +422,7 @@ export class RendererMain extends EventEmitter { fontEngines: this.settings.fontEngines, inspector: this.settings.inspector !== null, strictBounds: this.settings.strictBounds, + textureProcessingLimit: this.settings.textureProcessingLimit, }); // Extract the root node @@ -516,7 +539,7 @@ export class RendererMain extends EventEmitter { textureType: TxType, props: ExtractProps, ): InstanceType { - return this.stage.txManager.loadTexture(textureType, props); + return this.stage.txManager.createTexture(textureType, props); } /** @@ -656,7 +679,7 @@ export class RendererMain extends EventEmitter { * May not do anything if the render loop is running on a separate worker. */ rerender() { - throw new Error('Not implemented'); + this.stage.requestRender(); } /** diff --git a/visual-regression/certified-snapshots/chromium-ci/text-mixed-1.png b/visual-regression/certified-snapshots/chromium-ci/text-mixed-1.png new file mode 100644 index 00000000..bd4b1f06 Binary files /dev/null and b/visual-regression/certified-snapshots/chromium-ci/text-mixed-1.png differ diff --git a/visual-regression/certified-snapshots/chromium-ci/texture-spritemap-1.png b/visual-regression/certified-snapshots/chromium-ci/texture-spritemap-1.png new file mode 100644 index 00000000..65403ec4 Binary files /dev/null and b/visual-regression/certified-snapshots/chromium-ci/texture-spritemap-1.png differ diff --git a/visual-regression/certified-snapshots/chromium-ci/texture-svg-1.png b/visual-regression/certified-snapshots/chromium-ci/texture-svg-1.png index da29d34e..1870fd88 100644 Binary files a/visual-regression/certified-snapshots/chromium-ci/texture-svg-1.png and b/visual-regression/certified-snapshots/chromium-ci/texture-svg-1.png differ