I'm using the Twilio library for video calling and have implemented functionality to add a virtual background behind the video track. Initially, I used OffscreenCanvas, but it's not supported in Safari. Therefore, I switched to the WebGL pipeline. However, I encountered an issue where the fitType property doesn't apply when using WebGL. This causes the background to appear stretched when applied from mobile devices. Is there a solution to ensure the background looks properly contained across all operating systems and browsers?
const initializeVirtualBackgroundProcessor = (
videoTrack: LocalVideoTrack | RemoteVideoTrack,
backgroundUrl: string
) => {
const img = new Image();
img.onload = () => {
const newProcessor = new VirtualBackgroundProcessor({
assetsPath: '/',
backgroundImage: img,
debounce: true,
pipeline: Pipeline.WebGL2,
fitType: ImageFit.Contain
});
newProcessor
.loadModel()
.then(() => {
if (virtualBackgroundProcessor) {
videoTrack.removeProcessor(virtualBackgroundProcessor);
}
videoTrack.addProcessor(newProcessor, {
inputFrameBufferType: 'video',
outputFrameBufferContextType: 'webgl2'
});
setVirtualBackgroundProcessor(newProcessor);
setVirtualBackgroundApplied(true);
})
.catch((error) => {
console.error(
'Error initializing virtual background processor:',
error
);
});
};
img.onerror = (error) => {
console.error('Error loading background image:', error);
// Handle error loading image, maybe fallback to default or show an error message
};
img.src = backgroundUrl;
img.crossOrigin = 'anonymous';
};