I'm using the technique described here (code, demo) for using video frames as WebGL textures, and the simple scene (just showing the image in 2D, rather than a 3D rotating cube) from here.
The goal is a Tampermonkey userscript (with WebGL shaders, i.e. video effects) for YouTube.
The canvas is filled grey due to gl.clearColor(0.5,0.5,0.5,1). But the next lines of code, which should draw the frame from the video, have no visible effect. What part might be wrong? There are no errors.
I tried to shorten the code before posting, but apparently even simple WebGL scenes require a lot of boilerplate code.
// ==UserScript==
// @name tmp
// @namespace /
// @version 0.1
// @description try to take over the world!
// @author You
// @match /*
// @icon
// @grant none
// ==/UserScript==
(function() {
// will set to true when video can be copied to texture
var copyVideo = false;
const video = document.getElementsByTagName("video")[0];
// immediately after finding the video, create canvas and set its dimensions
let canvas = document.createElement('canvas');
canvas.setAttribute('id', 'glcanvas');
canvas.setAttribute('width', '300');
canvas.setAttribute('height', '200');
canvas.setAttribute('style', 'position: absolute;');
video.parentElement.appendChild(canvas);
var playing = false;
var timeupdate = false;
// Waiting for these 2 events ensures
// there is data in the video
video.addEventListener('playing', function() {
playing = true;
checkReady();
}, true);
video.addEventListener('timeupdate', function() {
timeupdate = true;
checkReady();
}, true);
function checkReady() {
if (playing && timeupdate) {
copyVideo = true;
}
}
// Initialize the GL context
const gl = canvas.getContext("webgl");
// Only continue if WebGL is available and working
if (gl === null) {
alert("Unable to initialize WebGL. Your browser or machine may not support it.");
return;
}
// Vertex shader program
const vsSource = `
attribute vec2 a_position;
attribute vec2 a_texCoord;
uniform vec2 u_resolution;
varying vec2 v_texCoord;
void main() {
// convert the rectangle from pixels to 0.0 to 1.0
vec2 zeroToOne = a_position / u_resolution;
// convert from 0->1 to 0->2
vec2 zeroToTwo = zeroToOne * 2.0;
// convert from 0->2 to -1->+1 (clipspace)
vec2 clipSpace = zeroToTwo - 1.0;
gl_Position = vec4(clipSpace * vec2(1, -1), 0, 1);
// pass the texCoord to the fragment shader
// The GPU will interpolate this value between points.
v_texCoord = a_texCoord;
}
`;
// Fragment shader program
const fsSource = `
precision mediump float;
// our texture
uniform sampler2D u_image;
// the texCoords passed in from the vertex shader.
varying vec2 v_texCoord;
void main() {
gl_FragColor = texture2D(u_image, v_texCoord).bgra;
}
`;
// Initialize a shader program, so WebGL knows how to draw our data
function initShaderProgram(gl, vsSource, fsSource) {
const vertexShader = loadShader(gl, gl.VERTEX_SHADER, vsSource);
const fragmentShader = loadShader(gl, gl.FRAGMENT_SHADER, fsSource);
// Create the shader program
const shaderProgram = gl.createProgram();
gl.attachShader(shaderProgram, vertexShader);
gl.attachShader(shaderProgram, fragmentShader);
gl.linkProgram(shaderProgram);
// If creating the shader program failed, alert
if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {
alert('Unable to initialize the shader program: ' + gl.getProgramInfoLog(shaderProgram));
return null;
}
return shaderProgram;
}
// creates a shader of the given type, uploads the source and piles it.
function loadShader(gl, type, source) {
const shader = gl.createShader(type);
// Send the source to the shader object
gl.shaderSource(shader, source);
// Compile the shader program
glpileShader(shader);
// See if it piled successfully
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
alert('An error occurred piling the shaders: ' + gl.getShaderInfoLog(shader));
gl.deleteShader(shader);
return null;
}
return shader;
}
// Initialize a shader program; this is where all the lighting
// for the vertices and so forth is established.
const shaderProgram = initShaderProgram(gl, vsSource, fsSource);
// look up where the vertex data needs to go.
var positionLocation = gl.getAttribLocation(shaderProgram, "a_position");
var texcoordLocation = gl.getAttribLocation(shaderProgram, "a_texCoord");
// Create a buffer to put three 2d clip space points in
var positionBuffer = gl.createBuffer();
// Bind it to ARRAY_BUFFER (think of it as ARRAY_BUFFER = positionBuffer)
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
// Set a rectangle the same size as the image.
setRectangle(gl, 0, 0, video.width, video.height);
// provide texture coordinates for the rectangle.
var texcoordBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer);
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
0.0, 0.0,
1.0, 0.0,
0.0, 1.0,
0.0, 1.0,
1.0, 0.0,
1.0, 1.0,
]), gl.STATIC_DRAW);
// Create a texture.
var texture = initTexture(gl);
function drawScene() {
// lookup uniforms
var resolutionLocation = gl.getUniformLocation(shaderProgram, "u_resolution");
//webglUtils.resizeCanvasToDisplaySize(gl.canvas);
// Tell WebGL how to convert from clip space to pixels
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
// Clear the canvas
gl.clearColor(0.5,0.5,0.5,1);
gl.clear(gl.COLOR_BUFFER_BIT);
// Tell it to use our program (pair of shaders)
gl.useProgram(shaderProgram);
// Turn on the position attribute
gl.enableVertexAttribArray(positionLocation);
// Bind the position buffer.
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
// Tell the position attribute how to get data out of positionBuffer (ARRAY_BUFFER)
var size = 2; // 2 ponents per iteration
var type = gl.FLOAT; // the data is 32bit floats
var normalize = false; // don't normalize the data
var stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
var offset = 0; // start at the beginning of the buffer
gl.vertexAttribPointer(
positionLocation, size, type, normalize, stride, offset);
// Turn on the texcoord attribute
gl.enableVertexAttribArray(texcoordLocation);
// bind the texcoord buffer.
gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer);
// Tell the texcoord attribute how to get data out of texcoordBuffer (ARRAY_BUFFER)
var size = 2; // 2 ponents per iteration
var type = gl.FLOAT; // the data is 32bit floats
var normalize = false; // don't normalize the data
var stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
var offset = 0; // start at the beginning of the buffer
gl.vertexAttribPointer(
texcoordLocation, size, type, normalize, stride, offset);
// set the resolution
gl.uniform2f(resolutionLocation, gl.canvas.width, gl.canvas.height);
// Draw the rectangle.
var primitiveType = gl.TRIANGLES;
var offset = 0;
var count = 6;
gl.drawArrays(primitiveType, offset, count);
}
function setRectangle(gl, x, y, width, height) {
var x1 = x;
var x2 = x + width;
var y1 = y;
var y2 = y + height;
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
x1, y1,
x2, y1,
x1, y2,
x1, y2,
x2, y1,
x2, y2,
]), gl.STATIC_DRAW);
}
var then = 0;
// Draw the scene repeatedly
function render(now) {
now *= 0.001; // convert to seconds
const deltaTime = now - then;
then = now;
if (copyVideo) {
updateTexture(gl, texture, video);
}
drawScene();
requestAnimationFrame(render);
}
requestAnimationFrame(render);
function initTexture(gl) {
const texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
// Because video has to be download over the internet
// they might take a moment until it's ready so
// put a single pixel in the texture so we can
// use it immediately.
const level = 0;
const internalFormat = gl.RGBA;
const width = 1;
const height = 1;
const border = 0;
const srcFormat = gl.RGBA;
const srcType = gl.UNSIGNED_BYTE;
const pixel = new Uint8Array([0, 0, 255, 255]); // opaque blue
gl.texImage2D(gl.TEXTURE_2D, level, internalFormat,
width, height, border, srcFormat, srcType,
pixel);
// Turn off mips and set wrapping to clamp to edge so it
// will work regardless of the dimensions of the video.
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
return texture;
}
function updateTexture(gl, texture, video) {
const level = 0;
const internalFormat = gl.RGBA;
const srcFormat = gl.RGBA;
const srcType = gl.UNSIGNED_BYTE;
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, level, internalFormat,
srcFormat, srcType, video);
}
})();
I'm using the technique described here (code, demo) for using video frames as WebGL textures, and the simple scene (just showing the image in 2D, rather than a 3D rotating cube) from here.
The goal is a Tampermonkey userscript (with WebGL shaders, i.e. video effects) for YouTube.
The canvas is filled grey due to gl.clearColor(0.5,0.5,0.5,1). But the next lines of code, which should draw the frame from the video, have no visible effect. What part might be wrong? There are no errors.
I tried to shorten the code before posting, but apparently even simple WebGL scenes require a lot of boilerplate code.
// ==UserScript==
// @name tmp
// @namespace http://tampermonkey/
// @version 0.1
// @description try to take over the world!
// @author You
// @match https://www.youtube./*
// @icon https://www.google./s2/favicons?domain=youtube.
// @grant none
// ==/UserScript==
(function() {
// will set to true when video can be copied to texture
var copyVideo = false;
const video = document.getElementsByTagName("video")[0];
// immediately after finding the video, create canvas and set its dimensions
let canvas = document.createElement('canvas');
canvas.setAttribute('id', 'glcanvas');
canvas.setAttribute('width', '300');
canvas.setAttribute('height', '200');
canvas.setAttribute('style', 'position: absolute;');
video.parentElement.appendChild(canvas);
var playing = false;
var timeupdate = false;
// Waiting for these 2 events ensures
// there is data in the video
video.addEventListener('playing', function() {
playing = true;
checkReady();
}, true);
video.addEventListener('timeupdate', function() {
timeupdate = true;
checkReady();
}, true);
function checkReady() {
if (playing && timeupdate) {
copyVideo = true;
}
}
// Initialize the GL context
const gl = canvas.getContext("webgl");
// Only continue if WebGL is available and working
if (gl === null) {
alert("Unable to initialize WebGL. Your browser or machine may not support it.");
return;
}
// Vertex shader program
const vsSource = `
attribute vec2 a_position;
attribute vec2 a_texCoord;
uniform vec2 u_resolution;
varying vec2 v_texCoord;
void main() {
// convert the rectangle from pixels to 0.0 to 1.0
vec2 zeroToOne = a_position / u_resolution;
// convert from 0->1 to 0->2
vec2 zeroToTwo = zeroToOne * 2.0;
// convert from 0->2 to -1->+1 (clipspace)
vec2 clipSpace = zeroToTwo - 1.0;
gl_Position = vec4(clipSpace * vec2(1, -1), 0, 1);
// pass the texCoord to the fragment shader
// The GPU will interpolate this value between points.
v_texCoord = a_texCoord;
}
`;
// Fragment shader program
const fsSource = `
precision mediump float;
// our texture
uniform sampler2D u_image;
// the texCoords passed in from the vertex shader.
varying vec2 v_texCoord;
void main() {
gl_FragColor = texture2D(u_image, v_texCoord).bgra;
}
`;
// Initialize a shader program, so WebGL knows how to draw our data
function initShaderProgram(gl, vsSource, fsSource) {
const vertexShader = loadShader(gl, gl.VERTEX_SHADER, vsSource);
const fragmentShader = loadShader(gl, gl.FRAGMENT_SHADER, fsSource);
// Create the shader program
const shaderProgram = gl.createProgram();
gl.attachShader(shaderProgram, vertexShader);
gl.attachShader(shaderProgram, fragmentShader);
gl.linkProgram(shaderProgram);
// If creating the shader program failed, alert
if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {
alert('Unable to initialize the shader program: ' + gl.getProgramInfoLog(shaderProgram));
return null;
}
return shaderProgram;
}
// creates a shader of the given type, uploads the source and piles it.
function loadShader(gl, type, source) {
const shader = gl.createShader(type);
// Send the source to the shader object
gl.shaderSource(shader, source);
// Compile the shader program
gl.pileShader(shader);
// See if it piled successfully
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
alert('An error occurred piling the shaders: ' + gl.getShaderInfoLog(shader));
gl.deleteShader(shader);
return null;
}
return shader;
}
// Initialize a shader program; this is where all the lighting
// for the vertices and so forth is established.
const shaderProgram = initShaderProgram(gl, vsSource, fsSource);
// look up where the vertex data needs to go.
var positionLocation = gl.getAttribLocation(shaderProgram, "a_position");
var texcoordLocation = gl.getAttribLocation(shaderProgram, "a_texCoord");
// Create a buffer to put three 2d clip space points in
var positionBuffer = gl.createBuffer();
// Bind it to ARRAY_BUFFER (think of it as ARRAY_BUFFER = positionBuffer)
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
// Set a rectangle the same size as the image.
setRectangle(gl, 0, 0, video.width, video.height);
// provide texture coordinates for the rectangle.
var texcoordBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer);
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
0.0, 0.0,
1.0, 0.0,
0.0, 1.0,
0.0, 1.0,
1.0, 0.0,
1.0, 1.0,
]), gl.STATIC_DRAW);
// Create a texture.
var texture = initTexture(gl);
function drawScene() {
// lookup uniforms
var resolutionLocation = gl.getUniformLocation(shaderProgram, "u_resolution");
//webglUtils.resizeCanvasToDisplaySize(gl.canvas);
// Tell WebGL how to convert from clip space to pixels
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
// Clear the canvas
gl.clearColor(0.5,0.5,0.5,1);
gl.clear(gl.COLOR_BUFFER_BIT);
// Tell it to use our program (pair of shaders)
gl.useProgram(shaderProgram);
// Turn on the position attribute
gl.enableVertexAttribArray(positionLocation);
// Bind the position buffer.
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
// Tell the position attribute how to get data out of positionBuffer (ARRAY_BUFFER)
var size = 2; // 2 ponents per iteration
var type = gl.FLOAT; // the data is 32bit floats
var normalize = false; // don't normalize the data
var stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
var offset = 0; // start at the beginning of the buffer
gl.vertexAttribPointer(
positionLocation, size, type, normalize, stride, offset);
// Turn on the texcoord attribute
gl.enableVertexAttribArray(texcoordLocation);
// bind the texcoord buffer.
gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer);
// Tell the texcoord attribute how to get data out of texcoordBuffer (ARRAY_BUFFER)
var size = 2; // 2 ponents per iteration
var type = gl.FLOAT; // the data is 32bit floats
var normalize = false; // don't normalize the data
var stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
var offset = 0; // start at the beginning of the buffer
gl.vertexAttribPointer(
texcoordLocation, size, type, normalize, stride, offset);
// set the resolution
gl.uniform2f(resolutionLocation, gl.canvas.width, gl.canvas.height);
// Draw the rectangle.
var primitiveType = gl.TRIANGLES;
var offset = 0;
var count = 6;
gl.drawArrays(primitiveType, offset, count);
}
function setRectangle(gl, x, y, width, height) {
var x1 = x;
var x2 = x + width;
var y1 = y;
var y2 = y + height;
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
x1, y1,
x2, y1,
x1, y2,
x1, y2,
x2, y1,
x2, y2,
]), gl.STATIC_DRAW);
}
var then = 0;
// Draw the scene repeatedly
function render(now) {
now *= 0.001; // convert to seconds
const deltaTime = now - then;
then = now;
if (copyVideo) {
updateTexture(gl, texture, video);
}
drawScene();
requestAnimationFrame(render);
}
requestAnimationFrame(render);
function initTexture(gl) {
const texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
// Because video has to be download over the internet
// they might take a moment until it's ready so
// put a single pixel in the texture so we can
// use it immediately.
const level = 0;
const internalFormat = gl.RGBA;
const width = 1;
const height = 1;
const border = 0;
const srcFormat = gl.RGBA;
const srcType = gl.UNSIGNED_BYTE;
const pixel = new Uint8Array([0, 0, 255, 255]); // opaque blue
gl.texImage2D(gl.TEXTURE_2D, level, internalFormat,
width, height, border, srcFormat, srcType,
pixel);
// Turn off mips and set wrapping to clamp to edge so it
// will work regardless of the dimensions of the video.
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
return texture;
}
function updateTexture(gl, texture, video) {
const level = 0;
const internalFormat = gl.RGBA;
const srcFormat = gl.RGBA;
const srcType = gl.UNSIGNED_BYTE;
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, level, internalFormat,
srcFormat, srcType, video);
}
})();
2 Answers
Reset to default 7The first problem was correctly pointed out by Atekihcan, you're getting things mixed up with your NDC coordinate calculations, and indeed it's a lot easier to just send them directly. Furthermore you can derive the texture coordinates from those rather easily and thus save the setup of the second buffer.
The second problem is that your events are not firing in the chain you expect them to (at least not for me reloading a playing video and running the script). I believe it should be enough to listen to the timeupdate event since the time won't update if the video can't play. The working code:
// ==UserScript==
// @name tmp
// @namespace http://tampermonkey/
// @version 0.1
// @description try to take over the world!
// @author You
// @match https://www.youtube./*
// @icon https://www.google./s2/favicons?domain=youtube.
// @grant none
// ==/UserScript==
(function() {
// will set to true when video can be copied to texture
var copyVideo = false;
const video = document.getElementsByTagName("video")[0];
// immediately after finding the video, create canvas and set its dimensions
let canvas = document.createElement('canvas');
canvas.setAttribute('id', 'glcanvas');
canvas.setAttribute('width', '300');
canvas.setAttribute('height', '200');
canvas.setAttribute('style', 'position: absolute;');
video.parentElement.appendChild(canvas);
video.addEventListener('timeupdate', function() {
copyVideo=true;
}, true);
// Initialize the GL context
const gl = canvas.getContext("webgl");
// Only continue if WebGL is available and working
if (gl === null) {
alert("Unable to initialize WebGL. Your browser or machine may not support it.");
return;
}
// Vertex shader program
const vsSource = `
attribute vec2 a_position;
varying vec2 v_texCoord;
void main() {
gl_Position = vec4(a_position, 0.0, 1.0);
v_texCoord = a_position*.5+.5;
v_texCoord.y = 1.-v_texCoord.y;
}
`;
// Fragment shader program
const fsSource = `
precision mediump float;
uniform sampler2D u_image;
varying vec2 v_texCoord;
void main() {
gl_FragColor = texture2D(u_image, v_texCoord);
}
`;
const positionData = new Float32Array([
-1.0,-1.0,
1.0,-1.0,
-1.0, 1.0,
1.0,-1.0,
1.0, 1.0,
-1.0, 1.0
]);
// Initialize a shader program, so WebGL knows how to draw our data
function initShaderProgram(gl, vsSource, fsSource) {
const shaderProgram = gl.createProgram();
gl.attachShader(shaderProgram, loadShader(gl, gl.VERTEX_SHADER, vsSource));
gl.attachShader(shaderProgram, loadShader(gl, gl.FRAGMENT_SHADER, fsSource));
gl.linkProgram(shaderProgram);
// If creating the shader program failed, alert
if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {
alert('Unable to initialize the shader program: ' + gl.getProgramInfoLog(shaderProgram));
return null;
}
return shaderProgram;
}
// creates a shader of the given type, uploads the source and piles it.
function loadShader(gl, type, source) {
const shader = gl.createShader(type);
gl.shaderSource(shader, source);
gl.pileShader(shader);
// See if it piled successfully
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
alert('An error occurred piling the shaders: ' + gl.getShaderInfoLog(shader));
gl.deleteShader(shader);
return null;
}
return shader;
}
// Initialize shader program
const shaderProgram = initShaderProgram(gl, vsSource, fsSource);
// look up where the vertex data needs to go.
var positionLocation = gl.getAttribLocation(shaderProgram, "a_position");
var textureLoc = gl.getUniformLocation(shaderProgram, "u_image");
// Create a vertex buffer
var positionBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.bufferData(gl.ARRAY_BUFFER, positionData, gl.STATIC_DRAW);
// Create texture
var texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, 1, 1, 0, gl.RGB, gl.UNSIGNED_BYTE, new Uint8Array([0, 0, 255]));
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
// Initialize rendering
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
gl.clearColor(1.0,0.0,0.0,1.0);
function drawScene() {
gl.clear(gl.COLOR_BUFFER_BIT);
gl.useProgram(shaderProgram);
// Turn on the vertex attribute
gl.enableVertexAttribArray(positionLocation);
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0);
// Draw the rectangle
gl.drawArrays(gl.TRIANGLES, 0, 6);
}
// Draw the scene repeatedly
function render() {
if (copyVideo)
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, gl.RGB, gl.UNSIGNED_BYTE, video);
drawScene();
requestAnimationFrame(render);
}
requestAnimationFrame(render);
})();
Note: I also changed the texture format to RGB (alpha channel will implicitly be 1), but that's irrelevant.
Edit: As it has been pointed out, first two sections of this answer are pletely wrong.
TLDR: This might not be feasible without a backend server first fetching the video data.
If you check the MDN tutorial you followed, the video object passed to texImage2D is actually an MP4 video. However, in your script, the video object you have access to (document.getElementsByTagName("video")[0]) is just a DOM object. You don't have the actual video data. And it is not easy to get access to that for YouTube. The YouTube player do not fetch the video data in one shot, rather the YouTube streaming server makes sure to stream chunks of the video. I am not absolutely sure on this, but I think it'll be very difficult to work around this if your goal is to have a real time video effects.
I found some discussion on this (link1, link2) which might help.
That being said, there are some issues in your code from WebGL perspective. Ideally the code you have should be showing a blue rectangle as that is the texture data you are creating, instead of the initial glClearColor color. And after the video starts to play, it should switch to the video texture (which will show as black due to the issue I have explained above).
I think it is due to the way you had setup your position data and doing clip space calculation in the shader. That can be skipped to directly send normalized device coordinate position data. Here is the updated code, with some cleaning up to make it shorter, which behaves as expected:
// ==UserScript==
// @name tmp
// @namespace http://tampermonkey/
// @version 0.1
// @description try to take over the world!
// @author You
// @match https://www.youtube./*
// @icon https://www.google./s2/favicons?domain=youtube.
// @grant none
// ==/UserScript==
(function() {
// will set to true when video can be copied to texture
var copyVideo = false;
const video = document.getElementsByTagName("video")[0];
// immediately after finding the video, create canvas and set its dimensions
let canvas = document.createElement('canvas');
canvas.setAttribute('id', 'glcanvas');
canvas.setAttribute('width', '300');
canvas.setAttribute('height', '200');
canvas.setAttribute('style', 'position: absolute;');
video.parentElement.appendChild(canvas);
var playing = false;
var timeupdate = false;
// Waiting for these 2 events ensures
// there is data in the video
video.addEventListener('playing', function() {
playing = true;
checkReady();
}, true);
video.addEventListener('timeupdate', function() {
timeupdate = true;
checkReady();
}, true);
function checkReady() {
if (playing && timeupdate) {
copyVideo = true;
}
}
// Initialize the GL context
const gl = canvas.getContext("webgl");
// Only continue if WebGL is available and working
if (gl === null) {
alert("Unable to initialize WebGL. Your browser or machine may not support it.");
return;
}
// Vertex shader program
const vsSource = `
attribute vec2 a_position;
attribute vec2 a_texCoord;
varying vec2 v_texCoord;
void main() {
gl_Position = vec4(a_position, 0.0, 1.0);
v_texCoord = a_texCoord;
}
`;
// Fragment shader program
const fsSource = `
precision mediump float;
uniform sampler2D u_image;
varying vec2 v_texCoord;
void main() {
gl_FragColor = texture2D(u_image, v_texCoord);
}
`;
const positionData = new Float32Array([
-1.0,-1.0,
1.0,-1.0,
-1.0, 1.0,
1.0,-1.0,
1.0, 1.0,
-1.0, 1.0
]);
const texcoordData = new Float32Array([
0.0, 0.0,
1.0, 0.0,
0.0, 1.0,
0.0, 1.0,
1.0, 0.0,
1.0, 1.0,
]);
// Initialize a shader program, so WebGL knows how to draw our data
function initShaderProgram(gl, vsSource, fsSource) {
const shaderProgram = gl.createProgram();
gl.attachShader(shaderProgram, loadShader(gl, gl.VERTEX_SHADER, vsSource));
gl.attachShader(shaderProgram, loadShader(gl, gl.FRAGMENT_SHADER, fsSource));
gl.linkProgram(shaderProgram);
// If creating the shader program failed, alert
if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {
alert('Unable to initialize the shader program: ' + gl.getProgramInfoLog(shaderProgram));
return null;
}
return shaderProgram;
}
// creates a shader of the given type, uploads the source and piles it.
function loadShader(gl, type, source) {
const shader = gl.createShader(type);
gl.shaderSource(shader, source);
gl.pileShader(shader);
// See if it piled successfully
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
alert('An error occurred piling the shaders: ' + gl.getShaderInfoLog(shader));
gl.deleteShader(shader);
return null;
}
return shader;
}
// Initialize shader program
const shaderProgram = initShaderProgram(gl, vsSource, fsSource);
// look up where the vertex data needs to go.
var positionLocation = gl.getAttribLocation(shaderProgram, "a_position");
var texcoordLocation = gl.getAttribLocation(shaderProgram, "a_texCoord");
var textureLoc = gl.getUniformLocation(shaderProgram, "u_image");
// Create a vertex buffer
var positionBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.bufferData(gl.ARRAY_BUFFER, positionData, gl.STATIC_DRAW);
// Create texture coordinate buffer
var texcoordBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer);
gl.bufferData(gl.ARRAY_BUFFER, texcoordData, gl.STATIC_DRAW);
// Create texture
var texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 1, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, new Uint8Array([0, 0, 255, 255]));
// Initialize rendering
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
gl.clearColor(1.0,0.0,0.0,1.0);
function drawScene() {
gl.clear(gl.COLOR_BUFFER_BIT);
gl.useProgram(shaderProgram);
// Turn on the vertex attribute
gl.enableVertexAttribArray(positionLocation);
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0);
// Turn on the texcoord attribute
gl.enableVertexAttribArray(texcoordLocation);
gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer);
gl.vertexAttribPointer(texcoordLocation, 2, gl.FLOAT, false, 0, 0);
// Draw the rectangle
gl.drawArrays(gl.TRIANGLES, 0, 6);
}
// Draw the scene repeatedly
function render() {
if (copyVideo) {
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video);
}
drawScene();
requestAnimationFrame(render);
}
requestAnimationFrame(render);
})();
发布者：admin，转转请注明出处：http://www.yc00.com/questions/1745187395a4615701.html
评论列表（0条）