I am working on a web app on React.js + flask which undergoes speech recognition, and requires both the transcripted message and the recorded audio to be passed to a handler.
My problem is this.parse(transcript, audio) always passes a null object for audio. Debugging on Chrome indicates mediaRecorder.onstop and mediaRecorder.ondataavailable is only triggered after this.parse(transcript, audio) is handled so nothing is assigned to audio.
Is there a way where I can capture the audio from mediaRecorder before this.parse(transcript, audio) is executed?
Also, which variable should I pass (audio, audioURL, blob...) for the actual audio file?
class MessageParser {
constructor(actionProvider, state) {
var audio = null, blob = null, audioURL = null, mediaRecorder = null;
let chunks = [];
this.actionProvider = actionProvider;
this.state = state;
this.recognition = new SpeechRecognition()
this.recognition.continuous = false
this.recognition.interimResults = false
this.recognition.lang = 'en-US'
this.recognition.maxAlternatives = 1;
this.recognition.start()
this.recognition.onstart = function(){
if (navigator.mediaDevices.getUserMedia) {
//console.log('getUserMedia supported.');
let chunks = [];
var options = {
audioBitsPerSecond: 128000,
mimeTyoe: 'audio/webm'
}
navigator.mediaDevices.getUserMedia({ audio: true, })
.then(function (stream) {
mediaRecorder = new MediaRecorder(stream, options);
mediaRecorder.start();
})
} else {
console.log('getUserMedia Unsupported.');
}
};
this.recognition.onresult = (e) => {
const transcript = e.results[0][0].transcript;
var audioURL = null;
this.recognition.abort()
mediaRecorder.stop();
mediaRecorder.onstop = function (e) {
console.log("data available after MediaRecorder.stop() called.");
audio = document.createElement('audio');
blob = new Blob(chunks, { 'type': 'audio/webm; codecs=opus' });
audioURL = window.URL.createObjectURL(blob);
audio.src = audioURL;
console.log("recorder stopped");
const recording = new Audio(audioURL)
recording.play()
}
mediaRecorder.ondataavailable = function (e) {
chunks.push(e.data);
}
this.parse(transcript, audio)
}
}
In this demo http://jsfiddle.net/mdz82oLn/ I have inserted videos (IDs) and images (png, jpeg, etc) which are also displayed by means of customized buttons. The demo works perfectly in the vision but reached the end of the last video it returns in loop when instead it should restart from the first video in the list, how to integrate this function?
var tag = document.createElement('script');
tag.src = "https://www.youtube.com/iframe_api";
var firstScriptTag = document.getElementsByTagName('script')[0];
firstScriptTag.parentNode.insertBefore(tag, firstScriptTag);
const playerElement = document.querySelector('#player');
const imageElement = document.querySelector('#slide');
const videos = {
'RGpr3Y6Q-1M': 'http://nothingbutgeek.com/wp-content/uploads/2018/06/automata_16x9.png',
'btxdcqLOGuc': 'https://live.staticflickr.com/2400/2078946248_d063d5a563_b.jpg',
'CIx0a1vcYPc': 'https://i.ytimg.com/vi/CIx0a1vcYPc/maxresdefault.jpg',
};
const videoIds = Object.keys(videos);
function onYouTubeIframeAPIReady() {
function onPlayerReady({ target }) {
var playButton = document.getElementById("play-button");
playButton.addEventListener("click", function() {
target.playVideo();
});
var pauseButton = document.getElementById("pause-button");
pauseButton.addEventListener("click", function() {
target.pauseVideo();
});
var next = document.getElementById("next");
next.addEventListener("click", function() {
target.nextVideo();
});
var pre = document.getElementById("previous");
pre.addEventListener("click", function() {
target.previousVideo();
});
target.loadPlaylist({
playlist: videoIds
});
}
function onPlayerStateChange({ data, target }) {
switch(data) {
case YT.PlayerState.ENDED:
target.nextVideo();
break;
case YT.PlayerState.BUFFERING:
const playlist = target.getPlaylist();
const playlistIndex = target.getPlaylistIndex();
const currentId = playlist[playlistIndex];
const image = videos[currentId];
if (imageElement.src !== image) {
imageElement.src = image;
}
break;
}
}
const player = new YT.Player(playerElement, {
height: '405',
width: '720',
playerVars: {
controls: 1,
},
events: {
'onReady': onPlayerReady,
'onStateChange': onPlayerStateChange
}
});
}
If I understand correctly you are stuck on last video and want to start the loop from 1 video:
In that case a change like this should work:
var next = document.getElementById("next");
next.addEventListener("click", function() {
target.nextVideo();
if(target.getPlaylistIndex() == videoIds.length -1){
target.loadPlaylist({
playlist: videoIds
});
}
});
Updated your Example
you can also update the switch case like this:
case YT.PlayerState.ENDED: {
if (target.getPlaylistIndex() == videoIds.length - 1) {
target.loadPlaylist({
playlist: videoIds
});
}
break;
}
I have Reactjs application in which I have a video call concept. I have used RTCMultiConnection package to achieve video call.
Peer server is connected and the video call works well in localhost.
But when I hosted the react js application in a server (has ssl certificate too) the video call is not working.
The onStream function is not getting calling when video call starts and hence the user video is not displaying.
I have hosted the react js app by taking build and tried it in Heroku server and in windows dedicated server (through IIS). But the video call didn't worked in either of the servers.
How to get the video call to work on server too.
My sample code:
export function AudioVideoConferenceStart(data, callback) {
var liveOrJoined = data.liveOrJoined;
var callType = data.callType;
var documentData = data.documentData;
var sectionCount = data.sectionCount;
var roomId = data.roomId;
var isChat = data.isChat;
var isOnetoOne = data.isOnetoOne;
globalVar.connection = new RTCMultiConnection();
globalVar.connection.socketURL = defaultVal.ENV === "dev" ? defaultVal.APILINK_DEV : defaultVal.APILINK_PROD;
globalVar.connection.socketMessageEvent = 'video-conference-demo';
globalVar.connection.session = {
audio: true,
video: true
};
globalVar.connection.sdpConstraints.mandatory = {
OfferToReceiveAudio: true,
OfferToReceiveVideo: true
};
var bitrates = 512;
var resolutions = 'Ultra-HD';
var videoConstraints = {};
if (resolutions == 'HD') {
videoConstraints = {
width: {
ideal: 1280
},
height: {
ideal: 720
},
frameRate: 30
};
}
if (resolutions == 'Ultra-HD') {
videoConstraints = {
width: {
ideal: 1920
},
height: {
ideal: 1080
},
frameRate: 30
};
}
globalVar.connection.mediaConstraints = {
video: videoConstraints,
audio: true
};
var CodecsHandler = globalVar.connection.CodecsHandler;
globalVar.connection.processSdp = function (sdp) {
var codecs = 'vp8';
if (codecs.length) {
sdp = CodecsHandler.preferCodec(sdp, codecs.toLowerCase());
}
if (resolutions == 'HD') {
sdp = CodecsHandler.setApplicationSpecificBandwidth(sdp, {
audio: 128,
video: bitrates,
screen: bitrates
});
sdp = CodecsHandler.setVideoBitrates(sdp, {
min: bitrates * 8 * 1024,
max: bitrates * 8 * 1024,
});
}
if (resolutions == 'Ultra-HD') {
sdp = CodecsHandler.setApplicationSpecificBandwidth(sdp, {
audio: 128,
video: bitrates,
screen: bitrates
});
sdp = CodecsHandler.setVideoBitrates(sdp, {
min: bitrates * 8 * 1024,
max: bitrates * 8 * 1024,
});
}
return sdp;
};
globalVar.connection.iceServers = [{
'urls': [
'stun:stun.l.google.com:19302',
'stun:stun1.l.google.com:19302',
'stun:stun2.l.google.com:19302',
'stun:stun.l.google.com:19302?transport=udp',
]
}];
globalVar.connection.onstream = function (event) {
alert("Stream Added");
};
globalVar.connection.onstreamended = function (event) {
alert("stream removed");
};
globalVar.connection.onMediaError = function (e) {
alert(e.message);
};
// document.getElementById('room-id').value = roomid;
localStorage.setItem(globalVar.connection.socketMessageEvent, roomid);
// auto-join-room
(function reCheckRoomPresence() {
globalVar.connection.checkPresence(roomid, function (isRoomExist) {
if (isRoomExist) {
globalVar.connection.join(roomid);
return;
}
setTimeout(reCheckRoomPresence, 5000);
});
})();
}
I have implemented a service which streams camera output on html5. But it works only if I use localhost:8080 localhost if I use IP address or machine name then it does not even detect the camera.
/*global logger*/
/*
VisualInspection
========================
#file : VisualInspection.js
#version : 1.0.0
#author :
#date : 7/28/2019
#copyright :
#license : Apache 2
Documentation
========================
Describe your widget here.
*/
// Required module list. Remove unnecessary modules, you can always get them back from the boilerplate.
define([
"dojo/_base/declare",
"mxui/widget/_WidgetBase",
"dijit/_TemplatedMixin",
"mxui/dom",
"dojo/dom",
"dojo/dom-prop",
"dojo/dom-geometry",
"dojo/dom-class",
"dojo/dom-style",
"dojo/dom-construct",
"dojo/_base/array",
"dojo/_base/lang",
"dojo/text",
"dojo/html",
"dojo/_base/event",
"VisualInspection/lib/jquery-1.11.2",
"dojo/text!VisualInspection/widget/template/VisualInspection.html",
"VisualInspection/widget/template/tf.min",
// "dojo/text!VisualInspection/widget/template/labels.json",
// "dojo/text!VisualInspection/widget/template/model.json"
], function (declare, _WidgetBase, _TemplatedMixin, dom, dojoDom, dojoProp, dojoGeometry, dojoClass, dojoStyle, dojoConstruct, dojoArray, lang, dojoText, dojoHtml, dojoEvent, _jQuery, widgetTemplate, tf) {
"use strict";
var $ = _jQuery.noConflict(true);
var LABELS_URL = "http://pni6w2465:7777/EasyPlan/model_web/labels.json"
var MODEL_JSON = "http://pni6w2465:7777/EasyPlan/model_web/model.json"
// var tf = require(['../../VisualInspection/node_modules/#tensorflow/tfjs']);
//////////////
const TFWrapper = model => {
const calculateMaxScores = (scores, numBoxes, numClasses) => {
const maxes = []
const classes = []
for (let i = 0; i < numBoxes; i++) {
let max = Number.MIN_VALUE
let index = -1
for (let j = 0; j < numClasses; j++) {
if (scores[i * numClasses + j] > max) {
max = scores[i * numClasses + j]
index = j
}
}
maxes[i] = max
classes[i] = index
}
return [maxes, classes]
}
const buildDetectedObjects = (
width,
height,
boxes,
scores,
indexes,
classes
) => {
const count = indexes.length
const objects = []
for (let i = 0; i < count; i++) {
const bbox = []
for (let j = 0; j < 4; j++) {
bbox[j] = boxes[indexes[i] * 4 + j]
}
const minY = bbox[0] * height
const minX = bbox[1] * width
const maxY = bbox[2] * height
const maxX = bbox[3] * width
bbox[0] = minX
bbox[1] = minY
bbox[2] = maxX - minX
bbox[3] = maxY - minY
objects.push({
bbox: bbox,
class: classes[indexes[i]],
score: scores[indexes[i]]
})
}
return objects
}
var img = null;
const detect = input => {
const batched = tf.tidy(() => {
const img = tf.browser.fromPixels(input)
// Reshape to a single-element batch so we can pass it to executeAsync.
// var img = null;
// //sid
// var canvas = document.querySelector("#canvasElement");
// if (canvas.getContext) {
// var ctx = canvas.getContext("2d");
// img = canvas.toDataURL("image/png");
// }
return img.expandDims(0)
})
const height = batched.shape[1]
const width = batched.shape[2]
// const height = img.height
// const width = img.width
return model.executeAsync(batched).then(result => {
const scores = result[0].dataSync()
const boxes = result[1].dataSync()
// clean the webgl tensors
batched.dispose()
tf.dispose(result)
const [maxScores, classes] = calculateMaxScores(
scores,
result[0].shape[1],
result[0].shape[2]
)
const prevBackend = tf.getBackend()
// run post process in cpu
tf.setBackend('cpu')
const indexTensor = tf.tidy(() => {
const boxes2 = tf.tensor2d(boxes, [
result[1].shape[1],
result[1].shape[3]
])
return tf.image.nonMaxSuppression(
boxes2,
maxScores,
20, // maxNumBoxes
0.5, // iou_threshold
0.5 // score_threshold
)
})
const indexes = indexTensor.dataSync()
indexTensor.dispose()
// restore previous backend
tf.setBackend(prevBackend)
return buildDetectedObjects(
width,
height,
boxes,
maxScores,
indexes,
classes
)
})
}
return {
detect: detect
}
}
//////////////////////
// Declare widget's prototype.
return declare("VisualInspection.widget.VisualInspection", [_WidgetBase, _TemplatedMixin], {
// _TemplatedMixin will create our dom node using this HTML template.
templateString: widgetTemplate,
// DOM elements
inputNodes: null,
colorSelectNode: null,
colorInputNode: null,
infoTextNode: null,
// Parameters configured in the Modeler.
mfToExecute: "",
messageString: "",
backgroundColor: "",
// Internal variables. Non-primitives created in the prototype are shared between all widget instances.
_handles: null,
_contextObj: null,
_alertDiv: null,
_readOnly: false,
// dojo.declare.constructor is called to construct the widget instance. Implement to initialize non-primitive properties.
constructor: function () {
logger.debug(this.id + ".constructor");
this._handles = [];
},
// dijit._WidgetBase.postCreate is called after constructing the widget. Implement to do extra setup work.
postCreate: function () {
logger.debug(this.id + ".postCreate");
if (this.readOnly || this.get("disabled") || this.readonly) {
this._readOnly = true;
}
this._updateRendering();
this._setupEvents();
var video = document.querySelector("#videoElement");
var canvas = document.querySelector("#canvasElement");
// if (navigator.mediaDevices.getUserMedia) {
// navigator.mediaDevices.getUserMedia({ video: true })
// .then(function (stream) {
// video.srcObject = stream;
// })
// .catch(function (err0r) {
// console.log("Something went wrong!");
// });
// }
this.componentDidMount();
},
////////////////////////////////////////////////////////
componentDidMount: function () {
var video = document.querySelector("#videoElement");
if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
const webCamPromise = navigator.mediaDevices
.getUserMedia({
audio: false,
video: {
facingMode: 'user'
}
})
.then(stream => {
window.stream = stream
video.srcObject = stream
return new Promise((resolve, _) => {
video.onloadedmetadata = () => {
resolve()
}
})
})
const modelPromise = tf.loadGraphModel(MODEL_JSON)
const labelsPromise = fetch(LABELS_URL).then(data => data.json())
Promise.all([modelPromise, labelsPromise, webCamPromise])
.then(values => {
const [model, labels] = values
this.detectFrame(video, model, labels)
})
.catch(error => {
console.error(error)
})
}
},
detectFrame: function (video, model, labels) {
TFWrapper(model)
.detect(video)
.then(predictions => {
this.renderPredictions(predictions, labels)
requestAnimationFrame(() => {
this.detectFrame(video, model, labels)
})
})
},
renderPredictions: function (predictions, labels) {
var canvas = document.querySelector("#canvasElement");
const ctx = canvas.getContext('2d')
ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height)
// Font options.
const font = '16px sans-serif'
ctx.font = font
ctx.textBaseline = 'top'
predictions.forEach(prediction => {
const x = prediction.bbox[0]
const y = prediction.bbox[1]
const width = prediction.bbox[2]
const height = prediction.bbox[3]
const label = labels[parseInt(prediction.class)]
// Draw the bounding box.
ctx.strokeStyle = '#00FFFF'
ctx.lineWidth = 4
ctx.strokeRect(x, y, width, height)
// Draw the label background.
ctx.fillStyle = '#00FFFF'
const textWidth = ctx.measureText(label).width
const textHeight = parseInt(font, 10) // base 10
ctx.fillRect(x, y, textWidth + 4, textHeight + 4)
})
predictions.forEach(prediction => {
const x = prediction.bbox[0]
const y = prediction.bbox[1]
const label = labels[parseInt(prediction.class)]
// Draw the text last to ensure it's on top.
ctx.fillStyle = '#000000'
ctx.fillText(label, x, y)
})
},
///////////////////////////////////////////////////////////
// mxui.widget._WidgetBase.update is called when context is changed or initialized. Implement to re-render and / or fetch data.
update: function (obj, callback) {
logger.debug(this.id + ".update");
this._contextObj = obj;
this._resetSubscriptions();
this._updateRendering(callback); // We're passing the callback to updateRendering to be called after DOM-manipulation
},
// mxui.widget._WidgetBase.enable is called when the widget should enable editing. Implement to enable editing if widget is input widget.
enable: function () {
logger.debug(this.id + ".enable");
},
// mxui.widget._WidgetBase.enable is called when the widget should disable editing. Implement to disable editing if widget is input widget.
disable: function () {
logger.debug(this.id + ".disable");
},
// mxui.widget._WidgetBase.resize is called when the page's layout is recalculated. Implement to do sizing calculations. Prefer using CSS instead.
resize: function (box) {
logger.debug(this.id + ".resize");
},
// mxui.widget._WidgetBase.uninitialize is called when the widget is destroyed. Implement to do special tear-down work.
uninitialize: function () {
logger.debug(this.id + ".uninitialize");
// Clean up listeners, helper objects, etc. There is no need to remove listeners added with this.connect / this.subscribe / this.own.
},
// We want to stop events on a mobile device
_stopBubblingEventOnMobile: function (e) {
logger.debug(this.id + "._stopBubblingEventOnMobile");
if (typeof document.ontouchstart !== "undefined") {
dojoEvent.stop(e);
}
},
// Attach events to HTML dom elements
_setupEvents: function () {
logger.debug(this.id + "._setupEvents");
this.connect(this.colorSelectNode, "change", function (e) {
// Function from mendix object to set an attribute.
this._contextObj.set(this.backgroundColor, this.colorSelectNode.value);
});
this.connect(this.infoTextNode, "click", function (e) {
// Only on mobile stop event bubbling!
this._stopBubblingEventOnMobile(e);
// If a microflow has been set execute the microflow on a click.
if (this.mfToExecute !== "") {
this._execMf(this.mfToExecute, this._contextObj.getGuid());
}
});
},
_execMf: function (mf, guid, cb) {
logger.debug(this.id + "._execMf");
if (mf && guid) {
mx.ui.action(mf, {
params: {
applyto: "selection",
guids: [guid]
},
callback: lang.hitch(this, function (objs) {
if (cb && typeof cb === "function") {
cb(objs);
}
}),
error: function (error) {
console.debug(error.description);
}
}, this);
}
},
// Rerender the interface.
_updateRendering: function (callback) {
logger.debug(this.id + "._updateRendering");
// Important to clear all validations!
this._clearValidations();
// The callback, coming from update, needs to be executed, to let the page know it finished rendering
this._executeCallback(callback, "_updateRendering");
},
// Handle validations.
_handleValidation: function (validations) {
logger.debug(this.id + "._handleValidation");
this._clearValidations();
var validation = validations[0],
message = validation.getReasonByAttribute(this.backgroundColor);
if (this._readOnly) {
validation.removeAttribute(this.backgroundColor);
} else if (message) {
this._addValidation(message);
validation.removeAttribute(this.backgroundColor);
}
},
// Clear validations.
_clearValidations: function () {
logger.debug(this.id + "._clearValidations");
dojoConstruct.destroy(this._alertDiv);
this._alertDiv = null;
},
// Show an error message.
_showError: function (message) {
logger.debug(this.id + "._showError");
if (this._alertDiv !== null) {
dojoHtml.set(this._alertDiv, message);
return true;
}
this._alertDiv = dojoConstruct.create("div", {
"class": "alert alert-danger",
"innerHTML": message
});
dojoConstruct.place(this._alertDiv, this.domNode);
},
// Add a validation.
_addValidation: function (message) {
logger.debug(this.id + "._addValidation");
this._showError(message);
},
// Reset subscriptions.
_resetSubscriptions: function () {
logger.debug(this.id + "._resetSubscriptions");
// Release handles on previous object, if any.
this.unsubscribeAll();
// When a mendix object exists create subscribtions.
if (this._contextObj) {
this.subscribe({
guid: this._contextObj.getGuid(),
callback: lang.hitch(this, function (guid) {
this._updateRendering();
})
});
this.subscribe({
guid: this._contextObj.getGuid(),
attr: this.backgroundColor,
callback: lang.hitch(this, function (guid, attr, attrValue) {
this._updateRendering();
})
});
this.subscribe({
guid: this._contextObj.getGuid(),
val: true,
callback: lang.hitch(this, this._handleValidation)
});
}
},
_executeCallback: function (cb, from) {
logger.debug(this.id + "._executeCallback" + (from ? " from " + from : ""));
if (cb && typeof cb === "function") {
cb();
}
}
});
});
require(["VisualInspection/widget/VisualInspection"]);
<div id="container">
<video autoplay="true" playsInline="true" width="600" height="500" id="videoElement" style="position: fixed;" >
</video>
<canvas id= "canvasElement" width="600" height="500" style="position: absolute;">
</canvas>
</div>
From Above code, When I run using localhost this code will execute :
<!-- begin snippet: js hide: false console: true babel: false -->
When I run using machine name or IP name :
This code is not considered in developer tool of chrome due to security or some reason