diff --git a/public/debugPane.ts b/public/debugPane.ts new file mode 100644 index 0000000..ae72489 --- /dev/null +++ b/public/debugPane.ts @@ -0,0 +1,58 @@ + +function updateUsername(username: string) { + let p = $.url().param() + p.worker = username + window.history.pushState($.url().param(), 'Audio annotation', '/gui.html?' + $.param(p)) + $('#worker-name').val(username) +} + +updateUsername($.url().param().worker) + +$('#change-user').click(function(e) { + recordMouseClick(e, "#change-user"); + // @ts-ignore + updateUsername($('#worker-name').val()) + $('#worker-name').blur() + reload(null) +}) + +$('#worker-name').keypress(function(event) { + if (event.which == 13) { + event.preventDefault() + $('#change-user').click() + $('#worker-name').blur() + } +}) + +function updateReferences(references: string[]) { + let p = $.url().param() + // TODO This is the old system + delete p.references + p.reference = references + window.history.pushState($.url().param(), 'Audio annotation', '/gui.html?' + $.param(p)) + $('#references-input').val(_.join(references, ' ')) +} + +function currentReferences() { + return _.concat( // TODO Legacy, remove 'references' and keep repeated 'reference' + _.isUndefined($.url().param().references) ? [] : _.split($.url().param().references, ',') + , _.isUndefined($.url().param().reference) ? [] : $.url().param().reference) +} + +updateReferences(currentReferences()) + +$('#edit-references').click(function(e) { + recordMouseClick(e, "#edit-references"); + // @ts-ignore + updateReferences(_.split($('#references-input').val(), ' ')) + $('#references-input').blur() + reload(null) +}) + +$('#references-input').keypress(function(event) { + if (event.which == 13) { + event.preventDefault() + $('#edit-references').click() + $('#references-input').blur() + } +}) diff --git a/public/extensions.ts b/public/extensions.ts new file mode 100644 index 0000000..82ee38e --- /dev/null +++ b/public/extensions.ts @@ -0,0 +1,92 @@ + +interface JQueryStatic { + url: any +} +interface JQuery { + bootstrapSwitch: any +} +interface AudioBufferSourceNode { + playbackState: any + PLAYING_STATE: any +} + +// https://gist.github.com/aaronk6/bff7cc600d863d31a7bf +/** + * Register ajax transports for blob send/recieve and array buffer send/receive via XMLHttpRequest Level 2 + * within the comfortable framework of the jquery ajax request, with full support for promises. + * + * Notice the +* in the dataType string? The + indicates we want this transport to be prepended to the list + * of potential transports (so it gets first dibs if the request passes the conditions within to provide the + * ajax transport, preventing the standard transport from hogging the request), and the * indicates that + * potentially any request with any dataType might want to use the transports provided herein. + * + * Remember to specify 'processData:false' in the ajax options when attempting to send a blob or arraybuffer - + * otherwise jquery will try (and fail) to convert the blob or buffer into a query string. + */ +$.ajaxTransport('+*', function(options, _originalOptions, jqXHR) { + // Test for the conditions that mean we can/want to send/receive blobs or arraybuffers - we need XMLHttpRequest + // level 2 (so feature-detect against window.FormData), feature detect against window.Blob or window.ArrayBuffer, + // and then check to see if the dataType is blob/arraybuffer or the data itself is a Blob/ArrayBuffer + if ( + FormData && + ((options.dataType && (options.dataType === 'blob' || options.dataType === 'arraybuffer')) || + (options.data && + ((window.Blob && options.data instanceof Blob) || (ArrayBuffer && options.data instanceof ArrayBuffer)))) + ) { + return { + /** + * Return a transport capable of sending and/or receiving blobs - in this case, we instantiate + * a new XMLHttpRequest and use it to actually perform the request, and funnel the result back + * into the jquery complete callback (such as the success function, done blocks, etc.) + * + * @param headers + * @param completeCallback + */ + send: function(headers, completeCallback) { + var xhr = new XMLHttpRequest(), + url = options.url || window.location.href, + type = options.type || 'GET', + dataType = options.dataType || 'text', + data = options.data || null, + async = options.async || true, + key + + xhr.addEventListener('load', function() { + var response = {}, + isSuccess + + isSuccess = (xhr.status >= 200 && xhr.status < 300) || xhr.status === 304 + + if (isSuccess) { + response[dataType] = xhr.response + } else { + // In case an error occured we assume that the response body contains + // text data - so let's convert the binary data to a string which we can + // pass to the complete callback. + response.text = String.fromCharCode.apply( + null, + // @ts-ignore + new Uint8Array(xhr.response) + ) + } + + // @ts-ignore + completeCallback(xhr.status, xhr.statusText, response, xhr.getAllResponseHeaders()) + }) + + xhr.open(type, url, async) + // @ts-ignore + xhr.responseType = dataType + + for (key in headers) { + if (headers.hasOwnProperty(key)) xhr.setRequestHeader(key, headers[key]) + } + // @ts-ignore + xhr.send(data) + }, + abort: function() { + jqXHR.abort() + }, + } + } +}) diff --git a/public/gui.ts b/public/gui.ts index 4a390fd..bb23948 100644 --- a/public/gui.ts +++ b/public/gui.ts @@ -8,1006 +8,185 @@ // TODO Maybe test for audio somehow before the person is qualified for the HIT // TODO If we haven't loaded in 30 seconds, do something about it -let guiRevision : string | null = null - -//////////////////////////////////////// Telemetry - -const telemetryEnabled : boolean = _.has($.url().param(), 'telemetry') ? $.url().param() === 'false' : true -let interactions : Interaction[] = []; - -function sendTelemetry() { - const is = interactions - interactions = [] - if(_.isArray(is) && is.length > 0) { - try { - $.ajax({ - type: 'POST', - data: {interactions: is, - worker: $.url().param().worker, - segment: segment, - token: token, - browser: browser, - width: canvas.width, - height: canvas.height, - words: words, - selected: selected, - start: startS, - end: endS, - startTime: startTime, - startOffset: startOffset, - lastClick: lastClick, - date: new Date(), - annotations: _.map(annotations, cloneAnnotation) - }, - dataType: 'application/json', - url: $.url().attr('protocol') + '://' + $.url().attr('host') + ':' + (parseInt($.url().attr('port'))+1) + '/telemetry' - }) - } catch(err) { - // We try our best to send back telemetry, but if it doesn't work, that's not an issue - } - } -} - -if(telemetryEnabled) - // every 10 seconds - setInterval(sendTelemetry, 5000) - -interface Interaction { - kind: string -} - -interface Message extends Interaction { - level: string, - data: string -} - -interface Click extends Interaction{ - x: number, - y: number, - relativeX: number, - relativeY: number, - elements: any[] -} - -interface DragStart extends Click{ -} - -interface DragEnd extends Click { -} - -interface Resize extends Interaction { - pagex: number, - pagey: number, -} - -interface Keypress extends Interaction { - key: string, - element?: string -} - -interface Send extends Interaction { - data: any, - server: string, - port: number, - why: string -} - -interface Receive extends Interaction { - response: any, - error: any, - status: string, - server: string - port: number, - why: string -} - -function recordMessage(i : Omit) { - const j : Message = {kind: 'message', - ... i} - interactions.push(j) -} - -function recordSend(i : Omit) { - const j : Send = {kind: 'send', - ... i} - interactions.push(j) -} - -_.mixin({ - deeply: - // @ts-ignore - function (obj, fn) { - if(_.isObjectLike(obj)) { - return _.mapValues(_.mapValues(obj, fn), function (v) { - // @ts-ignore - return _.isPlainObject(v) || _.isArray(v) ? _.deeply(v, fn) : v; - // return _.isPlainObject(v) ? _.deeply(v, fn) : _.isArray(v) ? v.map(function(x) { - // // @ts-ignore - // return _.deeply(x, fn); - // }) : v; - }) +function heightBottom(isReference: boolean) { + if (splitHeight) { + if (isReference) { + return '90%' } else { - return fn(obj) + return '0%' } - }, -}) - -function recordReceive(i : Omit) { - // @ts-ignore - i.response = _.deeply(i.response, function (val, key?) { - if(_.isArray(val) || _.isString(val) || _.isNumber(val) || _.isDate(val)) { - return val - } - if(_.isObjectLike(val)) { - val = _.omit(val, _.functions(val)) - if(_.has(val, 'responseJSON')) { - val = _.omit(val, ['responseText']) - } - return val - } - return null - }) - const j : Receive = {kind: 'receive', - ... i} - interactions.push(j) -} - -function recordKeypress(key: string, element?: string) { - const j : Keypress = {kind: 'keypress', - key: key, - element: element - } - interactions.push(j) -} - -function recordMouseClick(e : JQuery.Event, element: any, element2?: any) { - const j : Click = {kind: 'mouse', - elements: _.isUndefined(element2) ? [element] : [element, element2], - // @ts-ignore - relativeX: e.offsetX, // d3.event.layerX, - // @ts-ignore - relativeY: e.offsetY, // d3.event.layerY, - // @ts-ignore - x: e.pageX, // d3.event.x, - // @ts-ignore - y: e.pageY // d3.event.y - } - interactions.push(j) -} - -const preloadSegments = true - -function updateUsername(username: string) { - let p = $.url().param() - p.worker = username - window.history.pushState($.url().param(), 'Audio annotation', '/gui.html?' + $.param(p)) - $('#worker-name').val(username) -} - -updateUsername($.url().param().worker) - -$('#change-user').click(function (e) { - recordMouseClick(e, "#change-user"); - // @ts-ignore - updateUsername($('#worker-name').val()) - $('#worker-name').blur() - reload(null) -}) - -$('#worker-name').keypress(function (event) { - if (event.which == 13) { - event.preventDefault() - $('#change-user').click() - $('#worker-name').blur() - } -}) - -function updateReferences(references: string[]) { - let p = $.url().param() - // TODO This is the old system - delete p.references - p.reference = references - window.history.pushState($.url().param(), 'Audio annotation', '/gui.html?' + $.param(p)) - $('#references-input').val(_.join(references, ' ')) -} - -function currentReferences() { - return _.concat( // TODO Legacy, remove 'references' and keep repeated 'reference' - _.isUndefined($.url().param().references) ? [] : _.split($.url().param().references, ',') - ,_.isUndefined($.url().param().reference) ? [] : $.url().param().reference) -} - -updateReferences(currentReferences()) - -$('#edit-references').click(function (e) { - recordMouseClick(e, "#edit-references"); - // @ts-ignore - updateReferences(_.split($('#references-input').val(), ' ')) - $('#references-input').blur() - reload(null) -}) - -$('#references-input').keypress(function (event) { - if (event.which == 13) { - event.preventDefault() - $('#edit-references').click() - $('#references-input').blur() - } -}) - -let splitHeight = _.isUndefined($.url().param().splitHeight) ? true : $.url().param().splitHeight - -function heightBottom(isReference: boolean) { - if (splitHeight) { - if (isReference) { - return '90%' } else { - return '0%' + return '0%' } - } else { - return '0%' - } } function heightTop(isReference: boolean) { - if (splitHeight) { - if (isReference) { - return '100%' + if (splitHeight) { + if (isReference) { + return '100%' + } else { + return '90%' + } } else { - return '90%' + return '100%' } - } else { - return '100%' - } } function heightText(isReference: boolean) { - if (splitHeight) { - if (isReference) { - return '98%' - } else { - return '47%' - } - } else { - if (isReference) { - return '55%' + if (splitHeight) { + if (isReference) { + return '98%' + } else { + return '47%' + } } else { - return '47%' + if (isReference) { + return '55%' + } else { + return '47%' + } } - } } function heightTopLine(isReference: boolean) { - if (splitHeight) { - if (isReference) { - return '93%' + if (splitHeight) { + if (isReference) { + return '93%' + } else { + return '50%' + } } else { - return '50%' + return '50%' } - } else { - return '50%' - } -} - -//////////////////////////////////////// Basic types - -// https://www.everythingfrontend.com/posts/newtype-in-typescript.html -type TimeInBuffer = { value: number; readonly __tag: unique symbol } -type TimeInSegment = { value: number; readonly __tag: unique symbol } -type TimeInMovie = { value: number; readonly __tag: unique symbol } -type PositionInSpectrogram = { value: number; readonly __tag: unique symbol } - -function add(t1: T, t2: T): T { - return lift2(t1, t2, (a, b) => a + b) -} - -function sub(t1: T, t2: T): T { - return lift2(t1, t2, (a, b) => a - b) -} - -function addConst(t: T, c: number): T { - return lift(t, a => a + c) -} - -function subConst(t: T, c: number): T { - return lift(t, a => a - c) -} - -function addMin(t1: T, t2: T, t3: T): T { - return lift3(t1, t2, t3, (a, b, c) => Math.min(a + b, c)) -} - -function subMax(t1: T, t2: T, t3: T): T { - return lift3(t1, t2, t3, (a, b, c) => Math.max(a - b, c)) -} - -function to( - value: T['value'] -): T { - return (value as any) as T } -function from(value: T): T['value'] { - return (value as any) as T['value'] -} - -function lift( - value: T, - callback: (value: T['value']) => T['value'] -): T { - return callback(value) -} - -function lift2( - x: T, - y: T, - callback: (x: T['value'], y: T['value']) => T['value'] -): T { - return callback(x, y) -} - -function lift3( - x: T, - y: T, - z: T, - callback: (x: T['value'], y: T['value'], z: T['value']) => T['value'] -): T { - return callback(x, y, z) -} - -interface Annotation { - word: string - index: number - startTime?: TimeInMovie - endTime?: TimeInMovie - lastClickTimestamp?: number - id?: string | number - visuals?: Visuals -} - -interface Visuals { - group: d3.Selection - text: d3.Selection - startLine: d3.Selection - startLineHandle: d3.Selection - endLine: d3.Selection - endLineHandle: d3.Selection - filler: d3.Selection - topLine: d3.Selection -} - -interface Buffers { - normal: null | AudioBuffer - half: null | AudioBuffer -} - -enum BufferType { - normal = 'normal', - half = 'half', -} - -enum DragPosition { - start = 'startTime', - end = 'endTime', - both = 'both', -} - -enum LoadingState { - ready, - submitting, - loading -} - -// Some global extensions - -interface JQueryStatic { - url: any -} -interface JQuery { - bootstrapSwitch: any -} -interface AudioBufferSourceNode { - playbackState: any - PLAYING_STATE: any -} - -function isValidAnnotation(a: Annotation) { - return ( - _.has(a, 'startTime') && - !_.isUndefined(a.startTime) && - !_.isNull(a.startTime) && - _.has(a, 'endTime') && - !_.isUndefined(a.endTime) && - !_.isNull(a.endTime) - ) -} - -// https://gist.github.com/aaronk6/bff7cc600d863d31a7bf -/** - * Register ajax transports for blob send/recieve and array buffer send/receive via XMLHttpRequest Level 2 - * within the comfortable framework of the jquery ajax request, with full support for promises. - * - * Notice the +* in the dataType string? The + indicates we want this transport to be prepended to the list - * of potential transports (so it gets first dibs if the request passes the conditions within to provide the - * ajax transport, preventing the standard transport from hogging the request), and the * indicates that - * potentially any request with any dataType might want to use the transports provided herein. - * - * Remember to specify 'processData:false' in the ajax options when attempting to send a blob or arraybuffer - - * otherwise jquery will try (and fail) to convert the blob or buffer into a query string. - */ -$.ajaxTransport('+*', function (options, _originalOptions, jqXHR) { - // Test for the conditions that mean we can/want to send/receive blobs or arraybuffers - we need XMLHttpRequest - // level 2 (so feature-detect against window.FormData), feature detect against window.Blob or window.ArrayBuffer, - // and then check to see if the dataType is blob/arraybuffer or the data itself is a Blob/ArrayBuffer - if ( - FormData && - ((options.dataType && (options.dataType === 'blob' || options.dataType === 'arraybuffer')) || - (options.data && - ((window.Blob && options.data instanceof Blob) || (ArrayBuffer && options.data instanceof ArrayBuffer)))) - ) { - return { - /** - * Return a transport capable of sending and/or receiving blobs - in this case, we instantiate - * a new XMLHttpRequest and use it to actually perform the request, and funnel the result back - * into the jquery complete callback (such as the success function, done blocks, etc.) - * - * @param headers - * @param completeCallback - */ - send: function (headers, completeCallback) { - var xhr = new XMLHttpRequest(), - url = options.url || window.location.href, - type = options.type || 'GET', - dataType = options.dataType || 'text', - data = options.data || null, - async = options.async || true, - key - - xhr.addEventListener('load', function () { - var response = {}, - isSuccess - - isSuccess = (xhr.status >= 200 && xhr.status < 300) || xhr.status === 304 - - if (isSuccess) { - response[dataType] = xhr.response - } else { - // In case an error occured we assume that the response body contains - // text data - so let's convert the binary data to a string which we can - // pass to the complete callback. - response.text = String.fromCharCode.apply( - null, - // @ts-ignore - new Uint8Array(xhr.response) - ) - } - - // @ts-ignore - completeCallback(xhr.status, xhr.statusText, response, xhr.getAllResponseHeaders()) - }) - - xhr.open(type, url, async) - // @ts-ignore - xhr.responseType = dataType - - for (key in headers) { - if (headers.hasOwnProperty(key)) xhr.setRequestHeader(key, headers[key]) - } - // @ts-ignore - xhr.send(data) - }, - abort: function () { - jqXHR.abort() - }, - } - } -}) - -var loading : LoadingState = LoadingState.ready - -var viewer_width: number -var viewer_height: number -var viewer_border = 0 - -const canvas = $('#canvas')[0]! -const ctx = canvas.getContext('2d')! - function clickOnCanvas() { - clear() - stopPlaying() - setup(buffers[bufferKind]!) - lastClick = positionToAbsoluteTime(to(mousePosition().x)) - // @ts-ignore - play(timeInMovieToTimeInBuffer(lastClick), d3.event.shiftKey ? endS - startS : defaultPlayLength()) + clear() + stopPlaying() + setup(buffers[bufferKind]!) + lastClick = positionToAbsoluteTime(to(mousePosition().x)) + // @ts-ignore + play(timeInMovieToTimeInBuffer(lastClick), d3.event.shiftKey ? endS - startS : defaultPlayLength()) } function resizeCanvas() { - viewer_width = Math.min($('.panel').width()!, 3000) // 2240 // 1200 - viewer_height = Math.min(window.innerHeight * 0.5, 800) // 830 // 565 - viewer_border = 0 - canvas.width = viewer_width - canvas.height = viewer_height - $('#d3') - .attr('width', viewer_width) - .attr('height', viewer_height + viewer_border) - $('#container') - .css('width', viewer_width) - .css('height', viewer_height + viewer_border) + viewer_width = Math.min($('.panel').width()!, 3000) // 2240 // 1200 + viewer_height = Math.min(window.innerHeight * 0.5, 800) // 830 // 565 + viewer_border = 0 + canvas.width = viewer_width + canvas.height = viewer_height + $('#d3') + .attr('width', viewer_width) + .attr('height', viewer_height + viewer_border) + $('#container') + .css('width', viewer_width) + .css('height', viewer_height + viewer_border) } resizeCanvas() $(window).resize(() => { - stop() - resizeCanvas() + stop() + resizeCanvas() }) -var endTime = 100000 // infinity seconds.. - if (AudioContext) { - var context = new AudioContext() + context = new AudioContext() } else { - $('#loading').html( - '

Can\'t load audio context! Please use a recent free browser like the latest Chrome or Firefox.

' - ) + $('#loading').html( + '

Can\'t load audio context! Please use a recent free browser like the latest Chrome or Firefox.

' + ) } function setupAudioNodes() { - javascriptNode = context.createScriptProcessor(256, 1, 1) - javascriptNode.connect(context.destination) + javascriptNode = context!.createScriptProcessor(256, 1, 1) + javascriptNode.connect(context!.destination) } setupAudioNodes() function message(kind: string, msg: string) { - if(kind != 'success' && kind != 'warning') - recordMessage({level: kind, - data: msg}) + if (kind != 'success' && kind != 'warning') + recordMessage({ + level: kind, + data: msg + }) $('#loading') .html('

' + msg + '

') .removeClass('invisible') } -var segment: string -var startS: number -var endS: number -var movieName: string -var bufferKind: BufferType -// Fetched based on the segment -var words: string[] = [] -var mode: string -var token: string -var browser = navigator.userAgent.toString() -var other_annotations_by_worker: { [name: string]: Annotation[] } = {} // previous_annotation -// TODO Should expose this so that we can change the default -var current_reference_annotation = $.url().param().defaultReference - -// This has a race condition between stopping and start the audio, that's why we -// have a counter. 'onended' is called after starting a new audio playback, -// because the previous playback started. -var audioIsPlaying = 0 - -// For the transcript pane -var editingTranscriptMode = false - -function parseSegment(segmentName: string) { - const s = segmentName.split(':') - return { movieName: s[0], startTime: parseFloat(s[1]), endTime: parseFloat(s[2]) } -} - -function segmentString(details: { movieName: string; startTime: number; endTime: number }) { - return mkSegmentName(details.movieName, details.startTime, details.endTime) -} - function setSegment(segmentName: string) { - const s = segmentName.split(':') - segment = segmentName - movieName = s[0] - startS = parseFloat(s[1]) - endS = parseFloat(s[2]) + const s = segmentName.split(':') + segment = segmentName + movieName = s[0] + startS = parseFloat(s[1]) + endS = parseFloat(s[2]) } // TODO Check all properties here // TODO disable the default at some point if ($.url().param().token) { - token = $.url().param().token - $.ajax({ - type: 'POST', - data: JSON.stringify({ token: $.url().param().token }), - contentType: 'application/json', - async: false, - url: '/details', - success: function (data) { - if (data.response != 'ok') { - message('danger', 'Bad token!') - throw 'bad-token' - } - setSegment(data.segment) - }, - }) + token = $.url().param().token + $.ajax({ + type: 'POST', + data: JSON.stringify({ token: $.url().param().token }), + contentType: 'application/json', + async: false, + url: '/details', + success: function(data) { + if (data.response != 'ok') { + message('danger', 'Bad token!') + throw 'bad-token' + } + setSegment(data.segment) + }, + }) } else { - if ($.url().param().segment) setSegment($.url().param().segment) - else setSegment('test:0:1') + if ($.url().param().segment) setSegment($.url().param().segment) + else setSegment('test:0:1') } if ($.url().param().nohelp) $('#help-panel').remove() -function keyboardShortcutsOn() { - $(document).bind('keydown', 'p', () => { - clear() - recordKeypress('p') - $('#play').click() - }) - $(document).bind('keydown', 't', () => { - clear() - recordKeypress('w') - $('#stop').click() - }) - $(document).bind('keydown', 'd', () => { - clear() - recordKeypress('d') - $('#delete-selection').click() - }) - $(document).bind('keydown', 'y', () => { - clear() - recordKeypress('y') - $('#play-selection').click() - }) - $(document).bind('keydown', 'w', () => { - clear() - recordKeypress('w') - $('#start-next-word').click() - }) - $(document).bind('keydown', 'shift+w', () => { - clear() - recordKeypress('shift+y') - $('#start-next-word-after-current-word').click() - }) - $(document).bind('keydown', 'a', () => { - clear() - recordKeypress('a') - $('#toggle-speed').bootstrapSwitch('toggleState') - }) - $(document).bind('keydown', 'm', () => { - clear() - recordKeypress('m') - $('#toggle-audio').bootstrapSwitch('toggleState') - }) - $(document).bind('keydown', 'shift+b', () => { - clear() - recordKeypress('shift+b') - $('#back-save-4-sec').click() - }) - $(document).bind('keydown', 'b', () => { - clear() - recordKeypress('b') - $('#back-save-2-sec').click() - }) - $(document).bind('keydown', 'f', () => { - clear() - recordKeypress('f') - $('#forward-save-2-sec').click() - }) - $(document).bind('keydown', 'shift+f', () => { - clear() - recordKeypress('shift+f') - $('#forward-save-4-sec').click() - }) - $(document).bind('keydown', 's', () => { - clear() - recordKeypress('s') - $('#submit').click() - }) - $(document).bind('keydown', 'u', () => { - clear() - recordKeypress('u') - $('#fill-with-reference').click() - }) - $(document).bind('keydown', 't', e => { - clear() - recordKeypress('t') - $('#edit-transcript').click() - if (selected != null) { - const n = _.sum( - _.map( - _.filter(annotations, a => a.index < selected!), - a => a.word.length + 1 - ) - ) - // @ts-ignore - $('#transcript-input').focus()[0].setSelectionRange(n, n) - } else { - $('#transcript-input').focus() - } - e.preventDefault() - }) - $(document).bind('keydown', 'left', () => { - clear() - recordKeypress('left') - if (selected == null) { - const lastAnnotation = _.last(_.filter(annotations, isValidAnnotation)) - if (lastAnnotation) { - selectWord(lastAnnotation) - $('#play-selection').click() - } else { - message('warning', "Can't select the last word: no words are annotated") - return - } - } else { - const nextAnnotation = _.last(_.filter(_.take(annotations, selected), isValidAnnotation)) - if (nextAnnotation) { - selectWord(nextAnnotation) - $('#play-selection').click() - } else { - message('warning', 'At the first word, no other annotations to select') - return - } - } - }) - $(document).bind('keydown', 'right', () => { - clear() - recordKeypress('right') - if (selected == null) { - const firstAnnotation = _.head(_.filter(annotations, isValidAnnotation)) - if (firstAnnotation) { - selectWord(firstAnnotation) - $('#play-selection').click() - } else { - message('warning', "Can't select the first word: no words are annotated") - return - } - } else { - const nextAnnotation = _.head(_.filter(_.drop(annotations, selected + 1), isValidAnnotation)) - if (nextAnnotation) { - selectWord(nextAnnotation) - $('#play-selection').click() - } else { - message('warning', 'At the last word, no other annotations to select') - return - } - } - }) - $(document).bind('keydown', 'up', () => { - clear() - recordKeypress('up') - $('#play-selection').click() - }) - $(document).bind('keydown', 'down', () => { - clear() - recordKeypress('down') - $('#play-selection').click() - }) - $(document).bind('keydown', 'shift+left', () => { - clear() - recordKeypress('shift+left') - if (selected == null || !isValidAnnotation(annotations[selected])) { - message('warning', "Can't shift the start of the word earlier; no word is selected.") - return - } else { - annotations[selected].startTime = - verifyTranscriptOrder(selected, - subMax(annotations[selected].startTime!, keyboardShiftOffset, to(startS))) - updateWord(annotations[selected]) - } - }) - $(document).bind('keydown', 'shift+right', () => { - clear() - recordKeypress('shift+right') - if (selected == null || !isValidAnnotation(annotations[selected])) { - message('warning', "Can't shift the start of the word later; no word is selected.") - return - } else { - annotations[selected].startTime = verifyTranscriptOrder(selected, addMin( - annotations[selected].startTime!, - keyboardShiftOffset, - sub(annotations[selected].endTime!, keyboardShiftOffset))) - updateWord(annotations[selected]) - } - }) - $(document).bind('keydown', 'ctrl+left', () => { - clear() - recordKeypress('ctrl+left') - if (selected == null || !isValidAnnotation(annotations[selected])) { - message('warning', "Can't shift the end of the word earlier; no word is selected.") - return - } else { - annotations[selected].endTime = subMax( - annotations[selected].endTime!, - keyboardShiftOffset, - add(annotations[selected].startTime!, keyboardShiftOffset) - ) - updateWord(annotations[selected]) - } - }) - $(document).bind('keydown', 'ctrl+right', () => { - clear() - recordKeypress('ctrl+right') - if (selected == null || !isValidAnnotation(annotations[selected])) { - message('warning', "Can't shift the end of the word later; no word is selected.") - return - } else { - annotations[selected].endTime = addMin(annotations[selected].endTime!, keyboardShiftOffset, to(endS)) - updateWord(annotations[selected]) - } - }) - $(document).bind('keydown', 'shift+up', () => { - clear() - recordKeypress('shift+up') - if (selected == null || !isValidAnnotation(annotations[selected])) { - message('warning', "Can't shift the word later; no word is selected.") - return - } else { - annotations[selected].startTime = verifyTranscriptOrder(selected, addMin( - annotations[selected].startTime!, - keyboardShiftOffset, - sub(annotations[selected].endTime!, keyboardShiftOffset) - )) - annotations[selected].endTime = addMin(annotations[selected].endTime!, keyboardShiftOffset, to(endS)) - updateWord(annotations[selected]) - } - }) - $(document).bind('keydown', 'shift+down', () => { - clear() - recordKeypress('shift+down') - if (selected == null || !isValidAnnotation(annotations[selected])) { - message('warning', "Can't shift the word earlier; no word is selected.") - return - } else { - annotations[selected].startTime = verifyTranscriptOrder(selected, - subMax(annotations[selected].startTime!, keyboardShiftOffset, to(startS))) - annotations[selected].endTime = subMax( - annotations[selected].endTime!, - keyboardShiftOffset, - add(annotations[selected].startTime!, keyboardShiftOffset) - ) - updateWord(annotations[selected]) - } - }) -} - function tokenMode() { - stopPlaying() - mode = 'token' - bufferKind = BufferType.normal - $('.transcription-gui').addClass('display-none') - $('.annotation-gui').addClass('display-none') - // TOOD Maybe ressurect this one day? - // keyboardShortcutsOff() + stopPlaying() + mode = 'token' + bufferKind = BufferType.normal + $('.transcription-gui').addClass('display-none') + $('.annotation-gui').addClass('display-none') + // TOOD Maybe ressurect this one day? + // keyboardShortcutsOff() } function annotationMode() { - mode = 'annotation' - bufferKind = BufferType.normal - $('.transcription-gui').addClass('display-none') - $('.annotation-gui').removeClass('display-none') - keyboardShortcutsOn() - // TODO This is blocked by browsers anyway - // if(sourceNode) { - // stopPlaying() - // play(0) - // } + mode = 'annotation' + bufferKind = BufferType.normal + $('.transcription-gui').addClass('display-none') + $('.annotation-gui').removeClass('display-none') + keyboardShortcutsOn() + // TODO This is blocked by browsers anyway + // if(sourceNode) { + // stopPlaying() + // play(0) + // } } annotationMode() // delay between hearing a word, figuring out that it's the one // you want, pressing the button and the event firing -var fixedButtonOffset = 0.05 function defaultPlayLength(): TimeInBuffer { - switch (bufferKind) { - case BufferType.half: - return to(1.4) - case BufferType.normal: - return to(0.7) - } -} - -var buffers: Buffers = { normal: null, half: null } -var sourceNode: AudioBufferSourceNode -var javascriptNode: ScriptProcessorNode -var startTime: TimeInBuffer = to(0) -var startOffset: TimeInBuffer = to(0) -var lastClick: TimeInMovie | null = null -var selected: number | null = null -var annotations: Annotation[] = [] -var mute: boolean = false -const keyboardShiftOffset: TimeInMovie = to(0.01) -const handleOffset = 0 - -let dragStart: TimeInMovie | null = null -var svg = d3.select('#d3') -svg - .append('rect') - .attr('width', '100%') - .attr('height', '100%') - .attr('fill', '#ffffff') - .attr('fill-opacity', 0.0) - .call( - d3.behavior - .drag() - .on('dragstart', () => { - // @ts-ignore - recordMouseClick(d3.event.sourceEvent, "#d3", "dragstart") - // @ts-ignore - const x = d3.event.sourceEvent.layerX - lastClick = positionToAbsoluteTime(to(x)) - dragStart = lastClick - redraw() - }) - .on('dragend', () => { - // @ts-ignore - recordMouseClick(d3.event.sourceEvent, "d3", "dragend") - // @ts-ignore - const x = d3.event.sourceEvent.layerX - // @ts-ignore - const shift: bool = d3.event.sourceEvent.shiftKey - lastClick = positionToAbsoluteTime(to(x)) - const boundary1: TimeInMovie = dragStart! - const boundary2: TimeInMovie = lastClick! - dragStart = null - let start: TimeInMovie - let end: TimeInMovie - if (Math.abs(from(sub(boundary1!, boundary2!))) > 0.02) { - if (from(sub(boundary1!, boundary2)) < 0) { - start = boundary1! - end = boundary2! - } else { - start = boundary2! - end = boundary1! - } - } else { - start = lastClick - if (shift) { - end = to(endS) - } else { - end = to(Math.min(from(start) + from(defaultPlayLength()), endS)) - } - } - clear() - stopPlaying() - setup(buffers[bufferKind]!) - play(timeInMovieToTimeInBuffer(start), sub(timeInMovieToTimeInBuffer(end), timeInMovieToTimeInBuffer(start))) - redraw() - }) - .on('drag', () => { - // @ts-ignore - const x = d3.event.sourceEvent.layerX - lastClick = positionToAbsoluteTime(to(x)) - redraw() - }) - ) - -var svgReferenceAnnotations: d3.Selection = svg.append('g') -var svgAnnotations: d3.Selection = svg.append('g') -let lastChangedAnnotations: Annotation[] = [] - -function pushUndo(annotation: Annotation) { - lastChangedAnnotations.push(_.clone(annotation)) -} - -function popUndo() { - const last = _.last(lastChangedAnnotations) - lastChangedAnnotations = lastChangedAnnotations.slice(0, -1) - return last -} - -function clearUndo() { - lastChangedAnnotations = [] -} - -function undo() { - if (lastChangedAnnotations != []) { - const ann = popUndo()! - annotations[ann.index].startTime = ann.startTime - annotations[ann.index].endTime = ann.endTime - updateWord(annotations[ann.index]) - } else { - message('warning', 'Nothing to undo') - } + switch (bufferKind) { + case BufferType.half: + return to(1.4) + case BufferType.normal: + return to(0.7) + } } function verifyStartBeforeEnd(index: number, startTime: TimeInMovie) { - if(annotations[index] && from(annotations[index].endTime!)-0.01 <= from(startTime)) { + if (annotations[index] && from(annotations[index].endTime!) - 0.01 <= from(startTime)) { message('warning', "The start of word would be after the end") throw "The start of word would be after the end" } @@ -1015,7 +194,7 @@ function verifyStartBeforeEnd(index: number, startTime: TimeInMovie) { } function verifyEndAfterStart(index: number, endTime: TimeInMovie) { - if(annotations[index] && from(annotations[index].startTime!)+0.01 >= from(endTime)) { + if (annotations[index] && from(annotations[index].startTime!) + 0.01 >= from(endTime)) { message('warning', "The end of word would be before the start") throw "The end of word would be before the start" } @@ -1024,191 +203,174 @@ function verifyEndAfterStart(index: number, endTime: TimeInMovie) { function verifyTranscriptOrder(index: number, time: TimeInMovie) { // Words that appear before this one the transcript should have earlier start times - if(_.filter(annotations, - a => isValidAnnotation(a) - && (from(a.startTime!) > from(time) && a.index < index)) - .length > 0) { + if (_.filter(annotations, + a => isValidAnnotation(a) + && (from(a.startTime!) > from(time) && a.index < index)) + .length > 0) { message('warning', "This word would start before a word that is earlier in the transcript") throw "This word would start before a word that is earlier in the transcript" } else // Words that appear before this one the transcript should have earlier start times - if(_.filter(annotations, - a => isValidAnnotation(a) - && (from(a.startTime!) < from(time) && a.index > index)) - .length > 0) { - message('warning', "This word would start after a word that is later in the transcript") - throw "This word would start after a word that is later in the transcript" - } else { - return time - } + if (_.filter(annotations, + a => isValidAnnotation(a) + && (from(a.startTime!) < from(time) && a.index > index)) + .length > 0) { + message('warning', "This word would start after a word that is later in the transcript") + throw "This word would start after a word that is later in the transcript" + } else { + return time + } } -svgReferenceAnnotations.on('click', function (_d, _i) { - console.log("Q") - console.log(_d) - console.log(_i) - clickOnCanvas() +svgReferenceAnnotations.on('click', function(_d, _i) { + clickOnCanvas() }) -svgAnnotations.on('click', function (_d, _i) { - console.log("Y") - console.log(_d) - console.log(_i) - clickOnCanvas() +svgAnnotations.on('click', function(_d, _i) { + clickOnCanvas() }) function drag(annotation: Annotation, position: DragPosition) { - return d3.behavior - .drag() - .on('dragstart', () => pushUndo(annotation)) - .on('drag', function () { - // @ts-ignore - const dx = d3.event.dx - switch (position) { - case DragPosition.start: - annotation.startTime = verifyStartBeforeEnd(annotation.index, verifyTranscriptOrder(annotation.index, addConst(annotation.startTime!, from(positionToTime(to(dx)))))) - break; - case DragPosition.end: - annotation.endTime = verifyEndAfterStart(annotation.index, addConst(annotation.endTime!, from(positionToTime(to(dx))))) - break - case DragPosition.both: - annotation.startTime = verifyTranscriptOrder(annotation.index, addConst(annotation.startTime!, from(positionToTime(to(dx))))) - annotation.endTime = addConst(annotation.endTime!, from(positionToTime(to(dx)))) - break - } - updateWord(annotation) - }) - .on('dragend', function (_d, _i) { - selectWord(annotation) - $('#play-selection').click() - }) + return d3.behavior + .drag() + .on('dragstart', () => pushUndo(annotation)) + .on('drag', function() { + // @ts-ignore + const dx = d3.event.dx + switch (position) { + case DragPosition.start: + annotation.startTime = verifyStartBeforeEnd(annotation.index, verifyTranscriptOrder(annotation.index, addConst(annotation.startTime!, from(positionToTime(to(dx)))))) + break; + case DragPosition.end: + annotation.endTime = verifyEndAfterStart(annotation.index, addConst(annotation.endTime!, from(positionToTime(to(dx))))) + break + case DragPosition.both: + annotation.startTime = verifyTranscriptOrder(annotation.index, addConst(annotation.startTime!, from(positionToTime(to(dx))))) + annotation.endTime = addConst(annotation.endTime!, from(positionToTime(to(dx)))) + break + } + updateWord(annotation) + }) + .on('dragend', function(_d, _i) { + selectWord(annotation) + $('#play-selection').click() + }) } function loadSound(url: string, kind: BufferType, fn: any) { - var request = new XMLHttpRequest() - request.open('GET', url, true) - request.responseType = 'arraybuffer' - request.onload = () => { - context.decodeAudioData( - request.response, - function (audioBuffer) { - buffers[kind] = audioBuffer - setup(buffers[kind]!) - if (fn) { - fn() - } - }, - onError - ) - } - request.send() + var request = new XMLHttpRequest() + request.open('GET', url, true) + request.responseType = 'arraybuffer' + request.onload = () => { + context!.decodeAudioData( + request.response, + function(audioBuffer) { + buffers[kind] = audioBuffer + setup(buffers[kind]!) + if (fn) { + fn() + } + }, + onError + ) + } + request.send() } function clear() { - $('#loading').addClass('invisible') + $('#loading').addClass('invisible') } function setup(buffer: AudioBuffer) { - sourceNode = context.createBufferSource() - sourceNode.connect(javascriptNode) - sourceNode.buffer = buffer - startTime = to(context.currentTime) - sourceNode.onended = () => { - audioIsPlaying -= 1 - redraw() - } - if (!mute) sourceNode.connect(context.destination) - // Maybe? - // sourceNode.playbackRate.value = 0.5 - // sourceNode.loop = true + sourceNode = context!.createBufferSource() + sourceNode.connect(javascriptNode) + sourceNode.buffer = buffer + startTime = to(context!.currentTime) + sourceNode.onended = () => { + audioIsPlaying -= 1 + redraw() + } + if (!mute) sourceNode.connect(context!.destination) + // Maybe? + // sourceNode.playbackRate.value = 0.5 + // sourceNode.loop = true } function play(offset_: TimeInBuffer, duration_?: TimeInBuffer) { - const offset = from(offset_) - startTime = to(context.currentTime) - startOffset = offset_ - if (duration_ != null) { - const duration = from(duration_) - endTime = offset + duration - audioIsPlaying += 1 - sourceNode.start(0, offset, duration) - } else { - endTime = 1000000 // infinity seconds.. - audioIsPlaying += 1 - sourceNode.start(0, offset) - } + const offset = from(offset_) + startTime = to(context!.currentTime) + startOffset = offset_ + if (duration_ != null) { + const duration = from(duration_) + endTime = offset + duration + audioIsPlaying += 1 + sourceNode.start(0, offset, duration) + } else { + endTime = 1000000 // infinity seconds.. + audioIsPlaying += 1 + sourceNode.start(0, offset) + } } function stopPlaying() { - // Might need to do: player.sourceNode.noteOff(0) on some browsers? - try { - sourceNode.stop(0) - startOffset = to(context.currentTime - from(add(startTime, startOffset))) - redraw() - } catch (err) { - // Calling stop more than once should be safe, although - // catching all errors is bad form - } + // Might need to do: player.sourceNode.noteOff(0) on some browsers? + try { + sourceNode.stop(0) + startOffset = to(context!.currentTime - from(add(startTime, startOffset))) + redraw() + } catch (err) { + // Calling stop more than once should be safe, although + // catching all errors is bad form + } } function onError(e: any) { - console.log(e) + console.log(e) } function timeInMovieToPercent(time: TimeInMovie): string { - return 100 * ((from(time) - startS) / (endS - startS)) + '%' + return 100 * ((from(time) - startS) / (endS - startS)) + '%' } function timeInMovieToTimeInBuffer(time: TimeInMovie): TimeInBuffer { - return positionToTimeInBuffer(absoluteTimeToPosition(time)) + return positionToTimeInBuffer(absoluteTimeToPosition(time)) } function absoluteTimeToPosition(time: TimeInMovie): PositionInSpectrogram { - return to(((from(time) - startS) / (endS - startS)) * canvas.width) + return to(((from(time) - startS) / (endS - startS)) * canvas.width) } function timeToPosition(time: TimeInSegment): PositionInSpectrogram { - return to((from(time) / (endS - startS)) * canvas.width) + return to((from(time) / (endS - startS)) * canvas.width) } function timeInBufferToPosition(time: TimeInBuffer): PositionInSpectrogram { - return to((from(time) / sourceNode.buffer!.duration) * canvas.width) + return to((from(time) / sourceNode.buffer!.duration) * canvas.width) } function timeInMovieToPosition(time: TimeInMovie): PositionInSpectrogram { - return to(((from(time) - startS) / (endS - startS)) * canvas.width) + return to(((from(time) - startS) / (endS - startS)) * canvas.width) } function positionToTime(position: PositionInSpectrogram): TimeInSegment { - return to((from(position) * (endS - startS)) / canvas.width) + return to((from(position) * (endS - startS)) / canvas.width) } function positionToTimeInBuffer(position: PositionInSpectrogram): TimeInBuffer { - return to((from(position) * sourceNode.buffer!.duration) / canvas.width) + return to((from(position) * sourceNode.buffer!.duration) / canvas.width) } function positionToAbsoluteTime(position: PositionInSpectrogram): TimeInMovie { - return to(startS + (from(position) * (endS - startS)) / canvas.width) + return to(startS + (from(position) * (endS - startS)) / canvas.width) } function redraw(timeOffset?: TimeInBuffer) { - ctx.clearRect(0, 0, canvas.width, canvas.height) - if (timeOffset != null && from(timeOffset) < endTime) { - var offset = timeInBufferToPosition(timeOffset) - ctx.fillStyle = 'rgba(255, 255, 255, 0.9)' - ctx.fillRect(from(offset), 1, 1, canvas.height) - } - if (lastClick != null) { - ctx.fillStyle = 'rgba(200, 0, 0, 0.9)' - ctx.fillRect(from(timeInMovieToPosition(lastClick)), 1, 2, canvas.height) - } - if (dragStart != null) { - ctx.fillStyle = 'rgba(200, 0, 0, 0.9)' - ctx.fillRect(from(timeInMovieToPosition(dragStart)), 1, 2, canvas.height) - } -} - -function mousePosition() { - // @ts-ignore - const x = d3.event.layerX - // @ts-ignore - const y = d3.event.layerY - return { - x: x, - y: y, - } + ctx.clearRect(0, 0, canvas.width, canvas.height) + if (timeOffset != null && from(timeOffset) < endTime) { + var offset = timeInBufferToPosition(timeOffset) + ctx.fillStyle = 'rgba(255, 255, 255, 0.9)' + ctx.fillRect(from(offset), 1, 1, canvas.height) + } + if (lastClick != null) { + ctx.fillStyle = 'rgba(200, 0, 0, 0.9)' + ctx.fillRect(from(timeInMovieToPosition(lastClick)), 1, 2, canvas.height) + } + if (dragStart != null) { + ctx.fillStyle = 'rgba(200, 0, 0, 0.9)' + ctx.fillRect(from(timeInMovieToPosition(dragStart)), 1, 2, canvas.height) + } } function wordClickHandler(index: number) { @@ -1217,134 +379,77 @@ function wordClickHandler(index: number) { selectWord(annotation) $('#play-selection').click() } else { - if (lastClick != null) { - selectWord(startWord(index, lastClick)) - $('#play-selection').click() - } else if (selected != null && annotations[selected].endTime != null) { - selectWord(startWord(index, addConst(annotations[selected].endTime!, (Math.max(0, (index - selected - 1)) * 0.1)))) - $('#play-selection').click() - } else message('warning', 'Place the marker first by clicking on the image') + if (lastClick != null) { + selectWord(startWord(index, lastClick)) + $('#play-selection').click() + } else if (selected != null && annotations[selected].endTime != null) { + selectWord(startWord(index, addConst(annotations[selected].endTime!, (Math.max(0, (index - selected - 1)) * 0.1)))) + $('#play-selection').click() + } else message('warning', 'Place the marker first by clicking on the image') } } function updateWords(words: string[]) { - $('#words').empty() - annotations = [] - _.forEach(words, function (word, index) { - annotations[index] = { index: index, word: word } - $('#words') - .append($('').append($('').text(word).data('index', index))) - .append(' ') - }) + $('#words').empty() + annotations = [] + _.forEach(words, function(word, index) { + annotations[index] = { index: index, word: word } + $('#words') + .append($('').append($('').text(word).data('index', index))) + .append(' ') + }) $('.word').click(function(e) { clear() e.preventDefault() const index = $(this).data('index') - recordMouseClick(e, ".word", index+''); + recordMouseClick(e, ".word", index + ''); wordClickHandler(index) }) } -function levenshteinAlignment( - iWords: string[], - i: number, - jWords: string[], - j: number, - cache: (number | boolean)[][] -): any { - if (cache[i][j] !== false) { - return cache[i][j] - } - let out - if (i >= iWords.length) { - out = { distance: Math.abs(jWords.length - j) } - } else if (j >= jWords.length) { - out = { distance: Math.abs(iWords.length - i) } - } else { - let ret1 = _.clone(levenshteinAlignment(iWords, i + 1, jWords, j, cache)) - ret1.distance += 1 - let ret2 = _.clone(levenshteinAlignment(iWords, i, jWords, j + 1, cache)) - ret2.distance += 1 - let ret3 = _.clone(levenshteinAlignment(iWords, i + 1, jWords, j + 1, cache)) - if (iWords[i] === jWords[j]) ret3[i] = j - else ret3.distance += 1 - if (ret1.distance < ret2.distance && ret1.distance < ret3.distance) { - out = ret1 - } else if (ret2.distance < ret1.distance && ret2.distance < ret3.distance) { - out = ret2 - } else { - out = ret3 - } - } - cache[i][j] = out - return out -} - -function alignWords(newWords: string[], oldWords: string[]): any { - let cache: (number | boolean)[][] = [] - _.forEach(_.range(0, newWords.length + 2), i => { - cache[i] = [] - _.forEach(_.range(0, oldWords.length + 2), j => { - cache[i][j] = false - }) - }) - return levenshteinAlignment(newWords, 0, oldWords, 0, cache) -} - -// This clones without the UI elements -function cloneAnnotation(a: Annotation): Annotation { - return { - startTime: a.startTime, - endTime: a.endTime, - lastClickTimestamp: a.lastClickTimestamp, - word: a.word, - index: a.index, - } -} - function updateWordsWithAnnotations(newWords: string[]) { - $('#words').empty() - const oldWords = words - const oldAnnotations = _.cloneDeep(annotations) - _.forEach(annotations, removeAnnotation) - const alignment = alignWords(newWords, oldWords) - words = newWords - annotations = [] - _.forEach(words, function (word, index) { - annotations[index] = { word: word, index: index } - if (_.has(alignment, index)) { - const old = oldAnnotations[alignment[index]] - annotations[index].startTime = old.startTime - annotations[index].endTime = old.endTime - annotations[index].lastClickTimestamp = old.lastClickTimestamp - } else if (oldWords.length == newWords.length) { - // If there is no alignment but the number of words is unchanged, then - // we replaced one or more words. We preserve the annotations in that - // case. - const old = oldAnnotations[index] - annotations[index].startTime = old.startTime - annotations[index].endTime = old.endTime - annotations[index].lastClickTimestamp = old.lastClickTimestamp - } - }) - _.forEach(words, function (word, index) { - $('#words') - .append($('').append($('').text(word).data('index', index))) - .append(' ') - }) - $('.word').click(function (e) { + $('#words').empty() + const oldWords = words + const oldAnnotations = _.cloneDeep(annotations) + _.forEach(annotations, removeAnnotation) + const alignment = alignWords(newWords, oldWords) + words = newWords + annotations = [] + _.forEach(words, function(word, index) { + annotations[index] = { word: word, index: index } + if (_.has(alignment, index)) { + const old = oldAnnotations[alignment[index]] + annotations[index].startTime = old.startTime + annotations[index].endTime = old.endTime + annotations[index].lastClickTimestamp = old.lastClickTimestamp + } else if (oldWords.length == newWords.length) { + // If there is no alignment but the number of words is unchanged, then + // we replaced one or more words. We preserve the annotations in that + // case. + const old = oldAnnotations[index] + annotations[index].startTime = old.startTime + annotations[index].endTime = old.endTime + annotations[index].lastClickTimestamp = old.lastClickTimestamp + } + }) + _.forEach(words, function(word, index) { + $('#words') + .append($('').append($('').text(word).data('index', index))) + .append(' ') + }) + $('.word').click(function(e) { clear() e.preventDefault() const index = $(this).data('index') - recordMouseClick(e, ".word", index+''); + recordMouseClick(e, ".word", index + ''); wordClickHandler(index) - }) - _.forEach(annotations, updateWord) + }) + _.forEach(annotations, updateWord) } function startWord(index: number, time: TimeInMovie) { if ( - !_.find(annotations, function (key) { + !_.find(annotations, function(key) { return key.index != index && key.startTime == time }) ) { @@ -1368,39 +473,38 @@ function startWord(index: number, time: TimeInMovie) { } function closestWord(time: TimeInMovie) { - return _.sortBy( - _.filter(annotations, function (annotation: Annotation) { - return annotation.startTime != null && annotation.startTime < time - }), - function (annotation: Annotation, _index: number) { - return sub(time, annotation.startTime!) - } - )[0] + return _.sortBy( + _.filter(annotations, function(annotation: Annotation) { + return annotation.startTime != null && annotation.startTime < time + }), + function(annotation: Annotation, _index: number) { + return sub(time, annotation.startTime!) + } + )[0] } function annotationColor(annotation: Annotation, isReference: boolean) { - if (isReference) return '#5bc0de' - if (annotation.endTime != null) { - if (annotation.index == selected) return 'orange' - else return '#6fe200' - } else { - return 'red' - } + if (isReference) return '#5bc0de' + if (annotation.endTime != null) { + if (annotation.index == selected) return 'orange' + else return '#6fe200' + } else { + return 'red' + } } function clearWordLabels(annotation: Annotation) { - $('.word') - .eq(annotation.index) - .removeClass('label-success') - .removeClass('label-warning') - .removeClass('label-info') - .removeClass('label-primary') - .removeClass('label-danger') + $('.word') + .eq(annotation.index) + .removeClass('label-success') + .removeClass('label-warning') + .removeClass('label-info') + .removeClass('label-primary') + .removeClass('label-danger') } function handleClickOnAnnotatedWord(annotation: Annotation, isReference: boolean) { - return (e : any, j : any) => { - console.log(e) + return (e: any, j: any) => { recordMouseClick(e, "#click-on-word", [cloneAnnotation(annotation), isReference]); // @ts-ignore d3.event.stopPropagation() @@ -1415,323 +519,323 @@ function handleClickOnAnnotatedWord(annotation: Annotation, isReference: boolean } function handleDragOnAnnotatedWord(annotation: Annotation, isReference: boolean, position: DragPosition) { - if (!isReference) return drag(annotation, position) - else return () => null + if (!isReference) return drag(annotation, position) + else return () => null } function updateWordBySource(annotation: Annotation, isReference: boolean, worker: string) { - if (annotation.visuals == null) - // @ts-ignore - annotation.visuals = {} - // NB This check is redudant but it makes typescript understand that annotation.visuals != null - if (annotation.visuals != null) { - if (annotation.startTime != null) { - if (!isReference) { - clearWordLabels(annotation) - if (annotation.endTime == null) $('.word').eq(annotation.index).addClass('label-danger') - else if (annotation.index == selected) $('.word').eq(annotation.index).addClass('label-warning') - else $('.word').eq(annotation.index).addClass('label-success') - } - if (!annotation.visuals.group) { - annotation.visuals.group = (isReference ? svgReferenceAnnotations : svgAnnotations).append('g') - annotation.id = isReference ? worker + ':' + annotation.startTime : from(annotation.startTime) - annotation.visuals.group.datum(isReference ? worker + ':' + annotation.startTime : annotation.index) - } - if (!annotation.visuals.text) - annotation.visuals.text = annotation.visuals.group.append('text').text(annotation.word) - annotation.visuals.text - .attr('filter', 'url(#blackOutlineEffect)') - .attr('font-family', 'sans-serif') - .attr('font-size', '15px') - .attr('class', 'unselectable') - .attr('fill', annotationColor(annotation, isReference)) - .on('click', handleClickOnAnnotatedWord(annotation, isReference)) - if (!annotation.visuals.startLine) { - annotation.visuals.startLine = annotation.visuals.group.append('line') - annotation.visuals.startLineHandle = annotation.visuals.group - .append('line') - .call( - // @ts-ignore - handleDragOnAnnotatedWord(annotation, isReference, DragPosition.start) - ) - .on('click', handleClickOnAnnotatedWord(annotation, isReference)) - } - annotation.visuals.startLine - .attr('x1', timeInMovieToPercent(annotation.startTime!)) - .attr('x2', timeInMovieToPercent(annotation.startTime!)) - .attr('y1', heightBottom(isReference)) - .attr('y2', heightTop(isReference)) - .attr('stroke', annotationColor(annotation, isReference)) - .attr('opacity', 0.7) - .attr('stroke-width', '2') - annotation.visuals.startLineHandle - .attr('x1', timeInMovieToPercent(subConst(annotation.startTime!, handleOffset))) - .attr('x2', timeInMovieToPercent(subConst(annotation.startTime!, handleOffset))) - .attr('y1', heightBottom(isReference)) - .attr('y2', heightTop(isReference)) - .attr('stroke', annotationColor(annotation, isReference)) - .attr('opacity', 0) - .attr('stroke-width', '12') - .attr('name', 'startLine') - if (annotation.endTime != null) { - if (!annotation.visuals.filler) { - annotation.visuals.filler = annotation.visuals.group - .insert('rect', ':first-child') - .call( - // @ts-ignore - handleDragOnAnnotatedWord(annotation, isReference, DragPosition.both) - ) - .on('click', handleClickOnAnnotatedWord(annotation, isReference)) - } - annotation.visuals.filler - .attr('x', timeInMovieToPercent(annotation.startTime!)) - .attr('y', heightBottom(isReference)) - .attr('width', timeInMovieToPercent(addConst(sub(annotation.endTime!, annotation.startTime!), startS))) - .attr('height', heightTop(isReference)) - .attr('opacity', isReference ? 0 : 0.1) - .attr('stroke', annotationColor(annotation, isReference)) - .attr('fill', annotationColor(annotation, isReference)) - if (!annotation.visuals.endLine) { - annotation.visuals.endLine = annotation.visuals.group.append('line') - annotation.visuals.endLineHandle = annotation.visuals.group - .append('line') - .call( - // @ts-ignore - handleDragOnAnnotatedWord(annotation, isReference, DragPosition.end) - ) - .on('click', handleClickOnAnnotatedWord(annotation, isReference)) - } - annotation.visuals.endLine - .attr('x1', timeInMovieToPercent(annotation.endTime!)) - .attr('x2', timeInMovieToPercent(annotation.endTime!)) - .attr('y1', heightBottom(isReference)) - .attr('y2', heightTop(isReference)) - .attr('stroke', annotationColor(annotation, isReference)) - .attr('opacity', 1) - .attr('stroke-width', '2') - annotation.visuals.endLineHandle - .attr('x1', timeInMovieToPercent(addConst(annotation.endTime!, handleOffset))) - .attr('x2', timeInMovieToPercent(addConst(annotation.endTime!, handleOffset))) - .attr('y1', heightBottom(isReference)) - .attr('y2', heightTop(isReference)) - .attr('stroke', annotationColor(annotation, isReference)) - .attr('opacity', 0) - .attr('stroke-width', '12') - // .attr('name', 'endLine') - if (!annotation.visuals.topLine) annotation.visuals.topLine = annotation.visuals.group.append('line') - annotation.visuals.topLine - .attr('x1', timeInMovieToPercent(annotation.startTime!)) - .attr('x2', timeInMovieToPercent(annotation.endTime!)) - .attr('y1', heightTopLine(isReference)) - .attr('y2', heightTopLine(isReference)) - .attr('stroke', annotationColor(annotation, isReference)) - .attr('opacity', 0.7) - .style('stroke-dasharray', '3, 3') - .attr('stroke-width', '2') - let textLocationTime = from(sub(annotation.endTime!, annotation.startTime!)) / 2 + from(annotation.startTime!) - if(from(annotation.startTime) < startS) { - textLocationTime = startS - annotation.visuals.text.attr('text-anchor', 'start') - } else if(from(annotation.endTime) > endS) { - textLocationTime = endS - annotation.visuals.text.attr('text-anchor', 'end') + if (annotation.visuals == null) + // @ts-ignore + annotation.visuals = {} + // NB This check is redudant but it makes typescript understand that annotation.visuals != null + if (annotation.visuals != null) { + if (annotation.startTime != null) { + if (!isReference) { + clearWordLabels(annotation) + if (annotation.endTime == null) $('.word').eq(annotation.index).addClass('label-danger') + else if (annotation.index == selected) $('.word').eq(annotation.index).addClass('label-warning') + else $('.word').eq(annotation.index).addClass('label-success') + } + if (!annotation.visuals.group) { + annotation.visuals.group = (isReference ? svgReferenceAnnotations : svgAnnotations).append('g') + annotation.id = isReference ? worker + ':' + annotation.startTime : from(annotation.startTime) + annotation.visuals.group.datum(isReference ? worker + ':' + annotation.startTime : annotation.index) + } + if (!annotation.visuals.text) + annotation.visuals.text = annotation.visuals.group.append('text').text(annotation.word) + annotation.visuals.text + .attr('filter', 'url(#blackOutlineEffect)') + .attr('font-family', 'sans-serif') + .attr('font-size', '15px') + .attr('class', 'unselectable') + .attr('fill', annotationColor(annotation, isReference)) + .on('click', handleClickOnAnnotatedWord(annotation, isReference)) + if (!annotation.visuals.startLine) { + annotation.visuals.startLine = annotation.visuals.group.append('line') + annotation.visuals.startLineHandle = annotation.visuals.group + .append('line') + .call( + // @ts-ignore + handleDragOnAnnotatedWord(annotation, isReference, DragPosition.start) + ) + .on('click', handleClickOnAnnotatedWord(annotation, isReference)) + } + annotation.visuals.startLine + .attr('x1', timeInMovieToPercent(annotation.startTime!)) + .attr('x2', timeInMovieToPercent(annotation.startTime!)) + .attr('y1', heightBottom(isReference)) + .attr('y2', heightTop(isReference)) + .attr('stroke', annotationColor(annotation, isReference)) + .attr('opacity', 0.7) + .attr('stroke-width', '2') + annotation.visuals.startLineHandle + .attr('x1', timeInMovieToPercent(subConst(annotation.startTime!, handleOffset))) + .attr('x2', timeInMovieToPercent(subConst(annotation.startTime!, handleOffset))) + .attr('y1', heightBottom(isReference)) + .attr('y2', heightTop(isReference)) + .attr('stroke', annotationColor(annotation, isReference)) + .attr('opacity', 0) + .attr('stroke-width', '12') + .attr('name', 'startLine') + if (annotation.endTime != null) { + if (!annotation.visuals.filler) { + annotation.visuals.filler = annotation.visuals.group + .insert('rect', ':first-child') + .call( + // @ts-ignore + handleDragOnAnnotatedWord(annotation, isReference, DragPosition.both) + ) + .on('click', handleClickOnAnnotatedWord(annotation, isReference)) + } + annotation.visuals.filler + .attr('x', timeInMovieToPercent(annotation.startTime!)) + .attr('y', heightBottom(isReference)) + .attr('width', timeInMovieToPercent(addConst(sub(annotation.endTime!, annotation.startTime!), startS))) + .attr('height', heightTop(isReference)) + .attr('opacity', isReference ? 0 : 0.1) + .attr('stroke', annotationColor(annotation, isReference)) + .attr('fill', annotationColor(annotation, isReference)) + if (!annotation.visuals.endLine) { + annotation.visuals.endLine = annotation.visuals.group.append('line') + annotation.visuals.endLineHandle = annotation.visuals.group + .append('line') + .call( + // @ts-ignore + handleDragOnAnnotatedWord(annotation, isReference, DragPosition.end) + ) + .on('click', handleClickOnAnnotatedWord(annotation, isReference)) + } + annotation.visuals.endLine + .attr('x1', timeInMovieToPercent(annotation.endTime!)) + .attr('x2', timeInMovieToPercent(annotation.endTime!)) + .attr('y1', heightBottom(isReference)) + .attr('y2', heightTop(isReference)) + .attr('stroke', annotationColor(annotation, isReference)) + .attr('opacity', 1) + .attr('stroke-width', '2') + annotation.visuals.endLineHandle + .attr('x1', timeInMovieToPercent(addConst(annotation.endTime!, handleOffset))) + .attr('x2', timeInMovieToPercent(addConst(annotation.endTime!, handleOffset))) + .attr('y1', heightBottom(isReference)) + .attr('y2', heightTop(isReference)) + .attr('stroke', annotationColor(annotation, isReference)) + .attr('opacity', 0) + .attr('stroke-width', '12') + // .attr('name', 'endLine') + if (!annotation.visuals.topLine) annotation.visuals.topLine = annotation.visuals.group.append('line') + annotation.visuals.topLine + .attr('x1', timeInMovieToPercent(annotation.startTime!)) + .attr('x2', timeInMovieToPercent(annotation.endTime!)) + .attr('y1', heightTopLine(isReference)) + .attr('y2', heightTopLine(isReference)) + .attr('stroke', annotationColor(annotation, isReference)) + .attr('opacity', 0.7) + .style('stroke-dasharray', '3, 3') + .attr('stroke-width', '2') + let textLocationTime = from(sub(annotation.endTime!, annotation.startTime!)) / 2 + from(annotation.startTime!) + if (from(annotation.startTime) < startS) { + textLocationTime = startS + annotation.visuals.text.attr('text-anchor', 'start') + } else if (from(annotation.endTime) > endS) { + textLocationTime = endS + annotation.visuals.text.attr('text-anchor', 'end') + } else { + annotation.visuals.text.attr('text-anchor', 'middle') + } + annotation.visuals.text + .attr('x', timeInMovieToPercent(to(textLocationTime))) + .attr('y', heightText(isReference)) + } else { + annotation.visuals.text.attr('x', timeInMovieToPercent(addConst(annotation.startTime!, 0.1))).attr('y', '55%') + } } else { - annotation.visuals.text.attr('text-anchor', 'middle') + $('.word').eq(annotation.index).addClass('label-danger') } - annotation.visuals.text - .attr('x', timeInMovieToPercent(to(textLocationTime))) - .attr('y', heightText(isReference)) - } else { - annotation.visuals.text.attr('x', timeInMovieToPercent(addConst(annotation.startTime!, 0.1))).attr('y', '55%') - } - } else { - $('.word').eq(annotation.index).addClass('label-danger') } - } } function updateWord(annotation: Annotation) { - updateWordBySource(annotation, false, $.url().param().worker) + updateWordBySource(annotation, false, $.url().param().worker) } function removeAnnotation(annotation: Annotation) { - if (annotation.visuals) { - if (annotation.visuals.startLine) { - annotation.visuals.startLine.remove() - annotation.visuals.startLineHandle.remove() - } - if (annotation.visuals.endLine) { - annotation.visuals.endLine.remove() - annotation.visuals.endLineHandle.remove() - } - if (annotation.visuals.filler) { - annotation.visuals.filler.remove() - } - if (annotation.visuals.topLine) { - annotation.visuals.topLine.remove() - } - if (annotation.visuals.text) { - annotation.visuals.text.remove() - } - if (annotation.visuals.group) { - annotation.visuals.group.remove() + if (annotation.visuals) { + if (annotation.visuals.startLine) { + annotation.visuals.startLine.remove() + annotation.visuals.startLineHandle.remove() + } + if (annotation.visuals.endLine) { + annotation.visuals.endLine.remove() + annotation.visuals.endLineHandle.remove() + } + if (annotation.visuals.filler) { + annotation.visuals.filler.remove() + } + if (annotation.visuals.topLine) { + annotation.visuals.topLine.remove() + } + if (annotation.visuals.text) { + annotation.visuals.text.remove() + } + if (annotation.visuals.group) { + annotation.visuals.group.remove() + } + delete annotation.visuals } - delete annotation.visuals - } - return annotation + return annotation } function deleteWord(annotation: Annotation) { - if (selected != null) { - if (annotation.startTime != null) delete annotation.startTime - if (annotation.endTime != null) delete annotation.endTime - if (annotation.index != null) { - clearWordLabels(annotation) - updateWord(annotation) - } - removeAnnotation(annotation) - clearSelection() - } else message('warnig', 'Click a word to select it first') + if (selected != null) { + if (annotation.startTime != null) delete annotation.startTime + if (annotation.endTime != null) delete annotation.endTime + if (annotation.index != null) { + clearWordLabels(annotation) + updateWord(annotation) + } + removeAnnotation(annotation) + clearSelection() + } else message('warnig', 'Click a word to select it first') } function fillAnnotationPositions(annotation: Annotation) { - if (!annotation.lastClickTimestamp) annotation.lastClickTimestamp = -1 - return annotation + if (!annotation.lastClickTimestamp) annotation.lastClickTimestamp = -1 + return annotation } function updateBackgroundWord(worker: string, annotation: Annotation) { - updateWordBySource(annotation, true, worker) + updateWordBySource(annotation, true, worker) } function clearSelection() { - selected = null - _.forEach(annotations, updateWord) + selected = null + _.forEach(annotations, updateWord) } function find_annotation(id: string | number) { - if (typeof id === 'number') { - // return _.find(annotations, a => a.id == id); // TODO Switch to this - return annotations[id] - } - if (typeof id === 'string') { - return _.find(other_annotations_by_worker[id.split(':')[0]], a => a.id == id) - } + if (typeof id === 'number') { + // return _.find(annotations, a => a.id == id); // TODO Switch to this + return annotations[id] + } + if (typeof id === 'string') { + return _.find(other_annotations_by_worker[id.split(':')[0]], a => a.id == id) + } } function shuffleSelection() { - // TODO This function is terrible - let workerAnnotations = svgAnnotations.selectAll('g').sort(function (a, b) { - // TODO Selection - return d3.ascending( - // @ts-ignore - find_annotation(a).lastClickTimestamp, - // @ts-ignore - find_annotation(b).lastClickTimestamp - ) - })[0][0] - if (!_.isUndefined(workerAnnotations)) { - return workerAnnotations - } - return svgReferenceAnnotations.selectAll('g').sort(function (a, b) { - // TODO Selection - return d3.ascending( - // @ts-ignore - find_annotation(a).lastClickTimestamp, - // @ts-ignore - find_annotation(b).lastClickTimestamp - ) - // @ts-ignore - })[0][0].__data__ + // TODO This function is terrible + let workerAnnotations = svgAnnotations.selectAll('g').sort(function(a, b) { + // TODO Selection + return d3.ascending( + // @ts-ignore + find_annotation(a).lastClickTimestamp, + // @ts-ignore + find_annotation(b).lastClickTimestamp + ) + })[0][0] + if (!_.isUndefined(workerAnnotations)) { + return workerAnnotations + } + return svgReferenceAnnotations.selectAll('g').sort(function(a, b) { + // TODO Selection + return d3.ascending( + // @ts-ignore + find_annotation(a).lastClickTimestamp, + // @ts-ignore + find_annotation(b).lastClickTimestamp + ) + // @ts-ignore + })[0][0].__data__ } function selectWord(annotation: Annotation) { - if (annotation != null) { - lastClick = null - selected = annotation.index - annotation.lastClickTimestamp = Date.now() - _.forEach(annotations, updateWord) - shuffleSelection() - } + if (annotation != null) { + lastClick = null + selected = annotation.index + annotation.lastClickTimestamp = Date.now() + _.forEach(annotations, updateWord) + shuffleSelection() + } } function nextWord() { - var word = _.filter(annotations, function (annotation: Annotation) { - return annotation.startTime == null - })[0] - if (word) return word.index - else return null + var word = _.filter(annotations, function(annotation: Annotation) { + return annotation.startTime == null + })[0] + if (word) return word.index + else return null } function nextAnnotation(index: number) { - var word = _.filter(annotations, function (annotation: Annotation) { - return annotation.index > index && annotation.startTime != null - })[0] - if (word) return word.index - else return null + var word = _.filter(annotations, function(annotation: Annotation) { + return annotation.index > index && annotation.startTime != null + })[0] + if (word) return word.index + else return null } function previousAnnotation(index: number): number | null { - var word = _.last( - _.filter(annotations, function (annotation: Annotation) { - return annotation.index < index && annotation.startTime != null - }) - ) - if (word) return word.index - else return null + var word = _.last( + _.filter(annotations, function(annotation: Annotation) { + return annotation.index < index && annotation.startTime != null + }) + ) + if (word) return word.index + else return null } -$('#play').click(function (e) { +$('#play').click(function(e) { recordMouseClick(e, "#play"); - clear() - stopPlaying() - setup(buffers[bufferKind]!) - play(to(0)) + clear() + stopPlaying() + setup(buffers[bufferKind]!) + play(to(0)) }) -$('#stop').click(function (e) { +$('#stop').click(function(e) { recordMouseClick(e, "#stop"); - clear() - stopPlaying() - redraw(startOffset) + clear() + stopPlaying() + redraw(startOffset) }) -$('#delete-selection').click(function (e) { - recordMouseClick(e, "#delete-selection", selected+''); - clear() - if (selected != null) { - var index = annotations[selected].index - deleteWord(annotations[selected]) - const previous = previousAnnotation(index) - const next = nextAnnotation(index) - if (previous != null) selectWord(annotations[previous]) - else if (next != null) selectWord(annotations[next]) - else message('warning', 'Click a word to select it first') - } else message('warning', 'Click a word to select it first') +$('#delete-selection').click(function(e) { + recordMouseClick(e, "#delete-selection", selected + ''); + clear() + if (selected != null) { + var index = annotations[selected].index + deleteWord(annotations[selected]) + const previous = previousAnnotation(index) + const next = nextAnnotation(index) + if (previous != null) selectWord(annotations[previous]) + else if (next != null) selectWord(annotations[next]) + else message('warning', 'Click a word to select it first') + } else message('warning', 'Click a word to select it first') }) function playAnnotation(annotation: Annotation) { - stopPlaying() - setup(buffers[bufferKind]!) - if (annotation.endTime != null) - play( - timeInMovieToTimeInBuffer(annotation.startTime!), - sub(timeInMovieToTimeInBuffer(annotation.endTime), timeInMovieToTimeInBuffer(annotation.startTime!)) - ) - else play(timeInMovieToTimeInBuffer(annotation.startTime!), defaultPlayLength()) -} - -$('#play-selection').click(function (e) { - recordMouseClick(e, "#play-selection", selected+''); - clear() - if (selected != null) { stopPlaying() setup(buffers[bufferKind]!) - playAnnotation(annotations[selected]) - } else message('warning', 'Click a word to select it first') + if (annotation.endTime != null) + play( + timeInMovieToTimeInBuffer(annotation.startTime!), + sub(timeInMovieToTimeInBuffer(annotation.endTime), timeInMovieToTimeInBuffer(annotation.startTime!)) + ) + else play(timeInMovieToTimeInBuffer(annotation.startTime!), defaultPlayLength()) +} + +$('#play-selection').click(function(e) { + recordMouseClick(e, "#play-selection", selected + ''); + clear() + if (selected != null) { + stopPlaying() + setup(buffers[bufferKind]!) + playAnnotation(annotations[selected]) + } else message('warning', 'Click a word to select it first') }) -$('#start-next-word').click(function (e) { +$('#start-next-word').click(function(e) { clear() if (lastClick != null) { - recordMouseClick(e, "#start-next-word", lastClick+''); + recordMouseClick(e, "#start-next-word", lastClick + ''); const firstMissingWord = _.head(_.filter(annotations, a => !isValidAnnotation(a))) if (!firstMissingWord) { message('warning', "All words are already annotated; can't start another one") @@ -1744,8 +848,8 @@ $('#start-next-word').click(function (e) { recordMouseClick(e, "#start-next-word", selected + ''); if ( selected != null && - annotations[selected].endTime != null && - (annotations[selected + 1] == null || annotations[selected + 1].endTime == null) + annotations[selected].endTime != null && + (annotations[selected + 1] == null || annotations[selected + 1].endTime == null) ) { if (selected + 1 >= words.length) { message('warning', 'No next word to annotate') @@ -1760,13 +864,13 @@ $('#start-next-word').click(function (e) { } }) -$('#start-next-word-after-current-word').click(function (e) { +$('#start-next-word-after-current-word').click(function(e) { recordMouseClick(e, "#start-next-word-after-current-word", selected + ''); clear() if ( selected != null && - annotations[selected].endTime != null && - (annotations[selected + 1] == null || annotations[selected + 1].endTime == null) + annotations[selected].endTime != null && + (annotations[selected + 1] == null || annotations[selected + 1].endTime == null) ) { if (selected + 1 >= words.length) { message('warning', 'No next word to annotate') @@ -1780,21 +884,21 @@ $('#start-next-word-after-current-word').click(function (e) { } }) -$('#reset').click(function (e) { +$('#reset').click(function(e) { recordMouseClick(e, "#reset"); - clear() - location.reload() + clear() + location.reload() }) // TODO Next must clear the loading flag whne it is done! function submit(next: any) { - if (loading != LoadingState.ready) return - loading = LoadingState.submitting - clear() - message('warning', 'Submitting annotation') - sendTelemetry() - // TODO We should reenable this for mturk - // tokenMode() + if (loading != LoadingState.ready) return + loading = LoadingState.submitting + clear() + message('warning', 'Submitting annotation') + sendTelemetry() + // TODO We should reenable this for mturk + // tokenMode() const data = { segment: segment, token: token, @@ -1811,7 +915,7 @@ function submit(next: any) { worker: $.url().param().worker, annotations: _.map( _.filter(annotations, a => !_.isUndefined(a.startTime)), - function (a) { + function(a) { return { startTime: a.startTime!, endTime: a.endTime!, @@ -1821,49 +925,55 @@ function submit(next: any) { } ), } - recordSend({data: data, - server: $.url().attr().host, - port: $.url().attr().port, - why: 'submit'}) - $.ajax({ - type: 'POST', - data: JSON.stringify(data), - contentType: 'application/json', - url: '/submission', - success: function (data) { - recordReceive({response: data, error: null, status: '200', server: $.url().attr().host, - port: $.url().attr().port, - why: 'submit'}) - console.log(data) - if (data && data.response == 'ok') { - if (data.stoken != null && token != null) { - next() - message('success', 'Thanks!
Enter the following two characters back into Amazon Turk: ' + data.stoken) - } else { - next() - message('success', 'Submitted annotation') - } - } else { - loading = LoadingState.ready - message( - 'danger', - 'Failed to submit annotation!
Bad server reply!
Please email abarbu@csail.mit.edu with this message. Your work will not be lost and we will give you credit for it.
' + - JSON.stringify([data, annotations]) - ) - } - }, - error: function (data, status, error) { - recordReceive({response: data, error: error, status: status, server: $.url().attr().host, - port: $.url().attr().port, - why: 'submit'}) - loading = LoadingState.ready - message( - 'danger', - 'Failed to submit annotation!
Ajax error communicating with the server!
Please email abarbu@csail.mit.edu with this message. Your work will not be lost and we will give you credit for it.
' + - JSON.stringify([data, status, error, annotations]) - ) - }, - }) + recordSend({ + data: data, + server: $.url().attr().host, + port: $.url().attr().port, + why: 'submit' + }) + $.ajax({ + type: 'POST', + data: JSON.stringify(data), + contentType: 'application/json', + url: '/submission', + success: function(data) { + recordReceive({ + response: data, error: null, status: '200', server: $.url().attr().host, + port: $.url().attr().port, + why: 'submit' + }) + console.log(data) + if (data && data.response == 'ok') { + if (data.stoken != null && token != null) { + next() + message('success', 'Thanks!
Enter the following two characters back into Amazon Turk: ' + data.stoken) + } else { + next() + message('success', 'Submitted annotation') + } + } else { + loading = LoadingState.ready + message( + 'danger', + 'Failed to submit annotation!
Bad server reply!
Please email abarbu@csail.mit.edu with this message. Your work will not be lost and we will give you credit for it.
' + + JSON.stringify([data, annotations]) + ) + } + }, + error: function(data, status, error) { + recordReceive({ + response: data, error: error, status: status, server: $.url().attr().host, + port: $.url().attr().port, + why: 'submit' + }) + loading = LoadingState.ready + message( + 'danger', + 'Failed to submit annotation!
Ajax error communicating with the server!
Please email abarbu@csail.mit.edu with this message. Your work will not be lost and we will give you credit for it.
' + + JSON.stringify([data, status, error, annotations]) + ) + }, + }) } $('#submit').click(e => { @@ -1873,279 +983,279 @@ $('#submit').click(e => { $('input[type="checkbox"],[type="radio"]').not('#create-switch').bootstrapSwitch() $('#toggle-audio').on('switchChange.bootstrapSwitch', (e) => { - stopPlaying() - mute = !mute - recordMouseClick(e, "#toggle-audio", mute+''); + stopPlaying() + mute = !mute + recordMouseClick(e, "#toggle-audio", mute + ''); }) $('#toggle-speed').on('switchChange.bootstrapSwitch', (e) => { - stopPlaying() - if (bufferKind == BufferType.half) bufferKind = BufferType.normal - else if (bufferKind == BufferType.normal) bufferKind = BufferType.half - recordMouseClick(e, "#toggle-speed", bufferKind+''); + stopPlaying() + if (bufferKind == BufferType.half) bufferKind = BufferType.normal + else if (bufferKind == BufferType.normal) bufferKind = BufferType.half + recordMouseClick(e, "#toggle-speed", bufferKind + ''); }) -$('#edit-transcript').click(function (e) { - if (!editingTranscriptMode) { - $('#transcript-entry').removeClass('display-none') - $('#transcript-entry').addClass('display-inline') - $('#words').addClass('display-none') - $('#words').removeClass('display-inline') - $('#transcript-input').val(_.join(words, ' ')) - $('#edit-transcript').text('Finish editing') - $('#edit-transcript').removeClass('btn-primary') - $('#edit-transcript').addClass('btn-danger') - } else { - $('#transcript-entry').addClass('display-none') - $('#transcript-entry').removeClass('display-inline') - $('#words').removeClass('display-none') - $('#words').addClass('display-inline') - $('#edit-transcript').text('Edit transcript') - $('#edit-transcript').removeClass('btn-danger') - $('#edit-transcript').addClass('btn-primary') - updateWordsWithAnnotations(_.filter(_.split(String($('#transcript-input').val()), ' '), a => a !== '')) - } +$('#edit-transcript').click(function(e) { + if (!editingTranscriptMode) { + $('#transcript-entry').removeClass('display-none') + $('#transcript-entry').addClass('display-inline') + $('#words').addClass('display-none') + $('#words').removeClass('display-inline') + $('#transcript-input').val(_.join(words, ' ')) + $('#edit-transcript').text('Finish editing') + $('#edit-transcript').removeClass('btn-primary') + $('#edit-transcript').addClass('btn-danger') + } else { + $('#transcript-entry').addClass('display-none') + $('#transcript-entry').removeClass('display-inline') + $('#words').removeClass('display-none') + $('#words').addClass('display-inline') + $('#edit-transcript').text('Edit transcript') + $('#edit-transcript').removeClass('btn-danger') + $('#edit-transcript').addClass('btn-primary') + updateWordsWithAnnotations(_.filter(_.split(String($('#transcript-input').val()), ' '), a => a !== '')) + } recordMouseClick(e, "#edit-transcript", $('#transcript-input').val() + ''); - editingTranscriptMode = !editingTranscriptMode + editingTranscriptMode = !editingTranscriptMode }) -$('#transcript-input').keypress(function (e) { +$('#transcript-input').keypress(function(e) { recordMouseClick(e, "#transcript-input"); - if (e.which == 13) { - e.preventDefault() - $('#edit-transcript').click() - } + if (e.which == 13) { + e.preventDefault() + $('#edit-transcript').click() + } }) -$('#location-input').keypress(function (e) { +$('#location-input').keypress(function(e) { recordMouseClick(e, "#location-input"); - if (e.which == 13) { - e.preventDefault() - $('#go-to-location').click() - } + if (e.which == 13) { + e.preventDefault() + $('#go-to-location').click() + } }) -$('#go-to-location').click(function (e) { +$('#go-to-location').click(function(e) { recordMouseClick(e, "#go-to-location"); - const val = String($('#location-input').val()) - const n = parseInt(val) - if ('' + n !== $('#location-input').val()) { - message('danger', "Go to location isn't an integer") - } else { - const s = mkSegmentName(movieName, parseInt(val), parseInt(val) + (endS - startS)) - $.get('/spectrograms/' + movieName + '/' + s + '.jpg', () => { - reload(mkSegmentName(movieName, parseInt(val), parseInt(val) + (endS - startS))) - }).fail(() => { - message('danger', "That goto location doesn't exist in this movie") - }) - } + const val = String($('#location-input').val()) + const n = parseInt(val) + if ('' + n !== $('#location-input').val()) { + message('danger', "Go to location isn't an integer") + } else { + const s = mkSegmentName(movieName, parseInt(val), parseInt(val) + (endS - startS)) + $.get('/spectrograms/' + movieName + '/' + s + '.jpg', () => { + reload(mkSegmentName(movieName, parseInt(val), parseInt(val) + (endS - startS))) + }).fail(() => { + message('danger', "That goto location doesn't exist in this movie") + }) + } }) -$('#go-to-last').click(function (e) { - recordMouseClick(e,"#go-to-last"); - $.get( - '/last-annotation', - { - movieName: movieName, - startS: startS, - endS: endS, - worker: $.url().param().worker, - }, - a => { - if (a) { - const start = 2 * _.floor(_.floor(a.startTime) / 2) - reload(mkSegmentName(movieName, start, start + (endS - startS))) - } else { - message('danger', "You don't have any annotations, can't go to the last one") - } - } - ) +$('#go-to-last').click(function(e) { + recordMouseClick(e, "#go-to-last"); + $.get( + '/last-annotation', + { + movieName: movieName, + startS: startS, + endS: endS, + worker: $.url().param().worker, + }, + a => { + if (a) { + const start = 2 * _.floor(_.floor(a.startTime) / 2) + reload(mkSegmentName(movieName, start, start + (endS - startS))) + } else { + message('danger', "You don't have any annotations, can't go to the last one") + } + } + ) }) -$('#replace-with-reference-annotation').click(function (e) { +$('#replace-with-reference-annotation').click(function(e) { recordMouseClick(e, "#replace-with-reference-annotation"); - stopPlaying() - let reference_annotations = other_annotations_by_worker[current_reference_annotation] - if (reference_annotations) { - clear() - _.map(annotations, function (a) { - if (a) { - selectWord(a) - deleteWord(a) - } - }) - words = _.map(reference_annotations, a => a.word) - updateWords(_.map(reference_annotations, a => a.word)) - annotations = _.map(reference_annotations, (a, k) => { - let r = removeAnnotation(_.clone(a)) - r.index = k - return r - }) - _.map(annotations, function (a) { - if (a) { - updateWord(a) - } - }) - current_reference_annotation = undefined - $('.annotation').each((_i, a) => { - if ($(a).text() == 'none') { - $(a).removeClass('btn-default').addClass('btn-info') - } else { - $(a).removeClass('btn-info').addClass('btn-default') - } - }) - message('success', 'Loaded the reference annotation') - } else { - message('warning', 'No reference annotation exists') - } + stopPlaying() + let reference_annotations = other_annotations_by_worker[current_reference_annotation] + if (reference_annotations) { + clear() + _.map(annotations, function(a) { + if (a) { + selectWord(a) + deleteWord(a) + } + }) + words = _.map(reference_annotations, a => a.word) + updateWords(_.map(reference_annotations, a => a.word)) + annotations = _.map(reference_annotations, (a, k) => { + let r = removeAnnotation(_.clone(a)) + r.index = k + return r + }) + _.map(annotations, function(a) { + if (a) { + updateWord(a) + } + }) + current_reference_annotation = undefined + $('.annotation').each((_i, a) => { + if ($(a).text() == 'none') { + $(a).removeClass('btn-default').addClass('btn-info') + } else { + $(a).removeClass('btn-info').addClass('btn-default') + } + }) + message('success', 'Loaded the reference annotation') + } else { + message('warning', 'No reference annotation exists') + } }) $('#fill-with-reference').click(e => { - recordMouseClick(e, "#fill-with-reference"); - stopPlaying() - const referenceAnnotations = other_annotations_by_worker[current_reference_annotation] - if (referenceAnnotations) { - clear() - let existingAnnotations = _.map(annotations, cloneAnnotation) - _.forEach(annotations, a => { - if (a) { - selectWord(a) - deleteWord(a) - } - }) - existingAnnotations = _.filter(existingAnnotations, isValidAnnotation) - const lastAnnotationEndTime: TimeInMovie = to( - _.max( - _.concat( - -1, - _.map(existingAnnotations, a => from(a.endTime!)) + recordMouseClick(e, "#fill-with-reference"); + stopPlaying() + const referenceAnnotations = other_annotations_by_worker[current_reference_annotation] + if (referenceAnnotations) { + clear() + let existingAnnotations = _.map(annotations, cloneAnnotation) + _.forEach(annotations, a => { + if (a) { + selectWord(a) + deleteWord(a) + } + }) + existingAnnotations = _.filter(existingAnnotations, isValidAnnotation) + const lastAnnotationEndTime: TimeInMovie = to( + _.max( + _.concat( + -1, + _.map(existingAnnotations, a => from(a.endTime!)) + ) + )! ) - )! - ) - let mergedAnnotations = _.concat( - existingAnnotations, - // @ts-ignore - _.filter(referenceAnnotations, (a: Annotation) => a.startTime > lastAnnotationEndTime) - ) - words = _.map(mergedAnnotations, a => a.word) - updateWords(_.map(mergedAnnotations, a => a.word)) - mergedAnnotations = _.map(mergedAnnotations, (a, k: number) => { - let r = cloneAnnotation(a) - r.index = k - return r - }) - annotations = mergedAnnotations - _.map(mergedAnnotations, a => { - if (a) { - updateWord(a) - } - }) - message('success', 'Used reference to fill remanining annotations') - } else { - message('warning', 'No reference annotation exists') - } + let mergedAnnotations = _.concat( + existingAnnotations, + // @ts-ignore + _.filter(referenceAnnotations, (a: Annotation) => a.startTime > lastAnnotationEndTime) + ) + words = _.map(mergedAnnotations, a => a.word) + updateWords(_.map(mergedAnnotations, a => a.word)) + mergedAnnotations = _.map(mergedAnnotations, (a, k: number) => { + let r = cloneAnnotation(a) + r.index = k + return r + }) + annotations = mergedAnnotations + _.map(mergedAnnotations, a => { + if (a) { + updateWord(a) + } + }) + message('success', 'Used reference to fill remanining annotations') + } else { + message('warning', 'No reference annotation exists') + } }) function mkSegmentName(movieName: string, start: number, end: number) { - return movieName + ':' + ('' + start).padStart(5, '0') + ':' + ('' + end).padStart(5, '0') + return movieName + ':' + ('' + start).padStart(5, '0') + ':' + ('' + end).padStart(5, '0') } -$('#back-save-4-sec').click(function (e) { - recordMouseClick(e,"#back-save-4-sec"); - submit(() => - reload(movieName + ':' + ('' + (startS - 4)).padStart(5, '0') + ':' + ('' + (endS - 4)).padStart(5, '0')) - ) +$('#back-save-4-sec').click(function(e) { + recordMouseClick(e, "#back-save-4-sec"); + submit(() => + reload(movieName + ':' + ('' + (startS - 4)).padStart(5, '0') + ':' + ('' + (endS - 4)).padStart(5, '0')) + ) }) -$('#back-save-2-sec').click(function (e) { +$('#back-save-2-sec').click(function(e) { recordMouseClick(e, "#back-save-2-sec"); - submit(() => - reload(movieName + ':' + ('' + (startS - 2)).padStart(5, '0') + ':' + ('' + (endS - 2)).padStart(5, '0')) - ) + submit(() => + reload(movieName + ':' + ('' + (startS - 2)).padStart(5, '0') + ':' + ('' + (endS - 2)).padStart(5, '0')) + ) }) -$('#forward-save-2-sec').click(function (e) { - recordMouseClick(e,"#forward-save-2-sec"); - submit(() => - reload(movieName + ':' + ('' + (startS + 2)).padStart(5, '0') + ':' + ('' + (endS + 2)).padStart(5, '0')) - ) +$('#forward-save-2-sec').click(function(e) { + recordMouseClick(e, "#forward-save-2-sec"); + submit(() => + reload(movieName + ':' + ('' + (startS + 2)).padStart(5, '0') + ':' + ('' + (endS + 2)).padStart(5, '0')) + ) }) -$('#forward-save-4-sec').click(function (e) { +$('#forward-save-4-sec').click(function(e) { recordMouseClick(e, "#forward-save-4-sec"); - submit(() => - reload(movieName + ':' + ('' + (startS + 4)).padStart(5, '0') + ':' + ('' + (endS + 4)).padStart(5, '0')) - ) + submit(() => + reload(movieName + ':' + ('' + (startS + 4)).padStart(5, '0') + ':' + ('' + (endS + 4)).padStart(5, '0')) + ) }) -javascriptNode!.onaudioprocess = function (_audioProcessingEvent) { - if (sourceNode && audioIsPlaying) { - if (from(startTime) == -1) startTime = to(context.currentTime) - redraw(to(context.currentTime - from(startTime) + from(startOffset))) - } +javascriptNode!.onaudioprocess = function(_audioProcessingEvent) { + if (sourceNode && audioIsPlaying) { + if (from(startTime) == -1) startTime = to(context!.currentTime) + redraw(to(context!.currentTime - from(startTime) + from(startOffset))) + } } function render_other_annotations(worker: string) { - current_reference_annotation = worker - let reference_annotations = other_annotations_by_worker[worker] - if (reference_annotations) { - _.forEach(other_annotations_by_worker, as => _.forEach(as, removeAnnotation)) - $('.annotation').each((_i, a) => { - if ($(a).text() == worker && reference_annotations.length > 0) { - $(a).removeClass('btn-default').addClass('btn-info') - } else { - $(a).removeClass('btn-info').addClass('btn-default') - } - }) - _.map(reference_annotations, a => updateBackgroundWord(worker, a)) - message('success', 'Showing the reference annotation') - } else { - message('warning', 'Cannot show reference annotation') - } + current_reference_annotation = worker + let reference_annotations = other_annotations_by_worker[worker] + if (reference_annotations) { + _.forEach(other_annotations_by_worker, as => _.forEach(as, removeAnnotation)) + $('.annotation').each((_i, a) => { + if ($(a).text() == worker && reference_annotations.length > 0) { + $(a).removeClass('btn-default').addClass('btn-info') + } else { + $(a).removeClass('btn-info').addClass('btn-default') + } + }) + _.map(reference_annotations, a => updateBackgroundWord(worker, a)) + message('success', 'Showing the reference annotation') + } else { + message('warning', 'Cannot show reference annotation') + } } function register_other_annotations(worker: string) { - let reference_annotations = other_annotations_by_worker[worker] - if (reference_annotations && worker != $.url().param().worker) { - $('#annotations') - .append( - $('', - }) -}) - diff --git a/public/help.ts b/public/help.ts new file mode 100644 index 0000000..b07d07d --- /dev/null +++ b/public/help.ts @@ -0,0 +1,146 @@ + +$('#container-wrapper') + .addClass('bootstro') + .attr('data-bootstro-title', 'Task') + .attr( + 'data-bootstro-content', + "You're going to annotate the beginning and end of each word on this diagram. It's a representation of the audio. Click anyhwere on it to play a chunk of the audio." + ) + .attr('data-bootstro-placement', 'bottom') + .attr('data-bootstro-width', '700px') + .attr('data-bootstro-step', '0') + +$('#play') + .addClass('bootstro') + .attr('data-bootstro-title', 'Play') + .attr( + 'data-bootstro-content', + 'You can play the entire audio clip with this button. By default the audio plays at half the speed to make annotation easier.' + ) + .attr('data-bootstro-placement', 'bottom') + .attr('data-bootstro-step', '1') + +$('#toggle-speed-div') + .addClass('bootstro') + .attr('data-bootstro-title', 'Audio speed') + .attr( + 'data-bootstro-content', + 'It can be hard to catch each word and when it was said. We play the audio at half speed by default, you can change this to regular speed.' + ) + .attr('data-bootstro-placement', 'top') + .attr('data-bootstro-step', '2') + +$('#transcript-panel') + .addClass('bootstro') + .attr('data-bootstro-title', 'Words') + .attr('data-bootstro-content', 'We try to guess which words might have been said.') + .attr('data-bootstro-placement', 'top') + .attr('data-bootstro-step', '3') + +$('#edit-transcript') + .addClass('bootstro') + .attr('data-bootstro-title', 'Transcript') + .attr('data-bootstro-content', 'Listen to the audio and edit the words. Words may be wrong or missing.') + .attr('data-bootstro-placement', 'top') + .attr('data-bootstro-step', '4') + +$('#spectrogram') + .addClass('bootstro') + .attr('data-bootstro-title', 'Selecting') + .attr( + 'data-bootstro-content', + 'Once you have the words, place the red marker on the diagram. A short audio segment will play.' + ) + .attr('data-bootstro-placement', 'top') + .attr('data-bootstro-step', '5') + +$('#words') + .addClass('bootstro') + .attr('data-bootstro-title', 'Words') + .attr( + 'data-bootstro-content', + 'With the market in position you can choose which word to start at that location. We guess the word length, but you should adjust it.' + ) + .attr('data-bootstro-placement', 'top') + .attr('data-bootstro-step', '6') + +$('#container') + .addClass('bootstro') + .attr('data-bootstro-title', 'Adjusting') + .attr( + 'data-bootstro-content', + 'Drag the start and ends of words. Green words are ones that you annotated. The orange word is the currenctly selected one. If any white words exist, they are references we provide to make life eaiser.' + ) + .attr('data-bootstro-placement', 'bottom') + .attr('data-bootstro-step', '7') + +$('#d3') + .addClass('bootstro') + .attr('data-bootstro-title', 'Verifying') + .attr( + 'data-bootstro-content', + 'You should adjust the word boundaries by dragging them into the correct position on the diagram. The audio will automatically play. You can replay by clicking here or by using the keyboard shortcuts in red.' + ) + .attr('data-bootstro-placement', 'top') + .attr('data-bootstro-step', '8') + +$('#delete-selection') + .addClass('bootstro') + .attr('data-bootstro-title', 'Deleting') + .attr('data-bootstro-content', "If a word isn't relevant or annotated incorrectly you can remove it.") + .attr('data-bootstro-placement', 'top') + .attr('data-bootstro-step', '9') + +$('#annotations') + .addClass('bootstro') + .attr('data-bootstro-title', 'References') + .attr( + 'data-bootstro-content', + 'Sometimes we have reference annotation available. You can select any references here. This includes any previous work you have done. They appear in white on the audio. Your annotations are in green and your selected annotation is in orange. The white annotations cannot be changed.' + ) + .attr('data-bootstro-placement', 'top') + .attr('data-bootstro-step', '10') + +$('#replace-reference') + .addClass('bootstro') + .attr('data-bootstro-title', 'References') + .attr( + 'data-bootstro-content', + 'You can replace your annotation with the reference one or use the reference to fill in missing parts of your annotations.' + ) + .attr('data-bootstro-placement', 'top') + .attr('data-bootstro-step', '11') + +$('#save-and-seek') + .addClass('bootstro') + .attr('data-bootstro-title', 'Saving while navigating') + .attr( + 'data-bootstro-content', + 'You will usually annotate and then move on to the next segment. These buttons move you but also save your work each time.' + ) + .attr('data-bootstro-placement', 'top') + .attr('data-bootstro-step', '12') + +$('#submit-button') + .addClass('bootstro') + .attr('data-bootstro-title', 'Submit') + .attr('data-bootstro-content', 'You can save your work on the current segment by submitting it.') // TODO Update for MTurk + .attr('data-bootstro-placement', 'bottom') + .attr('data-bootstro-step', '13') + +// TODO This needs a unique location +// $('#submit').addClass('bootstro') +// .attr('data-bootstro-title', "Submitting") +// .attr('data-bootstro-content', "Once you're done with all of the words you can click here and we'll give you the token to enter into Amazon interface. It's ok to leave out a word if you can't recognize it, it's too noisy, or if it's not actually there. Thanks for helping us with our research!") +// .attr('data-bootstro-placement', "bottom") +// .attr('data-bootstro-step', '6') + +$('#intro').click((e) => { + recordMouseClick(e, '#intro') + // @ts-ignore + bootstro.start('.bootstro', { + finishButton: + '', + }) +}) + diff --git a/public/misc.ts b/public/misc.ts new file mode 100644 index 0000000..65af1f9 --- /dev/null +++ b/public/misc.ts @@ -0,0 +1,65 @@ +function levenshteinAlignment( + iWords: string[], + i: number, + jWords: string[], + j: number, + cache: (number | boolean)[][] +): any { + if (cache[i][j] !== false) { + return cache[i][j] + } + let out + if (i >= iWords.length) { + out = { distance: Math.abs(jWords.length - j) } + } else if (j >= jWords.length) { + out = { distance: Math.abs(iWords.length - i) } + } else { + let ret1 = _.clone(levenshteinAlignment(iWords, i + 1, jWords, j, cache)) + ret1.distance += 1 + let ret2 = _.clone(levenshteinAlignment(iWords, i, jWords, j + 1, cache)) + ret2.distance += 1 + let ret3 = _.clone(levenshteinAlignment(iWords, i + 1, jWords, j + 1, cache)) + if (iWords[i] === jWords[j]) ret3[i] = j + else ret3.distance += 1 + if (ret1.distance < ret2.distance && ret1.distance < ret3.distance) { + out = ret1 + } else if (ret2.distance < ret1.distance && ret2.distance < ret3.distance) { + out = ret2 + } else { + out = ret3 + } + } + cache[i][j] = out + return out +} + +function alignWords(newWords: string[], oldWords: string[]): any { + let cache: (number | boolean)[][] = [] + _.forEach(_.range(0, newWords.length + 2), i => { + cache[i] = [] + _.forEach(_.range(0, oldWords.length + 2), j => { + cache[i][j] = false + }) + }) + return levenshteinAlignment(newWords, 0, oldWords, 0, cache) +} + +function mousePosition() { + // @ts-ignore + const x = d3.event.layerX + // @ts-ignore + const y = d3.event.layerY + return { + x: x, + y: y, + } +} + +function parseSegment(segmentName: string) { + const s = segmentName.split(':') + return { movieName: s[0], startTime: parseFloat(s[1]), endTime: parseFloat(s[2]) } +} + +function segmentString(details: { movieName: string; startTime: number; endTime: number }) { + return mkSegmentName(details.movieName, details.startTime, details.endTime) +} diff --git a/public/shortcuts.ts b/public/shortcuts.ts new file mode 100644 index 0000000..ec13d49 --- /dev/null +++ b/public/shortcuts.ts @@ -0,0 +1,233 @@ + +function keyboardShortcutsOn() { + $(document).bind('keydown', 'p', () => { + clear() + recordKeypress('p') + $('#play').click() + }) + $(document).bind('keydown', 't', () => { + clear() + recordKeypress('w') + $('#stop').click() + }) + $(document).bind('keydown', 'd', () => { + clear() + recordKeypress('d') + $('#delete-selection').click() + }) + $(document).bind('keydown', 'y', () => { + clear() + recordKeypress('y') + $('#play-selection').click() + }) + $(document).bind('keydown', 'w', () => { + clear() + recordKeypress('w') + $('#start-next-word').click() + }) + $(document).bind('keydown', 'shift+w', () => { + clear() + recordKeypress('shift+y') + $('#start-next-word-after-current-word').click() + }) + $(document).bind('keydown', 'a', () => { + clear() + recordKeypress('a') + $('#toggle-speed').bootstrapSwitch('toggleState') + }) + $(document).bind('keydown', 'm', () => { + clear() + recordKeypress('m') + $('#toggle-audio').bootstrapSwitch('toggleState') + }) + $(document).bind('keydown', 'shift+b', () => { + clear() + recordKeypress('shift+b') + $('#back-save-4-sec').click() + }) + $(document).bind('keydown', 'b', () => { + clear() + recordKeypress('b') + $('#back-save-2-sec').click() + }) + $(document).bind('keydown', 'f', () => { + clear() + recordKeypress('f') + $('#forward-save-2-sec').click() + }) + $(document).bind('keydown', 'shift+f', () => { + clear() + recordKeypress('shift+f') + $('#forward-save-4-sec').click() + }) + $(document).bind('keydown', 's', () => { + clear() + recordKeypress('s') + $('#submit').click() + }) + $(document).bind('keydown', 'u', () => { + clear() + recordKeypress('u') + $('#fill-with-reference').click() + }) + $(document).bind('keydown', 't', e => { + clear() + recordKeypress('t') + $('#edit-transcript').click() + if (selected != null) { + const n = _.sum( + _.map( + _.filter(annotations, a => a.index < selected!), + a => a.word.length + 1 + ) + ) + // @ts-ignore + $('#transcript-input').focus()[0].setSelectionRange(n, n) + } else { + $('#transcript-input').focus() + } + e.preventDefault() + }) + $(document).bind('keydown', 'left', () => { + clear() + recordKeypress('left') + if (selected == null) { + const lastAnnotation = _.last(_.filter(annotations, isValidAnnotation)) + if (lastAnnotation) { + selectWord(lastAnnotation) + $('#play-selection').click() + } else { + message('warning', "Can't select the last word: no words are annotated") + return + } + } else { + const nextAnnotation = _.last(_.filter(_.take(annotations, selected), isValidAnnotation)) + if (nextAnnotation) { + selectWord(nextAnnotation) + $('#play-selection').click() + } else { + message('warning', 'At the first word, no other annotations to select') + return + } + } + }) + $(document).bind('keydown', 'right', () => { + clear() + recordKeypress('right') + if (selected == null) { + const firstAnnotation = _.head(_.filter(annotations, isValidAnnotation)) + if (firstAnnotation) { + selectWord(firstAnnotation) + $('#play-selection').click() + } else { + message('warning', "Can't select the first word: no words are annotated") + return + } + } else { + const nextAnnotation = _.head(_.filter(_.drop(annotations, selected + 1), isValidAnnotation)) + if (nextAnnotation) { + selectWord(nextAnnotation) + $('#play-selection').click() + } else { + message('warning', 'At the last word, no other annotations to select') + return + } + } + }) + $(document).bind('keydown', 'up', () => { + clear() + recordKeypress('up') + $('#play-selection').click() + }) + $(document).bind('keydown', 'down', () => { + clear() + recordKeypress('down') + $('#play-selection').click() + }) + $(document).bind('keydown', 'shift+left', () => { + clear() + recordKeypress('shift+left') + if (selected == null || !isValidAnnotation(annotations[selected])) { + message('warning', "Can't shift the start of the word earlier; no word is selected.") + return + } else { + annotations[selected].startTime = + verifyTranscriptOrder(selected, + subMax(annotations[selected].startTime!, keyboardShiftOffset, to(startS))) + updateWord(annotations[selected]) + } + }) + $(document).bind('keydown', 'shift+right', () => { + clear() + recordKeypress('shift+right') + if (selected == null || !isValidAnnotation(annotations[selected])) { + message('warning', "Can't shift the start of the word later; no word is selected.") + return + } else { + annotations[selected].startTime = verifyTranscriptOrder(selected, addMin( + annotations[selected].startTime!, + keyboardShiftOffset, + sub(annotations[selected].endTime!, keyboardShiftOffset))) + updateWord(annotations[selected]) + } + }) + $(document).bind('keydown', 'ctrl+left', () => { + clear() + recordKeypress('ctrl+left') + if (selected == null || !isValidAnnotation(annotations[selected])) { + message('warning', "Can't shift the end of the word earlier; no word is selected.") + return + } else { + annotations[selected].endTime = subMax( + annotations[selected].endTime!, + keyboardShiftOffset, + add(annotations[selected].startTime!, keyboardShiftOffset) + ) + updateWord(annotations[selected]) + } + }) + $(document).bind('keydown', 'ctrl+right', () => { + clear() + recordKeypress('ctrl+right') + if (selected == null || !isValidAnnotation(annotations[selected])) { + message('warning', "Can't shift the end of the word later; no word is selected.") + return + } else { + annotations[selected].endTime = addMin(annotations[selected].endTime!, keyboardShiftOffset, to(endS)) + updateWord(annotations[selected]) + } + }) + $(document).bind('keydown', 'shift+up', () => { + clear() + recordKeypress('shift+up') + if (selected == null || !isValidAnnotation(annotations[selected])) { + message('warning', "Can't shift the word later; no word is selected.") + return + } else { + annotations[selected].startTime = verifyTranscriptOrder(selected, addMin( + annotations[selected].startTime!, + keyboardShiftOffset, + sub(annotations[selected].endTime!, keyboardShiftOffset) + )) + annotations[selected].endTime = addMin(annotations[selected].endTime!, keyboardShiftOffset, to(endS)) + updateWord(annotations[selected]) + } + }) + $(document).bind('keydown', 'shift+down', () => { + clear() + recordKeypress('shift+down') + if (selected == null || !isValidAnnotation(annotations[selected])) { + message('warning', "Can't shift the word earlier; no word is selected.") + return + } else { + annotations[selected].startTime = verifyTranscriptOrder(selected, + subMax(annotations[selected].startTime!, keyboardShiftOffset, to(startS))) + annotations[selected].endTime = subMax( + annotations[selected].endTime!, + keyboardShiftOffset, + add(annotations[selected].startTime!, keyboardShiftOffset) + ) + updateWord(annotations[selected]) + } + }) +} diff --git a/public/state.ts b/public/state.ts new file mode 100644 index 0000000..0fe83f1 --- /dev/null +++ b/public/state.ts @@ -0,0 +1,120 @@ +let guiRevision: string | null = null +const preloadSegments = true + +let splitHeight = _.isUndefined($.url().param().splitHeight) ? true : $.url().param().splitHeight + +var loading: LoadingState = LoadingState.ready + +var viewer_width: number +var viewer_height: number +var viewer_border = 0 + +const canvas = $('#canvas')[0]! +const ctx = canvas.getContext('2d')! + +var endTime = 100000 // infinity seconds.. + +var context: AudioContext | null = null + +var segment: string +var startS: number +var endS: number +var movieName: string +var bufferKind: BufferType +// Fetched based on the segment +var words: string[] = [] +var mode: string +var token: string +var browser = navigator.userAgent.toString() +var other_annotations_by_worker: { [name: string]: Annotation[] } = {} // previous_annotation +// TODO Should expose this so that we can change the default +var current_reference_annotation = $.url().param().defaultReference + +// This has a race condition between stopping and start the audio, that's why we +// have a counter. 'onended' is called after starting a new audio playback, +// because the previous playback started. +var audioIsPlaying = 0 + +// For the transcript pane +var editingTranscriptMode = false + +var buffers: Buffers = { normal: null, half: null } +var sourceNode: AudioBufferSourceNode +var javascriptNode: ScriptProcessorNode +var startTime: TimeInBuffer = to(0) +var startOffset: TimeInBuffer = to(0) +var lastClick: TimeInMovie | null = null +var selected: number | null = null +var annotations: Annotation[] = [] +var mute: boolean = false +const keyboardShiftOffset: TimeInMovie = to(0.01) +const handleOffset = 0 + +let dragStart: TimeInMovie | null = null +var svg = d3.select('#d3') +svg + .append('rect') + .attr('width', '100%') + .attr('height', '100%') + .attr('fill', '#ffffff') + .attr('fill-opacity', 0.0) + .call( + d3.behavior + .drag() + .on('dragstart', () => { + // @ts-ignore + recordMouseClick(d3.event.sourceEvent, "#d3", "dragstart") + // @ts-ignore + const x = d3.event.sourceEvent.layerX + lastClick = positionToAbsoluteTime(to(x)) + dragStart = lastClick + redraw() + }) + .on('dragend', () => { + // @ts-ignore + recordMouseClick(d3.event.sourceEvent, "d3", "dragend") + // @ts-ignore + const x = d3.event.sourceEvent.layerX + // @ts-ignore + const shift: bool = d3.event.sourceEvent.shiftKey + lastClick = positionToAbsoluteTime(to(x)) + const boundary1: TimeInMovie = dragStart! + const boundary2: TimeInMovie = lastClick! + dragStart = null + let start: TimeInMovie + let end: TimeInMovie + if (Math.abs(from(sub(boundary1!, boundary2!))) > 0.02) { + if (from(sub(boundary1!, boundary2)) < 0) { + start = boundary1! + end = boundary2! + } else { + start = boundary2! + end = boundary1! + } + } else { + start = lastClick + if (shift) { + end = to(endS) + } else { + end = to(Math.min(from(start) + from(defaultPlayLength()), endS)) + } + } + clear() + stopPlaying() + setup(buffers[bufferKind]!) + play(timeInMovieToTimeInBuffer(start), sub(timeInMovieToTimeInBuffer(end), timeInMovieToTimeInBuffer(start))) + redraw() + }) + .on('drag', () => { + // @ts-ignore + const x = d3.event.sourceEvent.layerX + lastClick = positionToAbsoluteTime(to(x)) + redraw() + }) + ) + +var svgReferenceAnnotations: d3.Selection = svg.append('g') +var svgAnnotations: d3.Selection = svg.append('g') +let lastChangedAnnotations: Annotation[] = [] + +var fixedButtonOffset = 0.05 diff --git a/public/telemetry.ts b/public/telemetry.ts new file mode 100644 index 0000000..8895891 --- /dev/null +++ b/public/telemetry.ts @@ -0,0 +1,172 @@ + +const telemetryEnabled: boolean = _.has($.url().param(), 'telemetry') ? $.url().param() === 'false' : true +let interactions: Interaction[] = []; + +function sendTelemetry() { + const is = interactions + interactions = [] + if (_.isArray(is) && is.length > 0) { + try { + $.ajax({ + type: 'POST', + data: { + interactions: is, + worker: $.url().param().worker, + segment: segment, + token: token, + browser: browser, + width: canvas.width, + height: canvas.height, + words: words, + selected: selected, + start: startS, + end: endS, + startTime: startTime, + startOffset: startOffset, + lastClick: lastClick, + date: new Date(), + annotations: _.map(annotations, cloneAnnotation) + }, + dataType: 'application/json', + url: $.url().attr('protocol') + '://' + $.url().attr('host') + ':' + (parseInt($.url().attr('port')) + 1) + '/telemetry' + }) + } catch (err) { + // We try our best to send back telemetry, but if it doesn't work, that's not an issue + } + } +} + +if (telemetryEnabled) + // every 10 seconds + setInterval(sendTelemetry, 5000) + +interface Interaction { + kind: string +} + +interface Message extends Interaction { + level: string, + data: string +} + +interface Click extends Interaction { + x: number, + y: number, + relativeX: number, + relativeY: number, + elements: any[] +} + +interface DragStart extends Click { +} + +interface DragEnd extends Click { +} + +interface Resize extends Interaction { + pagex: number, + pagey: number, +} + +interface Keypress extends Interaction { + key: string, + element?: string +} + +interface Send extends Interaction { + data: any, + server: string, + port: number, + why: string +} + +interface Receive extends Interaction { + response: any, + error: any, + status: string, + server: string + port: number, + why: string +} + +function recordMessage(i: Omit) { + const j: Message = { + kind: 'message', + ...i + } + interactions.push(j) +} + +function recordSend(i: Omit) { + const j: Send = { + kind: 'send', + ...i + } + interactions.push(j) +} + +_.mixin({ + deeply: + // @ts-ignore + function(obj, fn) { + if (_.isObjectLike(obj)) { + return _.mapValues(_.mapValues(obj, fn), function(v) { + // @ts-ignore + return _.isPlainObject(v) || _.isArray(v) ? _.deeply(v, fn) : v; + // return _.isPlainObject(v) ? _.deeply(v, fn) : _.isArray(v) ? v.map(function(x) { + // // @ts-ignore + // return _.deeply(x, fn); + // }) : v; + }) + } else { + return fn(obj) + } + }, +}) + +function recordReceive(i: Omit) { + // @ts-ignore + i.response = _.deeply(i.response, function(val, key?) { + if (_.isArray(val) || _.isString(val) || _.isNumber(val) || _.isDate(val)) { + return val + } + if (_.isObjectLike(val)) { + val = _.omit(val, _.functions(val)) + if (_.has(val, 'responseJSON')) { + val = _.omit(val, ['responseText']) + } + return val + } + return null + }) + const j: Receive = { + kind: 'receive', + ...i + } + interactions.push(j) +} + +function recordKeypress(key: string, element?: string) { + const j: Keypress = { + kind: 'keypress', + key: key, + element: element + } + interactions.push(j) +} + +function recordMouseClick(e: JQuery.Event, element: any, element2?: any) { + const j: Click = { + kind: 'mouse', + elements: _.isUndefined(element2) ? [element] : [element, element2], + // @ts-ignore + relativeX: e.offsetX, // d3.event.layerX, + // @ts-ignore + relativeY: e.offsetY, // d3.event.layerY, + // @ts-ignore + x: e.pageX, // d3.event.x, + // @ts-ignore + y: e.pageY // d3.event.y + } + interactions.push(j) +} diff --git a/public/types.ts b/public/types.ts new file mode 100644 index 0000000..b5a92f6 --- /dev/null +++ b/public/types.ts @@ -0,0 +1,128 @@ +enum LoadingState { + ready, + submitting, + loading +} + +// https://www.everythingfrontend.com/posts/newtype-in-typescript.html +type TimeInBuffer = { value: number; readonly __tag: unique symbol } +type TimeInSegment = { value: number; readonly __tag: unique symbol } +type TimeInMovie = { value: number; readonly __tag: unique symbol } +type PositionInSpectrogram = { value: number; readonly __tag: unique symbol } + +function add(t1: T, t2: T): T { + return lift2(t1, t2, (a, b) => a + b) +} + +function sub(t1: T, t2: T): T { + return lift2(t1, t2, (a, b) => a - b) +} + +function addConst(t: T, c: number): T { + return lift(t, a => a + c) +} + +function subConst(t: T, c: number): T { + return lift(t, a => a - c) +} + +function addMin(t1: T, t2: T, t3: T): T { + return lift3(t1, t2, t3, (a, b, c) => Math.min(a + b, c)) +} + +function subMax(t1: T, t2: T, t3: T): T { + return lift3(t1, t2, t3, (a, b, c) => Math.max(a - b, c)) +} + +function to( + value: T['value'] +): T { + return (value as any) as T +} + +function from(value: T): T['value'] { + return (value as any) as T['value'] +} + +function lift( + value: T, + callback: (value: T['value']) => T['value'] +): T { + return callback(value) +} + +function lift2( + x: T, + y: T, + callback: (x: T['value'], y: T['value']) => T['value'] +): T { + return callback(x, y) +} + +function lift3( + x: T, + y: T, + z: T, + callback: (x: T['value'], y: T['value'], z: T['value']) => T['value'] +): T { + return callback(x, y, z) +} + +interface Annotation { + word: string + index: number + startTime?: TimeInMovie + endTime?: TimeInMovie + lastClickTimestamp?: number + id?: string | number + visuals?: Visuals +} + +interface Visuals { + group: d3.Selection + text: d3.Selection + startLine: d3.Selection + startLineHandle: d3.Selection + endLine: d3.Selection + endLineHandle: d3.Selection + filler: d3.Selection + topLine: d3.Selection +} + +interface Buffers { + normal: null | AudioBuffer + half: null | AudioBuffer +} + +enum BufferType { + normal = 'normal', + half = 'half', +} + +enum DragPosition { + start = 'startTime', + end = 'endTime', + both = 'both', +} + +// This clones without the UI elements +function cloneAnnotation(a: Annotation): Annotation { + return { + startTime: a.startTime, + endTime: a.endTime, + lastClickTimestamp: a.lastClickTimestamp, + word: a.word, + index: a.index, + } +} + +function isValidAnnotation(a: Annotation) { + return ( + _.has(a, 'startTime') && + !_.isUndefined(a.startTime) && + !_.isNull(a.startTime) && + _.has(a, 'endTime') && + !_.isUndefined(a.endTime) && + !_.isNull(a.endTime) + ) +} diff --git a/public/undo.ts b/public/undo.ts new file mode 100644 index 0000000..6765db3 --- /dev/null +++ b/public/undo.ts @@ -0,0 +1,25 @@ + +function pushUndo(annotation: Annotation) { + lastChangedAnnotations.push(_.clone(annotation)) +} + +function popUndo() { + const last = _.last(lastChangedAnnotations) + lastChangedAnnotations = lastChangedAnnotations.slice(0, -1) + return last +} + +function clearUndo() { + lastChangedAnnotations = [] +} + +function undo() { + if (lastChangedAnnotations != []) { + const ann = popUndo()! + annotations[ann.index].startTime = ann.startTime + annotations[ann.index].endTime = ann.endTime + updateWord(annotations[ann.index]) + } else { + message('warning', 'Nothing to undo') + } +} diff --git a/tsconfig.json b/tsconfig.json index 9512ff3..a11d938 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ /* Basic Options */ // "incremental": true, /* Enable incremental compilation */ "target": "ES2017", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */ - "module": "umd", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ + "module": "amd", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ // "lib": [], /* Specify library files to be included in the compilation. */ // "allowJs": true, /* Allow javascript files to be compiled. */ // "checkJs": true, /* Report errors in .js files. */ @@ -11,8 +11,8 @@ // "declaration": true, /* Generates corresponding '.d.ts' file. */ // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ "sourceMap": true, /* Generates corresponding '.map' file. */ - // "outFile": "./", /* Concatenate and emit output to single file. */ - // "outDir": "./", /* Redirect output structure to the directory. */ + "outFile": "public/gui.js", /* Concatenate and emit output to single file. */ + // "outDir": ".", /* Redirect output structure to the directory. */ // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ // "composite": true, /* Enable project compilation */ // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ @@ -59,5 +59,17 @@ /* Experimental Options */ // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ - } + }, + "files": [ + "public/types.ts", + "public/extensions.ts", + "public/misc.ts", + "public/state.ts", + "public/telemetry.ts", + "public/undo.ts", + "public/gui.ts", + "public/shortcuts.ts", + "public/help.ts", + "public/debugPane.ts", + ] }