Skip to content

Commit

Permalink
Add caption support
Browse files Browse the repository at this point in the history 
This provides support for rendering captions when the manifest's
annotation items have a text type rendering of format 'text/vtt'.

The implementation is modeled after that of MediaElement.js
  • Loading branch information
@DanOlson
DanOlson committed Jul 25, 2024
1 parent 0cb0421 commit 39e5d98
Show file tree
Hide file tree
Showing 3 changed files with 249 additions and 12 deletions.
153 changes: 143 additions & 10 deletions src/components/canvas-instance.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ export class CanvasInstance extends BaseComponent {
private _$canvasTime: JQuery;
private _$canvasTimelineContainer: JQuery;
private _$controlsContainer: JQuery;
private _$captionsButton: JQuery;
private _$captionsContainer: JQuery;
private _$durationHighlight: JQuery;
private _$hoverPreviewTemplate: JQuery;
private _$nextButton: JQuery;
Expand All @@ -69,6 +71,7 @@ export class CanvasInstance extends BaseComponent {
private _canvasClockTime: TimelineTime = 0 as TimelineTime;
private _canvasHeight = 0;
private _canvasWidth = 0;
private _captionsSelectedIndex = 0; // hide captions by default
private _waveformPanel?: WaveformPanel;
private _contentAnnotations: any[]; // todo: type as HTMLMediaElement?
private _data: IAVCanvasInstanceData = this.data();
Expand Down Expand Up @@ -102,7 +105,6 @@ export class CanvasInstance extends BaseComponent {
this._$element = $(this.options.target);
this._data = this.options.data;
this.$playerElement = $('<div class="player player--loading"></div>');
this.$playerElement.hide();
}

public loaded(): void {
Expand Down Expand Up @@ -274,12 +276,19 @@ export class CanvasInstance extends BaseComponent {
${this._data.content.fastRewind || ''}
</span>
</button>`);
this._$captionsButton = $(`
<button class="btn captions-button hide" title="Captions">
<i class="av-icon av-icon-captions" aria-hidden="true"></i>
<span class="sr-only>
</span>
</button>`);

this._$timeDisplay = $(
'<div class="time-display"><span class="canvas-time"></span> / <span class="canvas-duration"></span></div>'
);
this._$canvasTime = this._$timeDisplay.find('.canvas-time');
this._$canvasDuration = this._$timeDisplay.find('.canvas-duration');
this._$captionsContainer = $('<div class="captions-container"><span class="captions-text"></span></div>');

if (this.isVirtual()) {
this.$playerElement.addClass('virtual');
Expand Down Expand Up @@ -307,7 +316,8 @@ export class CanvasInstance extends BaseComponent {
this._data.enableFastForward ? this._$fastForward : null,
this._$nextButton,
this._$timeDisplay,
$volume
$volume,
this._$captionsButton
);
this._$canvasTimelineContainer.append(
this._$canvasHoverPreview,
Expand All @@ -321,7 +331,7 @@ export class CanvasInstance extends BaseComponent {
this._$controlsContainer
);
this.$playerElement.append(this._$canvasContainer, this._$optionsContainer);

this._$canvasContainer.append(this._$captionsContainer);
this._$canvasHoverPreview.hide();
this._$rangeHoverPreview.hide();

Expand Down Expand Up @@ -452,6 +462,17 @@ export class CanvasInstance extends BaseComponent {
return this._setCurrentTime(start);
});

this._$captionsButton.on("mouseenter focusin", () => {
this._$captionsButton.find(".caption-choices").removeClass("offscreen");
});

this._$captionsButton.on("mouseleave focusout", () => {
const choices = this._$captionsButton.find(".caption-choices");
if (!choices.is(":hover")) {
choices.addClass("offscreen");
}
});

if (newRanges) {
this._$canvasTimelineContainer.slider({
value: 0,
Expand Down Expand Up @@ -604,7 +625,19 @@ export class CanvasInstance extends BaseComponent {
vttRendering.getLabel().getValue() ??
vttRendering.getFormat().toString(),
id: vttRendering.id,
}));
}))
.sort((a, b) => {
if (a.label > b.label) {
return 1;
} else if (b.label > a.label) {
return -1;
} else {
return 0;
}
});
if (captions.length) {
captions.unshift({ id: "none", label: "None" });
}

// todo: type this
const itemData: any = {
Expand Down Expand Up @@ -641,6 +674,34 @@ export class CanvasInstance extends BaseComponent {
});
}
}

const contentAnnotation = this._contentAnnotations[0];
this._showApplicableCaptionChoices(contentAnnotation);
}

// Applies the caption choices UI (if any) for the given contentAnnotation
// to the captions button in the controls container. Since there are multiple
// contentAnnotations but just one controls container, we need to swap out
// the caption choices whenever a new contentAnnotation becomes active.
private _showApplicableCaptionChoices(contentAnnotation: any) {
// detach any caption choices pertaining to previously active media
this._$captionsButton.children('.caption-choices').detach();

if (contentAnnotation.captionChoices) {
// add caption choices for active media
this._$captionsButton.append(contentAnnotation.captionChoices);
this._$captionsButton.removeClass("hide");
} else {
this._$captionsButton.addClass("hide");
}
}

private _getSelectedCaptionsIndex() {
return this._captionsSelectedIndex;
}

private _setSelectedCaptionsIndex(index: number) {
this._captionsSelectedIndex = index;
}

private _getBody(bodies: AnnotationBody[]): AnnotationBody | null {
Expand Down Expand Up @@ -1356,11 +1417,32 @@ export class CanvasInstance extends BaseComponent {
$(`<source src="${data.source}" type="${data.format}">`)
);
if (data.captions && data.captions.length) {
data.captions.forEach(subtitle => {
const $captionChoices: JQuery = $('<div class="caption-choices offscreen"></div>');
const $captionChoicesList: JQuery = $('<ul class="caption-choices-list"></ul>');
data.captions.forEach((subtitle, idx) => {
// add the track to the media element
$mediaElement.append(
$(`<track label="${subtitle.label}" kind="subtitles" srclang="${subtitle.language}" src="${subtitle.id}" ${data.captions.indexOf(subtitle) === 0 ? "default" : ""}>`)
$(`<track label="${subtitle.label}" kind="captions" srclang="${subtitle.language}" src="${subtitle.id}" ${idx === 0 ? "default" : ""}>`)
);
})

const isSelected = idx === this._getSelectedCaptionsIndex();

// generate an entry in the caption choices list
const $captionButton: JQuery = $(
`<li class="caption-choices-list-item">
<input type="radio" class="caption-choices-input" name="caption-choice" id="caption-choice-${idx}" />
<label for="caption-choice-${idx}" class="caption-choices-label ${isSelected ? "caption-choices-selected" : ""}">${subtitle.label}</label>
</li>`
);
$captionButton.find("input").on("click", () => {
this._setSelectedCaptionsIndex(idx);
this._syncCaptions();
});
$captionChoicesList.append($captionButton);
});
$captionChoices.append($captionChoicesList);

data.captionChoices = $captionChoices;
}
}

Expand Down Expand Up @@ -1965,7 +2047,7 @@ export class CanvasInstance extends BaseComponent {
for (let i = 0; i < this._contentAnnotations.length; i++) {
contentAnnotation = this._contentAnnotations[i];

if (contentAnnotation.start <= this._canvasClockTime && contentAnnotation.end >= this._canvasClockTime) {
if (this._isActiveContentAnnotation(contentAnnotation)) {
this._checkMediaSynchronization();

if (!contentAnnotation.active) {
Expand Down Expand Up @@ -2002,6 +2084,55 @@ export class CanvasInstance extends BaseComponent {
}
}

private _isActiveContentAnnotation(contentAnnotation: { start: number; end: number; }): boolean {
return contentAnnotation.start <= this._canvasClockTime && contentAnnotation.end >= this._canvasClockTime;
}

// If the currently active contentAnnotation has captions, set up an event
// handler to render the caption text over the canvas.
private _syncCaptions(): void {
const activeAnnotation = this._contentAnnotations.find(annotation => {
return this._isActiveContentAnnotation(annotation);
});
if (!activeAnnotation) {
return;
}

const mediaElement = activeAnnotation.element[0];
const selectedCaptionsIdx = this._getSelectedCaptionsIndex();
const tracksList = mediaElement.textTracks;

this._setCaptionsText("");

$(".caption-choices-label")
.removeClass("caption-choices-selected");
$(`label[for="caption-choice-${selectedCaptionsIdx}"]`)
.addClass("caption-choices-selected");

let textTrack: TextTrack;
for (let i = 0; i < tracksList.length; i++) {
textTrack = mediaElement.textTracks[i];
textTrack.mode = "hidden";
if (i === selectedCaptionsIdx) {
$(textTrack).on("cuechange", (event: Event) => {
// @ts-ignore
const activeCues = event.target.activeCues;
const text = activeCues.length ? activeCues[0].text : "";
this._setCaptionsText(text);
});
} else {
$(textTrack).off("cuechange");
}
}
}

private _setCaptionsText(text: string) {
const textHolder = this._$captionsContainer.find(".captions-text");
textHolder.text(text);
// hide the textHolder if there is no text
textHolder.toggle(!!text);
}

private _synchronizeMedia(): void {
if (AVComponent.newRanges && this.isVirtual()) {
return;
Expand All @@ -2017,8 +2148,10 @@ export class CanvasInstance extends BaseComponent {
this._canvasClockTime - contentAnnotation.start + contentAnnotation.startOffset
);

if (contentAnnotation.start <= this._canvasClockTime && contentAnnotation.end >= this._canvasClockTime) {
if (this._isActiveContentAnnotation(contentAnnotation)) {
this._showApplicableCaptionChoices(contentAnnotation);
if (this._isPlaying) {
this._syncCaptions();
if (contentAnnotation.element[0].paused) {
const promise = contentAnnotation.element[0].play();
if (promise) {
Expand Down Expand Up @@ -2064,7 +2197,7 @@ export class CanvasInstance extends BaseComponent {
for (let i = 0, l = this._contentAnnotations.length; i < l; i++) {
contentAnnotation = this._contentAnnotations[i];

if (contentAnnotation.start <= this._canvasClockTime && contentAnnotation.end >= this._canvasClockTime) {
if (this._isActiveContentAnnotation(contentAnnotation)) {
if (this._isPlaying) {
if (contentAnnotation.element[0].readyState < 3) {
this._buffering = true;
Expand Down
108 changes: 106 additions & 2 deletions src/css/iiif-av-component.less
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@
border-right: 6px solid transparent;
}
}

&:focus .av-icon,
&:focus .uv-icon {
opacity: 1;
Expand Down Expand Up @@ -172,6 +172,25 @@
.circular-slider-handle();
}

.captions-container {
position: absolute;
bottom: 0;
height: 70px;
width: 100%;
display: flex;
justify-content: center;
align-items: center;
z-index: 1;
}

.captions-text {
background-color: rgba(20, 20, 20, 0.5);
color: white;
font-size: 16px;
padding: 0;
display: none;
}

// .ui-slider-handle:after {
// position: absolute;
// top: 0px;
Expand All @@ -183,7 +202,6 @@
// }

.controls-container {
overflow: hidden;
margin-top: 5px;
height: 36px;
}
Expand Down Expand Up @@ -284,6 +302,92 @@

}

i.av-icon-captions {
background-image: data-uri('@{img-path}cc.png');
width: 19px;
height: 15px
}

.captions-button {
display: flex;
justify-content: center;
align-items: center;
position: relative;
width: 30px;
height: 30px;

&.hide {
display: none;
}
}

.caption-choices {
background: rgba(50, 50, 50, 0.9);
border: solid 1px transparent;
border-radius: 0;
bottom: 90%;
padding: 0;
position: absolute;
visibility: visible;
width: 86px;
z-index: 10;
}

.caption-choices-list {
padding: 0;
margin: 0;
text-align: left;
}

.caption-choices-list-item {
color: #fff;
cursor: pointer;
display: block;
list-style-type: none !important;
margin: 0;
padding-bottom: 5px;
overflow: hidden;
}

.caption-choices-list-item:hover {
background-color: rgb(200, 200, 200) !important;
background-color: rgba(255, 255, 255, 0.4) !important;
}

.caption-choices-input {
clear: both;
float: left;
left: -1000px;
margin: 3px 3px 0 5px;
position: absolute;
}

.caption-choices-label {
cursor: pointer;
float: left;
font-size: 10px;
line-height: 15px;
padding: 4px 10px 0;
width: 100%;
}

.caption-choices-selected {
color: rgba(33, 248, 248, 1);
}

.offscreen {
border: 0;
clip: rect(1px, 1px, 1px, 1px);
clip-path: inset(50%);
height: 1px;
margin: -1px;
overflow: hidden;
padding: 0;
position: absolute;
width: 1px;
word-wrap: normal;
}

.av-icon-fast-forward {
background-image: data-uri('@{img-path}fast-forward.svg');
}
Expand Down
Binary file added src/css/img/cc.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.

0 comments on commit 39e5d98

Please sign in to comment.