From cc78354d1f71ca7f2cf25d2586027f55f9cee060 Mon Sep 17 00:00:00 2001 From: Glenn Engel Date: Sat, 26 Oct 2024 10:47:55 -0700 Subject: [PATCH] Improve mouse wheel handling --- native/ffreader/README.md | 29 ++++++ native/ffreader/package.json | 2 +- release/app/package.json | 2 +- release/app/yarn.lock | 4 +- src/main/store/store.ts | 4 +- src/renderer/shared/AppTypes.ts | 3 + src/renderer/util/MouseTracking.ts | 114 +++++++++++++++++++++++ src/renderer/video/AddSplitUtil.ts | 8 ++ src/renderer/video/Video.tsx | 62 +++++------- src/renderer/video/VideoFileUtils.tsx | 9 +- src/renderer/video/VideoOverlay.tsx | 89 +++++++++++++++++- src/renderer/video/VideoScrubber.tsx | 30 +++--- src/renderer/video/VideoSettingsView.tsx | 99 ++++++++++++++------ 13 files changed, 359 insertions(+), 96 deletions(-) create mode 100644 src/renderer/util/MouseTracking.ts diff --git a/native/ffreader/README.md b/native/ffreader/README.md index 50db3c5..f17020d 100644 --- a/native/ffreader/README.md +++ b/native/ffreader/README.md @@ -8,6 +8,35 @@ The package name cannot have dashes like most npm packages. This is because the If using yarn to add this module locally, you must use yarn link or the .erb/scripts/check-native-dep.js script fails running `npm ls `. Using yarn add file:../../native/ffreader from the release/app of an electron app also works. +## Package Size + +When the final app gets packaged, the .gitignore and .npmignore files tell the packager what files to leave out of the native module. The .gitignore and .npmignore need to be in the native library folder and not in a parent. + +When building the Electron app that utilizes this package files to exclude are added to the top level package.json file with ! prefix: + +```json +"build": { + "files": [ + "dist", + "node_modules", + "package.json", + "!node_modules/**/*.cpp", + "!node_modules/**/*.h", + "!node_modules/**/*.md", + "!node_modules/**/lib-build/**/*", + "!node_modules/**/ffmpeg*/**/*" + ], +} +``` + +## Building static opencv + +To avoid linking issues when deploying to other machines than the build machine opencv is built as a static library and linked directly to the native add-on. + +```bash +yarn opencv-build +``` + ## Building ffmpeg on Windows Native modules must be build with MSVC toolchains. diff --git a/native/ffreader/package.json b/native/ffreader/package.json index af71bee..ef2dddb 100644 --- a/native/ffreader/package.json +++ b/native/ffreader/package.json @@ -1,6 +1,6 @@ { "name": "crewtimer_video_reader", - "version": "1.0.10", + "version": "1.0.9", "description": "A node electron utility to read mp4 files for CrewTimer Video Review", "main": "index.js", "types": "index.d.ts", diff --git a/release/app/package.json b/release/app/package.json index 6c4b5e8..9f37a00 100644 --- a/release/app/package.json +++ b/release/app/package.json @@ -1,6 +1,6 @@ { "name": "crewtimer-video-review", - "version": "1.0.10", + "version": "1.0.11", "description": "Review finish line video and post timing data to CrewTimer.", "license": "MIT", "author": { diff --git a/release/app/yarn.lock b/release/app/yarn.lock index 8013f21..392452a 100644 --- a/release/app/yarn.lock +++ b/release/app/yarn.lock @@ -470,8 +470,8 @@ create-require@^1.1.0: resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== -crewtimer_video_reader@../../native/ffreader: - version "1.0.8" +crewtimer_video_reader@../../native/ffreader/: + version "1.0.9" dependencies: bindings "^1.5.0" prebuild-install "^7.1.1" diff --git a/src/main/store/store.ts b/src/main/store/store.ts index 01912b2..6757e45 100644 --- a/src/main/store/store.ts +++ b/src/main/store/store.ts @@ -7,7 +7,9 @@ import { notifyChange } from '../../renderer/util/Util'; import { getMainWindow } from '../mainWindow'; /** Stored data instance for on-disk storage */ -const store = new Store(); +const store = new Store({ + name: 'ct-video-review', // this will create 'ct-video-review.json' +}); /** In-memory cache for settings */ const memCache = new Map(); diff --git a/src/renderer/shared/AppTypes.ts b/src/renderer/shared/AppTypes.ts index 5669749..ebf3532 100644 --- a/src/renderer/shared/AppTypes.ts +++ b/src/renderer/shared/AppTypes.ts @@ -1,3 +1,5 @@ +import { KeyMap } from 'crewtimer-common'; + export interface AppImage { status: string; width: number; @@ -12,6 +14,7 @@ export interface AppImage { fileStartTime: number; fileEndTime: number; motion: { x: number; y: number; dt: number; valid: boolean }; + sidecar?: KeyMap; } export interface Rect { diff --git a/src/renderer/util/MouseTracking.ts b/src/renderer/util/MouseTracking.ts new file mode 100644 index 0000000..6c29546 --- /dev/null +++ b/src/renderer/util/MouseTracking.ts @@ -0,0 +1,114 @@ +/** + * Type representing a 2D position with x and y coordinates. + */ +type Position = { + x: number; + y: number; +}; + +/** + * Callback type for handling mouse velocity and position updates. + * + * @callback MouseVelocityCallback + * @param {Position} position - The current mouse position. + * @param {number} velocity - The calculated mouse velocity in pixels per second. + * @param {number} pct - Percent of the element that the mouse is over. + */ +type MouseVelocityCallback = ( + position: Position, + velocity: number, + pct: number +) => void; + +/** + * Creates a mouse move event handler function that calculates smoothed mouse velocity + * and updates the mouse position. The function adjusts the behavior based on velocity thresholds: + * - If velocity is less than 'velocityKnee' px/s, the position is reduced to 1/4 of the actual velocity. + * - If the position is less than expected and velocity is more than 'velocityKnee' px/s, the position gradually catches up. + * + * @param {MouseVelocityCallback} callback - A callback function to be called with updated position and velocity + * at approximately 10 times per second. + * @returns {(event: MouseEvent) => void} A mouse move event handler function. + * + * @example + * // Create a handler that logs the position and velocity + * const handleMouseMove = createMouseMoveHandler((position, velocity) => { + * console.log("Position:", position, "Velocity:", velocity); + * }); + * + * // Attach the handler to an element's onMouseMove event in a React component: + * //
+ */ +export function createMouseMoveHandler(callback: MouseVelocityCallback) { + let lastPosition: Position | null = null; + let lastTimestamp: number | null = null; + let smoothedVelocity = 0; + let smoothedX = 0; + const velocityKnee = 20; // velocity threshold for ramping up or down + const alpha = 0.5; // smoothing factor for exponential average + const xAlpha = 0.33; // smoothing factor for exponential average + + /** + * Updates the position based on the calculated velocity, adjusting the behavior if necessary. + * + * @param {Position} newPosition - The new mouse position. + * @param {number} velocity - The current velocity in pixels per second. + * @param {number} pct - Percent of the element that the mouse is over. + */ + const updatePosition = ( + newPosition: Position, + velocity: number, + pct: number + ) => { + if (velocity < velocityKnee) { + // Slow down the position calculation if velocity is below velocityKnee px/s + newPosition.x += (newPosition.x - lastPosition!.x) * 0.25; + } else if (lastPosition && velocity > velocityKnee) { + // Adjust position based on velocity, ramping up if necessary + const predictedX = lastPosition.x + smoothedVelocity / 10; + + if (newPosition.x < predictedX) + newPosition.x += (predictedX - newPosition.x) * 0.25; + } + + smoothedX = xAlpha * newPosition.x + (1 - xAlpha) * smoothedX; + newPosition.x = smoothedX; + callback(newPosition, smoothedVelocity, pct); + }; + + /** + * Handles the mouse move event, calculating the velocity and position adjustments, then calls the callback. + * + * @param {MouseEvent} event - The mouse move event containing the current mouse position. + */ + return (event: React.MouseEvent) => { + const currentTime = Date.now(); + const currentPosition = { x: event.clientX, y: event.clientY }; + let pct = 0.5; + if (event.target instanceof Element) { + const elementLeft = event.target.getBoundingClientRect().left; + currentPosition.x -= elementLeft; + pct = Math.min( + 100, + Math.max(0, currentPosition.x / (event.target.clientWidth - 1)) + ); + } + + if (lastPosition && lastTimestamp) { + const timeElapsed = (currentTime - lastTimestamp) / 1000; // convert to seconds + if (timeElapsed == 0) { + return; + } + const distance = currentPosition.x - lastPosition.x; + const velocity = distance / timeElapsed; + + // Exponential smoothing for velocity + smoothedVelocity = alpha * velocity + (1 - alpha) * smoothedVelocity; + + updatePosition(currentPosition, smoothedVelocity, pct); + } + + lastPosition = currentPosition; + lastTimestamp = currentTime; + }; +} diff --git a/src/renderer/video/AddSplitUtil.ts b/src/renderer/video/AddSplitUtil.ts index b5a9d79..3c9ce2b 100644 --- a/src/renderer/video/AddSplitUtil.ts +++ b/src/renderer/video/AddSplitUtil.ts @@ -15,6 +15,7 @@ import uuidgen from 'short-uuid'; import { setToast } from 'renderer/Toast'; import { getMobileConfig, getWaypoint } from 'renderer/util/UseSettings'; +let lastAddSplit = 0; export const performAddSplit = () => { const videoBow = getVideoBow(); const selectedEvent = getVideoEvent(); @@ -26,6 +27,13 @@ export const performAddSplit = () => { const activeEvent = mobileConfig?.eventList?.find( (event) => event.EventNum === selectedEvent ); + + const now = Date.now(); + const deltaT = now - lastAddSplit; + if (deltaT < 500) { + return; // probable double click + } + lastAddSplit = now; if (!mobileConfig || !activeEvent || disabled) { setToast({ severity: 'error', diff --git a/src/renderer/video/Video.tsx b/src/renderer/video/Video.tsx index 4f2a592..2520674 100644 --- a/src/renderer/video/Video.tsx +++ b/src/renderer/video/Video.tsx @@ -1,5 +1,11 @@ import { Box, Typography, Stack, Tooltip } from '@mui/material'; -import React, { useCallback, useEffect, useRef, useState } from 'react'; +import React, { + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from 'react'; import { convertTimestampToString } from '../shared/Util'; import { useDebouncedCallback } from 'use-debounce'; import makeStyles from '@mui/styles/makeStyles'; @@ -13,7 +19,6 @@ import { setVideoTimestamp, useResetZoomCounter, useImage, - useMouseWheelInverted, useTimezoneOffset, useTravelRightToLeft, useVideoError, @@ -213,7 +218,6 @@ const VideoImage: React.FC<{ width: number; height: number }> = ({ const [videoFile] = useVideoFile(); const holdoffChanges = useRef(false); const [videoError] = useVideoError(); - const [wheelInverted] = useMouseWheelInverted(); const [travelRightToLeft] = useTravelRightToLeft(); const [resetZoomCount] = useResetZoomCounter(); const destSize = useRef({ width, height }); @@ -239,9 +243,6 @@ const VideoImage: React.FC<{ width: number; height: number }> = ({ const videoOverlayRef = useRef(null); const offscreenCanvas = useRef(document.createElement('canvas')); - const wheelTime = useRef(0); - const deltaTAvg = useRef(0); - useEffect(() => { mouseTracking.current.mouseDown = false; srcCenter.current = { x: image.width / 2, y: image.height / 2 }; @@ -368,6 +369,19 @@ const VideoImage: React.FC<{ width: number; height: number }> = ({ } }, [image]); + const videoOverlay = useMemo( + () => ( + + ), + [width, height] + ); + const selectLane = (point: Point) => { const laneLines = getVideoSettings() .guides.filter((lane) => lane.dir === Dir.Horiz && lane.enabled) @@ -438,6 +452,7 @@ const VideoImage: React.FC<{ width: number; height: number }> = ({ } } else if (!isZooming()) { const finish = getFinishLine(); + applyZoom({ zoom: 5, srcPoint: { @@ -523,32 +538,6 @@ const VideoImage: React.FC<{ width: number; height: number }> = ({ [image] ); - const handleWheel = useCallback((event: React.WheelEvent) => { - if (holdoffChanges.current) { - return; - } - - const now = Date.now(); - const deltaT = Math.min(500, now - wheelTime.current); - - if (deltaT >= 500 || Math.sign(deltaT) !== Math.sign(deltaTAvg.current)) { - deltaTAvg.current = deltaT; - } - const alpha = 0.75; - deltaTAvg.current = deltaTAvg.current * alpha + deltaT * (1 - alpha); - wheelTime.current = now; - // console.log( - // `Wheel: ${event.deltaY} dt: ${deltaT}, avg: ${deltaTAvg.current}` - // ); - - // Use 1 for slow click rate. Use 6 for fast click rate unless zooming and then use 2. - const delta = - (wheelInverted ? -1 : 1) * - Math.sign(event.deltaY) * - (deltaTAvg.current > 200 ? 1 : isZooming() ? 2 : 6); - moveToFrame(getVideoFrameNum(), delta); - }, []); - const handleRightClick = (event: React.MouseEvent) => { event.preventDefault(); event.stopPropagation(); @@ -578,7 +567,6 @@ const VideoImage: React.FC<{ width: number; height: number }> = ({ return ( = ({ } /> )} - + {videoOverlay} ); diff --git a/src/renderer/video/VideoFileUtils.tsx b/src/renderer/video/VideoFileUtils.tsx index 68386fb..7f7d671 100644 --- a/src/renderer/video/VideoFileUtils.tsx +++ b/src/renderer/video/VideoFileUtils.tsx @@ -167,6 +167,9 @@ const doRequestVideoFrame = async ({ (1000000 * imageStart.numFrames) / imageStart.fps ); + const fileStatus = getFileStatusList().find((f) => { + return f.filename === videoFile; + }); openFileStatus = { filename: videoFile, open: true, @@ -175,9 +178,8 @@ const doRequestVideoFrame = async ({ endTime: imageEndTime, duration: imageEndTime - imageStart.tsMicro, fps: imageStart.fps, - sidecar: {}, + sidecar: fileStatus?.sidecar || {}, }; - // console.log(JSON.stringify(openFileStatus, null, 2)); } let seekPos = @@ -243,6 +245,7 @@ const doRequestVideoFrame = async ({ imageStart.fileStartTime = openFileStatus.startTime / 1000; imageStart.fileEndTime = openFileStatus.endTime / 1000; + imageStart.sidecar = openFileStatus.sidecar; setImage(imageStart); if (fromClick) { // force a jump in the VideoScrubber @@ -425,7 +428,7 @@ export const refreshDirList = async (videoDir: string) => { endTime: 300, duration: 300, fps: 60, - sidecar: {}, + sidecar: videoSidecar || {}, }; if (videoSidecar.file) { fileStatus.numFrames = videoSidecar.file.numFrames; diff --git a/src/renderer/video/VideoOverlay.tsx b/src/renderer/video/VideoOverlay.tsx index 98b393e..1d1ce6a 100644 --- a/src/renderer/video/VideoOverlay.tsx +++ b/src/renderer/video/VideoOverlay.tsx @@ -11,16 +11,19 @@ import { showErrorDialog } from 'renderer/util/ErrorDialog'; import { saveVideoSidecar } from './VideoFileUtils'; import { Dir, + getImage, getVideoFile, + getVideoFrameNum, getVideoScaling, getVideoSettings, GuideLine, - useImage, + useMouseWheelInverted, useVideoScaling, useVideoSettings, } from './VideoSettings'; import { drawText, + moveToFrame, notifiyGuideChanged, Point, translateMouseEvent2Src, @@ -63,15 +66,94 @@ const VideoOverlay = forwardRef( const [adjustingOverlay, setAdjustingOverlay] = useAdjustingOverlay(); const [nearEdge] = useNearEdge(); const [courseConfig, setCourseConfig] = useState(getVideoSettings()); - const [image] = useImage(); const [videoSettings, setVideoSettings] = useVideoSettings(); const [videoScaling] = useVideoScaling(); + const [wheelInverted] = useMouseWheelInverted(); + const wheelTracking = useRef({ + millis: 0, + dtAvg: 0, + count: 0, + velocity: 0, + }); + const isZooming = () => videoScaling.zoom !== 1; useEffect(() => { // init volatile copy used while moving the mouse setCourseConfig(videoSettings); }, [videoSettings]); + useEffect(() => { + let lastT = wheelTracking.current.count; + const timer = setInterval(() => { + if (lastT !== wheelTracking.current.count) { + lastT = wheelTracking.current.count; + // console.log( + // `wheel v=${wheelTracking.current.velocity}, deltaT=${ + // wheelTracking.current.dtAvg + // }, tc=${wheelTracking.current.count} ${Date.now() % 1000}` + // ); + } + }, 100); + return () => clearInterval(timer); + }, []); + + const onWheelMove = useCallback( + (event: React.WheelEvent) => { + let scrollAmount; + + switch (event.deltaMode) { + case 0: // Pixel-based scroll (likely from a trackpad) + scrollAmount = event.deltaY; + break; + case 1: // Line-based scroll (likely from a traditional mouse) + scrollAmount = event.deltaY * 16; // Approximate conversion to pixels + break; + case 2: // Page-based scroll + scrollAmount = event.deltaY * window.innerHeight; + break; + default: + scrollAmount = event.deltaY; + } + + const now = Date.now(); + const deltaT = Math.min(500, now - wheelTracking.current.millis); + if (deltaT <= 0) { + return; + } + const velocity = Math.abs(scrollAmount) / deltaT; + wheelTracking.current.millis = now; + // if (deltaT > 200 && deltaT < 500) { + // return; // Ignore as the user pauses to restart scroll + // } + if (deltaT === 500 || wheelTracking.current.dtAvg === 500) { + wheelTracking.current.dtAvg = deltaT; + wheelTracking.current.velocity = velocity; + } else { + const alpha = 0.5; + wheelTracking.current.dtAvg = + wheelTracking.current.dtAvg * alpha + deltaT * (1 - alpha); + wheelTracking.current.velocity = + wheelTracking.current.velocity * alpha + velocity * (1 - alpha); + } + wheelTracking.current.count++; + + // console.log( + // `deltaT=${deltaT} avg=${wheelTracking.current.dtAvg} dir=${Math.sign( + // event.deltaY + // )}` + // ); + + // If mouse moving 'slow', the use 1 for frame delta. Otherwise use a larger + // factor depending on the zoom level. + const delta = + (wheelInverted ? -1 : 1) * + Math.sign(event.deltaY) * + (wheelTracking.current.velocity < 1 ? 1 : isZooming() ? 2 : 3); + setTimeout(() => moveToFrame(getVideoFrameNum(), delta), 10); + }, + [] + ); + const drawBox = useCallback( ( context: CanvasRenderingContext2D, @@ -132,6 +214,7 @@ const VideoOverlay = forwardRef( useEffect(() => { const canvas = canvasRef.current; const context = canvas?.getContext('2d'); + const image = getImage(); if (canvas && context) { canvas.width = videoScaling.destWidth; canvas.height = videoScaling.destHeight; @@ -247,6 +330,7 @@ const VideoOverlay = forwardRef( if (!withinBounds) { return; } + const image = getImage(); const videoScaling = getVideoScaling(); const margin = 20 * videoScaling.pixScale; @@ -367,6 +451,7 @@ const VideoOverlay = forwardRef( { relativePositions.sort((a, b) => a.pos - b.pos); // Find the nearest click and jump there if found - const candidates = relativePositions.map(({ pos: p }) => p); - const pos = getImage().frameNum / getImage().numFrames; - const [index] = findClosestNumAndIndex(candidates, pos); - const item = relativePositions[index]?.data; - if (item) { - setTimeout(() => { - seekToTimestamp(item.Time, true); - if (item.EventNum !== '?') { - setSelectedEvent(item.EventNum); - } - if (item.Bow && item.Bow !== '*') { - setSelectedBow(item.Bow); - } - }, 100); - } + // const candidates = relativePositions.map(({ pos: p }) => p); + // const pos = getImage().frameNum / getImage().numFrames; + // const [index] = findClosestNumAndIndex(candidates, pos); + // const item = relativePositions[index]?.data; + // if (item) { + // setTimeout(() => { + // seekToTimestamp(item.Time, true); + // if (item.EventNum !== '?') { + // setSelectedEvent(item.EventNum); + // } + // if (item.Bow && item.Bow !== '*') { + // setSelectedBow(item.Bow); + // } + // }, 100); + // } return [filteredTimes, filteredScoredTimes, relativePositions] as [ typeof filteredTimes, diff --git a/src/renderer/video/VideoSettingsView.tsx b/src/renderer/video/VideoSettingsView.tsx index c17cce2..1253486 100644 --- a/src/renderer/video/VideoSettingsView.tsx +++ b/src/renderer/video/VideoSettingsView.tsx @@ -4,10 +4,14 @@ import { Button, Checkbox, Container, + FormControl, FormControlLabel, Grid, IconButton, - Slider, + InputLabel, + MenuItem, + Select, + SelectChangeEvent, Stack, SxProps, Theme, @@ -18,7 +22,6 @@ import { import MenuIcon from '@mui/icons-material/Settings'; import { setDialogConfig } from '../util/ConfirmDialog'; import { - useMouseWheelFactor, useMouseWheelInverted, useTravelRightToLeft, useVideoSettings, @@ -57,14 +60,13 @@ export const VideoSettingsDialog = () => { const classes = useStyles(); const [videoSettings, setVideoSettings] = useVideoSettings(); const [mc] = useMobileConfig(); - const [wheelFactor, setWheelFactor] = useMouseWheelFactor(); const [wheelInverted, setWheelInverted] = useMouseWheelInverted(); const [rightToLeft, setRightToLeft] = useTravelRightToLeft(); // Handler to update the wheelFactor state - const handleSliderChange = (_event: Event, newValue: number | number[]) => { - setWheelFactor(newValue as number); - }; + // const handleSliderChange = (_event: Event, newValue: number | number[]) => { + // setWheelFactor(newValue as number); + // }; let waypointList = ['Start']; const waypoints = mc?.info.Waypoints || ''; @@ -111,37 +113,72 @@ export const VideoSettingsDialog = () => { - { + + + + Travel Direction + + + + + + + + + Lane Position + + + @@ -190,7 +227,7 @@ export const VideoSettingsDialog = () => { } /> - + {/* Wheel Factor @@ -206,7 +243,7 @@ export const VideoSettingsDialog = () => { aria-labelledby="wheel-factor-slider" /> - + */}