Skip to content

Commit

Permalink
[RSDK-9653, RSDK-9673] - Make resilient to unknown and changing frame…
Browse files Browse the repository at this point in the history 
… sizes (#33)

* Live reinit working

* Code cleanup

* Lint

* Update attr documentation

* Remove libjpeg dep

* Update tests

* Fix keyframe handling across resize boundaries

* Add some comments

* More cleanup

* Do not allow negative framerates

* Remove codec from constructor

* reinit to frameDimsChanged

* Use struct for encoded result

* Prevent side effects from partial encoder initialization

* Flag encoder to reinit if segmenter init fails

* Use debug logs to avoid spam

* Fail validation for negative framerate

* Move to validate

* Explicit encodeResult init
  • Loading branch information
@seanavery
seanavery authored Jan 13, 2025
1 parent 38d2ec8 commit a535d7b
Show file tree
Hide file tree
Showing 8 changed files with 196 additions and 175 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ FFMPEG_OPTS ?= --prefix=$(FFMPEG_BUILD) \

CGO_LDFLAGS := -L$(FFMPEG_BUILD)/lib -lavcodec -lavutil -lavformat -lz
ifeq ($(SOURCE_OS),linux)
CGO_LDFLAGS += -l:libjpeg.a -l:libx264.a
CGO_LDFLAGS += -l:libx264.a
endif
ifeq ($(SOURCE_OS),darwin)
CGO_LDFLAGS += $(HOMEBREW_PREFIX)/Cellar/x264/r3108/lib/libx264.a -liconv
Expand Down
17 changes: 2 additions & 15 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,6 @@ On the new component panel, copy and paste the following attribute template into
"storage": {
"segment_seconds": <int>,
"size_gb": <int>,
},
"cam_props": {
"width": <int>,
"height": <int>,
"framerate": <int>
}
}
```
Expand All @@ -70,13 +65,10 @@ Additionally, make sure to add your configured data manager service to the `depe
| | `upload_path` | string | optional | Custom path to use for uploading files. If not under `~/.viam/capture`, you will need to add to `additional_sync_paths` in datamanager service configuration. |
| `video` | | object | optional | |
| | `format` | string | optional | Name of video format to use (e.g., mp4). |
| | `codec` | string | optional | Name of video codec to use (e.g., h264). |
| | `codec` | string | optional | Name of video codec to use (e.g., h264). |
| | `bitrate` | integer | optional | Throughput of encoder in bits per second. Higher for better quality video, and lower for better storage efficiency. |
| | `preset` | string | optional | Name of codec video preset to use. See [here](https://trac.ffmpeg.org/wiki/Encode/H.264#a2.Chooseapresetandtune) for preset options. |
| `cam_props` | | object | optional | |
| | `width` | integer | optional | Width of the source camera frames in pixels. If unspecified, will try to autodetect by fetching a frame from the source camera. |
| | `height` | integer | optional | Height of the source camera frames in pixels. If unspecified, will try to autodetect by fetching a frame from the source camera. |
| | `framerate` | integer | optional | Number of frames per second provided by the source camera. Default is 20. |
| `framerate` | | integer | optional | Frame rate of the video in frames per second. Default value is 20 if not set. |

### Example Configuration

Expand All @@ -92,11 +84,6 @@ Additionally, make sure to add your configured data manager service to the `depe
"storage": {
"segment_seconds": 10,
"size_gb": 50,
},
"cam_props": {
"width": 640,
"height": 480,
"framerate": 25
}
},
"depends_on": [
Expand Down
88 changes: 35 additions & 53 deletions cam/cam.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ type videostore struct {
cam camera.Camera
latestFrame atomic.Pointer[image.Image]
workers *utils.StoppableWorkers
framerate int

enc *encoder
seg *segmenter
Expand All @@ -78,21 +79,13 @@ type video struct {
Format string `json:"format,omitempty"`
}

type cameraProperties struct {
Width int `json:"width"`
Height int `json:"height"`
Framerate int `json:"framerate"`
}

// Config is the configuration for the video storage camera component.
type Config struct {
Camera string `json:"camera"`
Sync string `json:"sync"`
Storage storage `json:"storage"`
Video video `json:"video,omitempty"`

// TODO(seanp): Remove once camera properties are returned from camera component.
Properties cameraProperties `json:"cam_props"`
Camera string `json:"camera"`
Sync string `json:"sync"`
Storage storage `json:"storage"`
Video video `json:"video,omitempty"`
Framerate int `json:"framerate,omitempty"`
}

// Validate validates the configuration for the video storage camera component.
Expand All @@ -109,6 +102,9 @@ func (cfg *Config) Validate(path string) ([]string, error) {
if cfg.Sync == "" {
return nil, utils.NewConfigValidationFieldRequiredError(path, "sync")
}
if cfg.Framerate < 0 {
return nil, fmt.Errorf("invalid framerate %d, must be greater than 0", cfg.Framerate)
}

return []string{cfg.Camera}, nil
}
Expand All @@ -123,7 +119,7 @@ func init() {
}

func newvideostore(
ctx context.Context,
_ context.Context,
deps resource.Dependencies,
conf resource.Config,
logger logging.Logger,
Expand All @@ -150,11 +146,11 @@ func newvideostore(
ffmppegLogLevel(logLevel)

// Create encoder to handle encoding of frames.
// TODO(seanp): Forcing h264 for now until h265 is supported.
codec := defaultVideoCodec
// TODO(seanp): Ignoring codec and using h264 for now until h265 is supported.
bitrate := defaultVideoBitrate
preset := defaultVideoPreset
format := defaultVideoFormat
vs.framerate = defaultFramerate
if newConf.Video.Bitrate != 0 {
bitrate = newConf.Video.Bitrate
}
Expand All @@ -164,39 +160,15 @@ func newvideostore(
if newConf.Video.Format != "" {
format = newConf.Video.Format
}

if newConf.Properties.Width == 0 && newConf.Properties.Height == 0 {
vs.logger.Info("received unspecified frame width and height, fetching frame to get dimensions")
for range make([]struct{}, numFetchFrameAttempts) {
frame, err := camera.DecodeImageFromCamera(ctx, rutils.MimeTypeJPEG, nil, vs.cam)
if err != nil {
vs.logger.Warn("failed to get and decode frame from camera, retrying. Error: ", err)
time.Sleep(retryInterval * time.Second)
continue
}
bounds := frame.Bounds()
newConf.Properties.Width = bounds.Dx()
newConf.Properties.Height = bounds.Dy()
vs.logger.Infof("received frame width and height: %d, %d", newConf.Properties.Width, newConf.Properties.Height)
break
}
}
if newConf.Properties.Width == 0 && newConf.Properties.Height == 0 {
return nil, fmt.Errorf("failed to get source camera width and height after %d attempts", numFetchFrameAttempts)
}

if newConf.Properties.Framerate == 0 {
newConf.Properties.Framerate = defaultFramerate
if newConf.Framerate != 0 {
vs.framerate = newConf.Framerate
}

vs.enc, err = newEncoder(
logger,
codec,
bitrate,
preset,
newConf.Properties.Width,
newConf.Properties.Height,
newConf.Properties.Framerate,
vs.framerate,
)
if err != nil {
return nil, err
Expand Down Expand Up @@ -240,7 +212,6 @@ func newvideostore(
vs.storagePath = storagePath
vs.seg, err = newSegmenter(
logger,
vs.enc,
sizeGB,
segmentSeconds,
storagePath,
Expand Down Expand Up @@ -359,7 +330,7 @@ func (vs *videostore) Properties(_ context.Context) (camera.Properties, error) {
// fetchFrames reads frames from the camera at the framerate interval
// and stores the decoded image in the latestFrame atomic pointer.
func (vs *videostore) fetchFrames(ctx context.Context) {
frameInterval := time.Second / time.Duration(vs.conf.Properties.Framerate)
frameInterval := time.Second / time.Duration(vs.framerate)
ticker := time.NewTicker(frameInterval)
defer ticker.Stop()
for {
Expand All @@ -376,7 +347,7 @@ func (vs *videostore) fetchFrames(ctx context.Context) {
lazyImage, ok := frame.(*rimage.LazyEncodedImage)
if !ok {
vs.logger.Error("frame is not of type *rimage.LazyEncodedImage")
return
continue
}
decodedImage := lazyImage.DecodedImage()
vs.latestFrame.Store(&decodedImage)
Expand All @@ -387,7 +358,7 @@ func (vs *videostore) fetchFrames(ctx context.Context) {
// processFrames grabs the latest frame, encodes, and writes to the segmenter
// which chunks video stream into clip files inside the storage directory.
func (vs *videostore) processFrames(ctx context.Context) {
frameInterval := time.Second / time.Duration(vs.conf.Properties.Framerate)
frameInterval := time.Second / time.Duration(vs.framerate)
ticker := time.NewTicker(frameInterval)
defer ticker.Stop()
for {
Expand All @@ -400,15 +371,26 @@ func (vs *videostore) processFrames(ctx context.Context) {
vs.logger.Debug("latest frame is not available yet")
continue
}
encoded, pts, dts, err := vs.enc.encode(*latestFrame)
result, err := vs.enc.encode(*latestFrame)
if err != nil {
vs.logger.Error("failed to encode frame", err)
return
vs.logger.Debug("failed to encode frame", err)
continue
}
err = vs.seg.writeEncodedFrame(encoded, pts, dts)
if result.frameDimsChanged {
vs.logger.Info("reinitializing segmenter due to encoder refresh")
err = vs.seg.initialize(vs.enc.codecCtx)
if err != nil {
vs.logger.Debug("failed to reinitialize segmenter", err)
// Hack that flags the encoder to reinitialize if segmenter fails to
// ensure that encoder and segmenter inits are in sync.
vs.enc.codecCtx = nil
continue
}
}
err = vs.seg.writeEncodedFrame(result.encodedData, result.pts, result.dts)
if err != nil {
vs.logger.Error("failed to segment frame", err)
return
vs.logger.Debug("failed to segment frame", err)
continue
}
}
}
Expand Down
Loading

0 comments on commit a535d7b

Please sign in to comment.