diff --git a/src/_avif.c b/src/_avif.c
index 14ec10bf5a8..a01c2556192 100644
--- a/src/_avif.c
+++ b/src/_avif.c
@@ -18,7 +18,6 @@ static PyTypeObject AvifEncoder_Type;
 // Decoder type
 typedef struct {
     PyObject_HEAD avifDecoder *decoder;
-    PyObject *data;
     char *mode;
 } AvifDecoderObject;
 
@@ -664,7 +663,7 @@ _encoder_finish(AvifEncoderObject *self) {
 // Decoder functions
 PyObject *
 AvifDecoderNew(PyObject *self_, PyObject *args) {
-    PyObject *avif_bytes;
+    Py_buffer buffer;
     AvifDecoderObject *self = NULL;
     avifDecoder *decoder;
 
@@ -674,7 +673,7 @@ AvifDecoderNew(PyObject *self_, PyObject *args) {
 
     avifResult result;
 
-    if (!PyArg_ParseTuple(args, "Ssi", &avif_bytes, &codec_str, &max_threads)) {
+    if (!PyArg_ParseTuple(args, "y*si", &buffer, &codec_str, &max_threads)) {
         return NULL;
     }
 
@@ -690,9 +689,6 @@ AvifDecoderNew(PyObject *self_, PyObject *args) {
         return NULL;
     }
 
-    Py_INCREF(avif_bytes);
-    self->data = avif_bytes;
-
     decoder = avifDecoderCreate();
 #if AVIF_VERSION >= 80400
     decoder->maxThreads = max_threads;
@@ -707,9 +703,7 @@ AvifDecoderNew(PyObject *self_, PyObject *args) {
 #endif
     decoder->codecChoice = codec;
 
-    result = avifDecoderSetIOMemory(
-        decoder, (uint8_t *)PyBytes_AS_STRING(self->data), PyBytes_GET_SIZE(self->data)
-    );
+    result = avifDecoderSetIOMemory(decoder, buffer.buf, buffer.len);
     if (result != AVIF_RESULT_OK) {
         PyErr_Format(
             exc_type_for_avif_result(result),
@@ -749,7 +743,6 @@ _decoder_dealloc(AvifDecoderObject *self) {
     if (self->decoder) {
         avifDecoderDestroy(self->decoder);
     }
-    Py_XDECREF(self->data);
     Py_RETURN_NONE;
 }