diff --git a/libavformat/internal.h b/libavformat/internal.h
index bffb8e66ff..f24c68703f 100644
--- a/libavformat/internal.h
+++ b/libavformat/internal.h
@@ -82,6 +82,17 @@ typedef struct FFFormatContext {
      */
     int nb_interleaved_streams;
 
+    /**
+     * Whether the timestamp shift offset has already been determined.
+     * -1: disabled, 0: not yet determined, 1: determined.
+     */
+    enum {
+        AVOID_NEGATIVE_TS_DISABLED = -1,
+        AVOID_NEGATIVE_TS_UNKNOWN  = 0,
+        AVOID_NEGATIVE_TS_KNOWN    = 1,
+    } avoid_negative_ts_status;
+#define AVOID_NEGATIVE_TS_ENABLED(status) ((status) >= 0)
+
     /**
      * The interleavement function in use. Always set for muxers.
      */
@@ -135,18 +146,6 @@ typedef struct FFFormatContext {
      */
     int raw_packet_buffer_size;
 
-    /**
-     * Offset to remap timestamps to be non-negative.
-     * Expressed in timebase units.
-     * @see AVStream.mux_ts_offset
-     */
-    int64_t offset;
-
-    /**
-     * Timebase for the timestamp offset.
-     */
-    AVRational offset_timebase;
-
 #if FF_API_COMPUTE_PKT_FIELDS2
     int missing_ts_warning;
 #endif
diff --git a/libavformat/mux.c b/libavformat/mux.c
index a1917878a5..0810b674a7 100644
--- a/libavformat/mux.c
+++ b/libavformat/mux.c
@@ -388,6 +388,8 @@ fail:
 
 static int init_pts(AVFormatContext *s)
 {
+    FFFormatContext *const si = ffformatcontext(s);
+
     /* init PTS generation */
     for (unsigned i = 0; i < s->nb_streams; i++) {
         AVStream *const st = s->streams[i];
@@ -418,13 +420,16 @@ static int init_pts(AVFormatContext *s)
         }
     }
 
+    si->avoid_negative_ts_status = AVOID_NEGATIVE_TS_UNKNOWN;
     if (s->avoid_negative_ts < 0) {
         av_assert2(s->avoid_negative_ts == AVFMT_AVOID_NEG_TS_AUTO);
         if (s->oformat->flags & (AVFMT_TS_NEGATIVE | AVFMT_NOTIMESTAMPS)) {
             s->avoid_negative_ts = AVFMT_AVOID_NEG_TS_DISABLED;
+            si->avoid_negative_ts_status = AVOID_NEGATIVE_TS_DISABLED;
         } else
             s->avoid_negative_ts = AVFMT_AVOID_NEG_TS_MAKE_NON_NEGATIVE;
-    }
+    } else if (s->avoid_negative_ts == AVFMT_AVOID_NEG_TS_DISABLED)
+        si->avoid_negative_ts_status = AVOID_NEGATIVE_TS_DISABLED;
 
     return 0;
 }
@@ -638,6 +643,64 @@ static void guess_pkt_duration(AVFormatContext *s, AVStream *st, AVPacket *pkt)
     }
 }
 
+static void handle_avoid_negative_ts(FFFormatContext *si, FFStream *sti,
+                                     AVPacket *pkt)
+{
+    AVFormatContext *const s = &si->pub;
+    int64_t offset;
+
+    if (!AVOID_NEGATIVE_TS_ENABLED(si->avoid_negative_ts_status))
+        return;
+
+    if (si->avoid_negative_ts_status == AVOID_NEGATIVE_TS_UNKNOWN) {
+        int use_pts = si->avoid_negative_ts_use_pts;
+        int64_t ts = use_pts ? pkt->pts : pkt->dts;
+
+        if (ts == AV_NOPTS_VALUE)
+            return;
+        if (ts < 0 ||
+            ts > 0 && s->avoid_negative_ts == AVFMT_AVOID_NEG_TS_MAKE_ZERO) {
+            for (unsigned i = 0; i < s->nb_streams; i++) {
+                AVStream *const st2  = s->streams[i];
+                FFStream *const sti2 = ffstream(st2);
+                sti2->mux_ts_offset = av_rescale_q_rnd(-ts,
+                                                       sti->pub.time_base,
+                                                       st2->time_base,
+                                                       AV_ROUND_UP);
+            }
+        }
+        si->avoid_negative_ts_status = AVOID_NEGATIVE_TS_KNOWN;
+    }
+
+    offset = sti->mux_ts_offset;
+
+    if (pkt->dts != AV_NOPTS_VALUE)
+        pkt->dts += offset;
+    if (pkt->pts != AV_NOPTS_VALUE)
+        pkt->pts += offset;
+
+    if (si->avoid_negative_ts_use_pts) {
+        if (pkt->pts != AV_NOPTS_VALUE && pkt->pts < 0) {
+            av_log(s, AV_LOG_WARNING, "failed to avoid negative "
+                   "pts %s in stream %d.\n"
+                   "Try -avoid_negative_ts 1 as a possible workaround.\n",
+                   av_ts2str(pkt->pts),
+                   pkt->stream_index
+            );
+        }
+    } else {
+        if (pkt->dts != AV_NOPTS_VALUE && pkt->dts < 0) {
+            av_log(s, AV_LOG_WARNING,
+                   "Packets poorly interleaved, failed to avoid negative "
+                   "timestamp %s in stream %d.\n"
+                   "Try -max_interleave_delta 0 as a possible workaround.\n",
+                   av_ts2str(pkt->dts),
+                   pkt->stream_index
+            );
+        }
+    }
+}
+
 /**
  * Shift timestamps and call muxer; the original pts/dts are not kept.
  *
@@ -663,51 +726,7 @@ static int write_packet(AVFormatContext *s, AVPacket *pkt)
         if (pkt->pts != AV_NOPTS_VALUE)
             pkt->pts += offset;
     }
-
-    if (s->avoid_negative_ts > 0) {
-        int64_t offset = sti->mux_ts_offset;
-        int64_t ts = si->avoid_negative_ts_use_pts ? pkt->pts : pkt->dts;
-
-        if (si->offset == AV_NOPTS_VALUE && ts != AV_NOPTS_VALUE &&
-            (ts < 0 || s->avoid_negative_ts == AVFMT_AVOID_NEG_TS_MAKE_ZERO)) {
-            si->offset = -ts;
-            si->offset_timebase = st->time_base;
-        }
-
-        if (si->offset != AV_NOPTS_VALUE && !offset) {
-            offset = sti->mux_ts_offset =
-                av_rescale_q_rnd(si->offset,
-                                 si->offset_timebase,
-                                 st->time_base,
-                                 AV_ROUND_UP);
-        }
-
-        if (pkt->dts != AV_NOPTS_VALUE)
-            pkt->dts += offset;
-        if (pkt->pts != AV_NOPTS_VALUE)
-            pkt->pts += offset;
-
-        if (si->avoid_negative_ts_use_pts) {
-            if (pkt->pts != AV_NOPTS_VALUE && pkt->pts < 0) {
-                av_log(s, AV_LOG_WARNING, "failed to avoid negative "
-                    "pts %s in stream %d.\n"
-                    "Try -avoid_negative_ts 1 as a possible workaround.\n",
-                    av_ts2str(pkt->pts),
-                    pkt->stream_index
-                );
-            }
-        } else {
-            if (pkt->dts != AV_NOPTS_VALUE && pkt->dts < 0) {
-                av_log(s, AV_LOG_WARNING,
-                    "Packets poorly interleaved, failed to avoid negative "
-                    "timestamp %s in stream %d.\n"
-                    "Try -max_interleave_delta 0 as a possible workaround.\n",
-                    av_ts2str(pkt->dts),
-                    pkt->stream_index
-                );
-            }
-        }
-    }
+    handle_avoid_negative_ts(si, sti, pkt);
 
     if ((pkt->flags & AV_PKT_FLAG_UNCODED_FRAME)) {
         AVFrame **frame = (AVFrame **)pkt->data;
diff --git a/libavformat/options.c b/libavformat/options.c
index 1634388acb..2d55d3ad6e 100644
--- a/libavformat/options.c
+++ b/libavformat/options.c
@@ -174,7 +174,6 @@ AVFormatContext *avformat_alloc_context(void)
         return NULL;
     }
 
-    si->offset = AV_NOPTS_VALUE;
     si->shortest_end = AV_NOPTS_VALUE;
 
     return s;
diff --git a/tests/fate/matroska.mak b/tests/fate/matroska.mak
index 3bc35d0f4c..3d8110a434 100644
--- a/tests/fate/matroska.mak
+++ b/tests/fate/matroska.mak
@@ -94,8 +94,8 @@ fate-matroska-dovi-write-config7: CMD = transcode mov $(TARGET_SAMPLES)/mov/dovi
 # the first packet (with the overall lowest dts) is a video packet,
 # whereas an audio packet to be muxed later has the overall lowest pts
 # which happens to be negative and therefore needs to be shifted.
-# This is currently buggy (the timestamps of the video frames muxed
-# before the first audio frame are not shifted).
+# This is currently buggy (the timestamps are not shifted properly:
+# the first audio packet has negative timestamps).
 # Also tests muxing DOVI.
 FATE_MATROSKA_FFMPEG_FFPROBE-$(call ALLYES, FILE_PROTOCOL MOV_DEMUXER        \
                                             AAC_FIXED_DECODER HEVC_DECODER   \
diff --git a/tests/ref/fate/matroska-avoid-negative-ts b/tests/ref/fate/matroska-avoid-negative-ts
index 9c23f4721c..5bc71c76f7 100644
--- a/tests/ref/fate/matroska-avoid-negative-ts
+++ b/tests/ref/fate/matroska-avoid-negative-ts
@@ -1,4 +1,4 @@
-6048ff1b45660eb544c1f0db450afff5 *tests/data/fate/matroska-avoid-negative-ts.matroska
+e31928477981a8ffad351379f6d5f14a *tests/data/fate/matroska-avoid-negative-ts.matroska
 3618353 tests/data/fate/matroska-avoid-negative-ts.matroska
 #extradata 0:      551, 0xa18acf66
 #tb 0: 1/1000
@@ -12,32 +12,32 @@
 #sample_rate 1: 44100
 #channel_layout 1: 3
 #channel_layout_name 1: stereo
-0,        -67,          0,       33,    63375, 0xc76606ab, S=1,        8
-0,        -34,        133,       33,    46706, 0x0e08a7e5, F=0x0
-0,          0,         73,       33,    29766, 0x753c031a, F=0x0
+0,        -62,          5,       33,    63375, 0xc76606ab, S=1,        8
+0,        -29,        138,       33,    46706, 0x0e08a7e5, F=0x0
 1,          0,          0,       34,      834, 0x7e7776bd
-1,         35,         35,       34,      836, 0x14a3a0ff
-0,         39,         39,       33,    19409, 0x4b948b6c, F=0x0
-1,         70,         70,       34,      836, 0xf55e9a61
-0,         73,        106,       33,    21086, 0x1b9412ce, F=0x0
-1,        105,        105,       34,      836, 0x415591f1
-0,        106,        273,       33,    62043, 0xc2356b56, F=0x0
-0,        133,        206,       33,    36175, 0x0a7df38c, F=0x0
-1,        140,        140,       34,      836, 0xe26c9bad
-0,        173,        173,       33,    16028, 0xa57fcbe9, F=0x0
-1,        174,        174,       34,      836, 0xbc8c9b66
-0,        206,        239,       33,    15428, 0x9a91f357, F=0x0
-1,        209,        209,       34,      836, 0xddeb9643
-0,        239,        406,       33,    66072, 0xa542b6d7, F=0x0
-1,        244,        244,       34,      836, 0x08a494eb
-0,        273,        339,       33,    34985, 0xbfd8ff45, F=0x0
-1,        279,        279,       34,      836, 0x94f09bb4
-0,        306,        306,       33,    16036, 0xfc39c6ea, F=0x0
-1,        314,        314,       34,      836, 0xd6358a3a
-0,        339,        373,       33,    19893, 0x7e746f4e, F=0x0
-1,        348,        348,       34,      836, 0x76ac91f1
-0,        373,        539,       33,    77576, 0xeba2e5c8, F=0x0
-1,        383,        383,       34,      836, 0xb32a86ac
+0,          5,         72,       33,    29766, 0x753c031a, F=0x0
+1,         34,         34,       34,      836, 0x14a3a0ff
+0,         38,         38,       33,    19409, 0x4b948b6c, F=0x0
+1,         69,         69,       34,      836, 0xf55e9a61
+0,         72,        105,       33,    21086, 0x1b9412ce, F=0x0
+1,        104,        104,       34,      836, 0x415591f1
+0,        105,        272,       33,    62043, 0xc2356b56, F=0x0
+0,        138,        205,       33,    36175, 0x0a7df38c, F=0x0
+1,        139,        139,       34,      836, 0xe26c9bad
+0,        172,        172,       33,    16028, 0xa57fcbe9, F=0x0
+1,        173,        173,       34,      836, 0xbc8c9b66
+0,        205,        238,       33,    15428, 0x9a91f357, F=0x0
+1,        208,        208,       34,      836, 0xddeb9643
+0,        238,        405,       33,    66072, 0xa542b6d7, F=0x0
+1,        243,        243,       34,      836, 0x08a494eb
+0,        272,        338,       33,    34985, 0xbfd8ff45, F=0x0
+1,        278,        278,       34,      836, 0x94f09bb4
+0,        305,        305,       33,    16036, 0xfc39c6ea, F=0x0
+1,        313,        313,       34,      836, 0xd6358a3a
+0,        338,        372,       33,    19893, 0x7e746f4e, F=0x0
+1,        347,        347,       34,      836, 0x76ac91f1
+0,        372,        538,       33,    77576, 0xeba2e5c8, F=0x0
+1,        382,        382,       34,      836, 0xb32a86ac
 [STREAM]
 [SIDE_DATA]
 side_data_type=DOVI configuration record
