diff --git a/lib/media/media_source_engine.js b/lib/media/media_source_engine.js
index 035bda2b16..3c54b0184e 100644
--- a/lib/media/media_source_engine.js
+++ b/lib/media/media_source_engine.js
@@ -891,8 +891,7 @@ shaka.media.MediaSourceEngine = class {
     const ContentType = shaka.util.ManifestParserUtils.ContentType;
 
     if (contentType == ContentType.TEXT) {
-      if (this.manifestType_ == shaka.media.ManifestParser.HLS &&
-          reference.discontinuitySequence >= 0) {
+      if (this.manifestType_ == shaka.media.ManifestParser.HLS) {
         // This won't be known until the first video segment is appended.
         const offset = await this.textSequenceModeOffset_;
         this.textEngine_.setTimestampOffset(offset);
diff --git a/lib/text/vtt_text_parser.js b/lib/text/vtt_text_parser.js
index 2985364ea1..a05b6ee4e3 100644
--- a/lib/text/vtt_text_parser.js
+++ b/lib/text/vtt_text_parser.js
@@ -84,10 +84,15 @@ shaka.text.VttTextParser = class {
     // HLS content, which should use X-TIMESTAMP-MAP and periodStart instead.
     let offset = time.vttOffset;
 
-    // Only use 'X-TIMESTAMP-MAP' with HLS.  This overrides offset above.
-    if (blocks[0].includes('X-TIMESTAMP-MAP') &&
-        this.manifestType_ == shaka.media.ManifestParser.HLS) {
-      offset = this.computeHlsOffset_(blocks[0], time);
+    if (this.manifestType_ == shaka.media.ManifestParser.HLS) {
+      // Only use 'X-TIMESTAMP-MAP' with HLS.
+      if (blocks[0].includes('X-TIMESTAMP-MAP')) {
+        offset = this.computeHlsOffset_(blocks[0], time);
+      } else if (time.periodStart && time.vttOffset == time.periodStart) {
+        // In the case where X-TIMESTAMP-MAP is not used and it is HLS, we
+        // should not use offset unless segment-relative times are used.
+        offset = 0;
+      }
     }
 
     // Parse VTT regions.