From ff8e9ef81cea11b56389f6017160b81914a2e0a7 Mon Sep 17 00:00:00 2001 From: ryanheise Date: Mon, 4 Jan 2021 01:04:23 +1000 Subject: [PATCH] Gapless looping on iOS (#273) --- just_audio/darwin/Classes/AudioPlayer.m | 180 +++++++++++------- .../darwin/Classes/ClippingAudioSource.m | 19 +- .../darwin/Classes/IndexedAudioSource.m | 12 +- just_audio/darwin/Classes/UriAudioSource.m | 41 ++-- just_audio/ios/Classes/IndexedAudioSource.h | 3 + just_audio/macos/Classes/IndexedAudioSource.h | 3 + 6 files changed, 171 insertions(+), 87 deletions(-) diff --git a/just_audio/darwin/Classes/AudioPlayer.m b/just_audio/darwin/Classes/AudioPlayer.m index 0cdf0ec..54f2d64 100644 --- a/just_audio/darwin/Classes/AudioPlayer.m +++ b/just_audio/darwin/Classes/AudioPlayer.m @@ -40,6 +40,7 @@ BOOL _automaticallyWaitsToMinimizeStalling; BOOL _playing; float _speed; + BOOL _justAdvanced; } - (instancetype)initWithRegistrar:(NSObject *)registrar playerId:(NSString*)idParam { @@ -76,6 +77,7 @@ _playResult = nil; _automaticallyWaitsToMinimizeStalling = YES; _speed = 1.0f; + _justAdvanced = NO; __weak __typeof__(self) weakSelf = self; [_methodChannel setMethodCallHandler:^(FlutterMethodCall* call, FlutterResult result) { [weakSelf handleMethodCall:call result:result]; @@ -208,6 +210,9 @@ IndexedAudioSource *audioSource = _indexedAudioSources[i]; while (audioSource != oldIndexedAudioSources[j]) { [self removeItemObservers:oldIndexedAudioSources[j].playerItem]; + if (oldIndexedAudioSources[j].playerItem2) { + [self removeItemObservers:oldIndexedAudioSources[j].playerItem2]; + } if (j < _index) { _index--; } else if (j == _index) { @@ -282,12 +287,12 @@ } - (void)enterBuffering:(NSString *)reason { - NSLog(@"ENTER BUFFERING: %@", reason); + //NSLog(@"ENTER BUFFERING: %@", reason); _processingState = buffering; } - (void)leaveBuffering:(NSString *)reason { - NSLog(@"LEAVE BUFFERING: %@", reason); + //NSLog(@"LEAVE BUFFERING: %@", reason); _processingState = ready; } @@ -370,8 +375,8 @@ [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(onItemStalled:) name:AVPlayerItemPlaybackStalledNotification object:playerItem]; } -- (NSMutableArray *)decodeAudioSources:(NSArray *)data { - NSMutableArray *array = [[NSMutableArray alloc] init]; +- (NSMutableArray *)decodeAudioSources:(NSArray *)data { + NSMutableArray *array = (NSMutableArray *)[[NSMutableArray alloc] init]; for (int i = 0; i < [data count]; i++) { AudioSource *source = [self decodeAudioSource:data[i]]; [array addObject:source]; @@ -409,6 +414,7 @@ } - (void)enqueueFrom:(int)index { + //NSLog(@"### enqueueFrom:%d", index); _index = index; // Update the queue while keeping the currently playing item untouched. @@ -447,16 +453,40 @@ /* [self dumpQueue]; */ // Regenerate queue - BOOL include = NO; - for (int i = 0; i < [_order count]; i++) { - int si = [_order[i] intValue]; - if (si == _index) include = YES; - if (include && _indexedAudioSources[si].playerItem != existingItem) { - //NSLog(@"inserting item %d", si); - [_player insertItem:_indexedAudioSources[si].playerItem afterItem:nil]; + if (!existingItem || _loopMode != loopOne) { + BOOL include = NO; + for (int i = 0; i < [_order count]; i++) { + int si = [_order[i] intValue]; + if (si == _index) include = YES; + if (include && _indexedAudioSources[si].playerItem != existingItem) { + //NSLog(@"inserting item %d", si); + [_player insertItem:_indexedAudioSources[si].playerItem afterItem:nil]; + if (_loopMode == loopOne) { + // We only want one item in the queue. + break; + } + } } } + // Add next loop item if we're looping + if (_loopMode == loopAll) { + int si = [_order[0] intValue]; + //NSLog(@"### add loop item:%d", si); + if (!_indexedAudioSources[si].playerItem2) { + [_indexedAudioSources[si] preparePlayerItem2]; + [self addItemObservers:_indexedAudioSources[si].playerItem2]; + } + [_player insertItem:_indexedAudioSources[si].playerItem2 afterItem:nil]; + } else if (_loopMode == loopOne) { + //NSLog(@"### add loop item:%d", _index); + if (!_indexedAudioSources[_index].playerItem2) { + [_indexedAudioSources[_index] preparePlayerItem2]; + [self addItemObservers:_indexedAudioSources[_index].playerItem2]; + } + [_player insertItem:_indexedAudioSources[_index].playerItem2 afterItem:nil]; + } + /* NSLog(@"after reorder: _player.items.count: ", _player.items.count); */ /* [self dumpQueue]; */ @@ -495,6 +525,9 @@ if (_indexedAudioSources) { for (int i = 0; i < [_indexedAudioSources count]; i++) { [self removeItemObservers:_indexedAudioSources[i].playerItem]; + if (_indexedAudioSources[i].playerItem2) { + [self removeItemObservers:_indexedAudioSources[i].playerItem2]; + } } } // Decode audio source @@ -572,6 +605,10 @@ _player.rate = _speed; } [self broadcastPlaybackEvent]; + /* NSLog(@"load:"); */ + /* for (int i = 0; i < [_indexedAudioSources count]; i++) { */ + /* NSLog(@"- %@", _indexedAudioSources[i].sourceId); */ + /* } */ } - (void)updateOrder { @@ -605,55 +642,27 @@ - (void)onComplete:(NSNotification *)notification { NSLog(@"onComplete"); - if (_loopMode == loopOne) { - __weak __typeof__(self) weakSelf = self; - [self seek:kCMTimeZero index:@(_index) completionHandler:^(BOOL finished) { - // XXX: Not necessary? - [weakSelf play]; - }]; - } else { - IndexedPlayerItem *endedPlayerItem = (IndexedPlayerItem *)notification.object; - IndexedAudioSource *endedSource = endedPlayerItem.audioSource; - if ([_orderInv[_index] intValue] + 1 < [_order count]) { - // When an item ends, seek back to its beginning. - [endedSource seek:kCMTimeZero]; - // account for automatic move to next item - _index = [_order[[_orderInv[_index] intValue] + 1] intValue]; - NSLog(@"advance to next: index = %d", _index); - [self updateEndAction]; - [self broadcastPlaybackEvent]; - } else { - // reached end of playlist - if (_loopMode == loopAll) { - NSLog(@"Loop back to first item"); - // Loop back to the beginning - // TODO: Currently there will be a gap at the loop point. - // Maybe we can do something clever by temporarily adding the - // first playlist item at the end of the queue, although this - // will affect any code that assumes the queue always - // corresponds to a contiguous region of the indexed audio - // sources. - // For now we just do a seek back to the start. - if ([_order count] == 1) { - __weak __typeof__(self) weakSelf = self; - [self seek:kCMTimeZero index:_order[0] completionHandler:^(BOOL finished) { - // XXX: Necessary? - [weakSelf play]; - }]; - } else { - // When an item ends, seek back to its beginning. - [endedSource seek:kCMTimeZero]; - __weak __typeof__(self) weakSelf = self; - [self seek:kCMTimeZero index:_order[0] completionHandler:^(BOOL finished) { - // XXX: Necessary? - [weakSelf play]; - }]; - } - } else { - [self complete]; - } - } + IndexedPlayerItem *endedPlayerItem = (IndexedPlayerItem *)notification.object; + IndexedAudioSource *endedSource = endedPlayerItem.audioSource; + + if (_loopMode == loopOne) { + [endedSource seek:kCMTimeZero]; + _justAdvanced = YES; + } else if (_loopMode == loopAll) { + [endedSource seek:kCMTimeZero]; + _index = [_order[([_orderInv[_index] intValue] + 1) % _order.count] intValue]; + [self broadcastPlaybackEvent]; + _justAdvanced = YES; + } else if ([_orderInv[_index] intValue] + 1 < [_order count]) { + [endedSource seek:kCMTimeZero]; + _index++; + [self updateEndAction]; + [self broadcastPlaybackEvent]; + _justAdvanced = YES; + } else { + // reached end of playlist + [self complete]; } } @@ -790,7 +799,9 @@ } } } else if ([keyPath isEqualToString:@"currentItem"] && _player.currentItem) { - if (_player.currentItem.status == AVPlayerItemStatusFailed) { + IndexedPlayerItem *playerItem = (IndexedPlayerItem *)change[NSKeyValueChangeNewKey]; + IndexedPlayerItem *oldPlayerItem = (IndexedPlayerItem *)change[NSKeyValueChangeOldKey]; + if (playerItem.status == AVPlayerItemStatusFailed) { if ([_orderInv[_index] intValue] + 1 < [_order count]) { // account for automatic move to next item _index = [_order[[_orderInv[_index] intValue] + 1] intValue]; @@ -802,7 +813,7 @@ } return; } else { - int expectedIndex = [self indexForItem:(IndexedPlayerItem *)_player.currentItem]; + int expectedIndex = [self indexForItem:playerItem]; if (_index != expectedIndex) { // AVQueuePlayer will sometimes skip over error items without // notifying this observer. @@ -816,9 +827,9 @@ _bufferUnconfirmed = YES; // If we've skipped or transitioned to a new item and we're not // currently in the middle of a seek - /* if (CMTIME_IS_INVALID(_seekPos) && _player.currentItem.status == AVPlayerItemStatusReadyToPlay) { */ + /* if (CMTIME_IS_INVALID(_seekPos) && playerItem.status == AVPlayerItemStatusReadyToPlay) { */ /* [self updatePosition]; */ - /* IndexedAudioSource *source = ((IndexedPlayerItem *)_player.currentItem).audioSource; */ + /* IndexedAudioSource *source = playerItem.audioSource; */ /* // We should already be at position zero but for */ /* // ClippingAudioSource it might be off by some milliseconds so we */ /* // consider anything <= 100 as close enough. */ @@ -850,6 +861,22 @@ /* // Already at zero, no need to seek. */ /* } */ /* } */ + + if (_justAdvanced) { + IndexedAudioSource *audioSource = playerItem.audioSource; + if (_loopMode == loopOne) { + [audioSource flip]; + [self enqueueFrom:_index]; + } else if (_loopMode == loopAll) { + if (_index == [_order[0] intValue] && playerItem == audioSource.playerItem2) { + [audioSource flip]; + [self enqueueFrom:_index]; + } else { + [self updateEndAction]; + } + } + _justAdvanced = NO; + } } else if ([keyPath isEqualToString:@"loadedTimeRanges"]) { IndexedPlayerItem *playerItem = (IndexedPlayerItem *)object; if (playerItem != _player.currentItem) return; @@ -889,7 +916,7 @@ - (int)indexForItem:(IndexedPlayerItem *)playerItem { for (int i = 0; i < _indexedAudioSources.count; i++) { - if (_indexedAudioSources[i].playerItem == playerItem) { + if (_indexedAudioSources[i].playerItem == playerItem || _indexedAudioSources[i].playerItem2 == playerItem) { return i; } } @@ -989,14 +1016,20 @@ } - (void)setLoopMode:(int)loopMode { + if (loopMode == _loopMode) return; _loopMode = loopMode; - [self updateEndAction]; + [self enqueueFrom:_index]; } - (void)updateEndAction { - // Should update this whenever the audio source changes and whenever _index changes. + // Should be called in the following situations: + // - when the audio source changes + // - when _index changes + // - when the loop mode changes. + // - when the shuffle order changes. (TODO) + // - when the shuffle mode changes. if (!_player) return; - if (_audioSource && [_orderInv[_index] intValue] + 1 < [_order count] && _loopMode != loopOne) { + if (_audioSource && (_loopMode != loopOff || [_orderInv[_index] intValue] + 1 < [_order count])) { _player.actionAtItemEnd = AVPlayerActionAtItemEndAdvance; } else { _player.actionAtItemEnd = AVPlayerActionAtItemEndPause; // AVPlayerActionAtItemEndNone @@ -1014,19 +1047,15 @@ } - (void)setShuffleOrder:(NSDictionary *)dict { + // TODO: update order and enqueue. [_audioSource decodeShuffleOrder:dict]; } - (void)dumpQueue { for (int i = 0; i < _player.items.count; i++) { IndexedPlayerItem *playerItem = (IndexedPlayerItem *)_player.items[i]; - for (int j = 0; j < _indexedAudioSources.count; j++) { - IndexedAudioSource *source = _indexedAudioSources[j]; - if (source.playerItem == playerItem) { - NSLog(@"- %d", j); - break; - } - } + int j = [self indexForItem:playerItem]; + NSLog(@"- %d", j); } } @@ -1189,6 +1218,9 @@ if (_indexedAudioSources) { for (int i = 0; i < [_indexedAudioSources count]; i++) { [self removeItemObservers:_indexedAudioSources[i].playerItem]; + if (_indexedAudioSources[i].playerItem2) { + [self removeItemObservers:_indexedAudioSources[i].playerItem2]; + } } _indexedAudioSources = nil; } diff --git a/just_audio/darwin/Classes/ClippingAudioSource.m b/just_audio/darwin/Classes/ClippingAudioSource.m index 29c1a43..14f29ca 100644 --- a/just_audio/darwin/Classes/ClippingAudioSource.m +++ b/just_audio/darwin/Classes/ClippingAudioSource.m @@ -30,8 +30,8 @@ - (void)attach:(AVQueuePlayer *)player { [super attach:player]; + // Prepare clip to start/end at the right timestamps. _audioSource.playerItem.forwardPlaybackEndTime = _end; - // XXX: Not needed since currentItem observer handles it? [self seek:kCMTimeZero]; } @@ -39,6 +39,10 @@ return _audioSource.playerItem; } +- (IndexedPlayerItem *)playerItem2 { + return _audioSource.playerItem2; +} + - (NSArray *)getShuffleIndices { return @[@(0)]; } @@ -61,6 +65,19 @@ } } +- (void)flip { + [_audioSource flip]; +} + +- (void)preparePlayerItem2 { + if (self.playerItem2) return; + [_audioSource preparePlayerItem2]; + IndexedPlayerItem *item = _audioSource.playerItem2; + // Prepare loop clip to start/end at the right timestamps. + item.forwardPlaybackEndTime = _end; + [item seekToTime:_start toleranceBefore:kCMTimeZero toleranceAfter:kCMTimeZero completionHandler:nil]; +} + - (CMTime)duration { return CMTimeSubtract(CMTIME_IS_INVALID(_end) ? self.playerItem.duration : _end, _start); } diff --git a/just_audio/darwin/Classes/IndexedAudioSource.m b/just_audio/darwin/Classes/IndexedAudioSource.m index cf54757..34c2606 100644 --- a/just_audio/darwin/Classes/IndexedAudioSource.m +++ b/just_audio/darwin/Classes/IndexedAudioSource.m @@ -9,7 +9,7 @@ } - (instancetype)initWithId:(NSString *)sid { - self = [super init]; + self = [super initWithId:sid]; NSAssert(self, @"super init cannot be nil"); _isAttached = NO; _queuedSeekPos = kCMTimeInvalid; @@ -33,6 +33,10 @@ return nil; } +- (IndexedPlayerItem *)playerItem2 { + return nil; +} + - (BOOL)isAttached { return _isAttached; } @@ -66,6 +70,12 @@ } } +- (void)flip { +} + +- (void)preparePlayerItem2 { +} + - (CMTime)duration { return kCMTimeInvalid; } diff --git a/just_audio/darwin/Classes/UriAudioSource.m b/just_audio/darwin/Classes/UriAudioSource.m index cb9500f..49d1715 100644 --- a/just_audio/darwin/Classes/UriAudioSource.m +++ b/just_audio/darwin/Classes/UriAudioSource.m @@ -6,6 +6,7 @@ @implementation UriAudioSource { NSString *_uri; IndexedPlayerItem *_playerItem; + IndexedPlayerItem *_playerItem2; /* CMTime _duration; */ } @@ -13,28 +14,33 @@ self = [super initWithId:sid]; NSAssert(self, @"super init cannot be nil"); _uri = uri; - if ([_uri hasPrefix:@"file://"]) { - _playerItem = [[IndexedPlayerItem alloc] initWithURL:[NSURL fileURLWithPath:[[_uri stringByReplacingPercentEscapesUsingEncoding:NSUTF8StringEncoding] substringFromIndex:7]]]; + _playerItem = [self createPlayerItem:uri]; + _playerItem2 = nil; + return self; +} + +- (IndexedPlayerItem *)createPlayerItem:(NSString *)uri { + IndexedPlayerItem *item; + if ([uri hasPrefix:@"file://"]) { + item = [[IndexedPlayerItem alloc] initWithURL:[NSURL fileURLWithPath:[[uri stringByReplacingPercentEscapesUsingEncoding:NSUTF8StringEncoding] substringFromIndex:7]]]; } else { - _playerItem = [[IndexedPlayerItem alloc] initWithURL:[NSURL URLWithString:_uri]]; + item = [[IndexedPlayerItem alloc] initWithURL:[NSURL URLWithString:uri]]; } if (@available(macOS 10.13, iOS 11.0, *)) { // This does the best at reducing distortion on voice with speeds below 1.0 - _playerItem.audioTimePitchAlgorithm = AVAudioTimePitchAlgorithmTimeDomain; + item.audioTimePitchAlgorithm = AVAudioTimePitchAlgorithmTimeDomain; } - /* NSKeyValueObservingOptions options = */ - /* NSKeyValueObservingOptionOld | NSKeyValueObservingOptionNew; */ - /* [_playerItem addObserver:self */ - /* forKeyPath:@"duration" */ - /* options:options */ - /* context:nil]; */ - return self; + return item; } - (IndexedPlayerItem *)playerItem { return _playerItem; } +- (IndexedPlayerItem *)playerItem2 { + return _playerItem2; +} + - (NSArray *)getShuffleIndices { return @[@(0)]; } @@ -61,6 +67,19 @@ } } +- (void)flip { + IndexedPlayerItem *temp = _playerItem; + _playerItem = _playerItem2; + _playerItem2 = temp; +} + +- (void)preparePlayerItem2 { + if (!_playerItem2) { + _playerItem2 = [self createPlayerItem:_uri]; + _playerItem2.audioSource = _playerItem.audioSource; + } +} + - (CMTime)duration { NSValue *seekableRange = _playerItem.seekableTimeRanges.lastObject; if (seekableRange) { diff --git a/just_audio/ios/Classes/IndexedAudioSource.h b/just_audio/ios/Classes/IndexedAudioSource.h index f2dd9af..0d70eb0 100644 --- a/just_audio/ios/Classes/IndexedAudioSource.h +++ b/just_audio/ios/Classes/IndexedAudioSource.h @@ -6,6 +6,7 @@ @interface IndexedAudioSource : AudioSource @property (readonly, nonatomic) IndexedPlayerItem *playerItem; +@property (readonly, nonatomic) IndexedPlayerItem *playerItem2; @property (readwrite, nonatomic) CMTime duration; @property (readonly, nonatomic) CMTime position; @property (readonly, nonatomic) CMTime bufferedPosition; @@ -18,5 +19,7 @@ - (void)stop:(AVQueuePlayer *)player; - (void)seek:(CMTime)position; - (void)seek:(CMTime)position completionHandler:(void (^)(BOOL))completionHandler; +- (void)preparePlayerItem2; +- (void)flip; @end diff --git a/just_audio/macos/Classes/IndexedAudioSource.h b/just_audio/macos/Classes/IndexedAudioSource.h index 8c39ad0..506b998 100644 --- a/just_audio/macos/Classes/IndexedAudioSource.h +++ b/just_audio/macos/Classes/IndexedAudioSource.h @@ -6,6 +6,7 @@ @interface IndexedAudioSource : AudioSource @property (readonly, nonatomic) IndexedPlayerItem *playerItem; +@property (readonly, nonatomic) IndexedPlayerItem *playerItem2; @property (readwrite, nonatomic) CMTime duration; @property (readonly, nonatomic) CMTime position; @property (readonly, nonatomic) CMTime bufferedPosition; @@ -18,5 +19,7 @@ - (void)stop:(AVQueuePlayer *)player; - (void)seek:(CMTime)position; - (void)seek:(CMTime)position completionHandler:(void (^)(BOOL))completionHandler; +- (void)preparePlayerItem2; +- (void)flip; @end