Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
## 2.1.0 (#unreleased)

- Feature [#468](https://github.com/SimformSolutionsPvtLtd/audio_waveforms/pull/468) - Add macOS support
- Fixed [#477](https://github.com/SimformSolutionsPvtLtd/audio_waveforms/issues/477) - Serialize extractor teardown with a lock and clear codec references to prevent stopping an already-released codec on Android
- Partially fixed [#478](https://github.com/SimformSolutionsPvtLtd/audio_waveforms/issues/478) - Guard decode callbacks against released resources to prevent crash while extracting waveform from ID3v2.4 files on Android
- Fixed [#488](https://github.com/SimformSolutionsPvtLtd/audio_waveforms/issues/488) - Auto-retry playback on transient network loss and run extraction setup off the main thread to prevent crashes and ANR on Android

## 2.0.2

Expand Down
72 changes: 70 additions & 2 deletions android/src/main/kotlin/com/simform/audio_waveforms/AudioPlayer.kt
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,22 @@ class AudioPlayer(
private var player: ExoPlayer? = null
private var playerListener: Player.Listener? = null
private var isPlayerPrepared: Boolean = false
// Guards the prepare result so it is delivered exactly once (success or error).
private var hasReplied: Boolean = false
private var finishMode = FinishMode.Stop
private var key = playerKey
private var updateFrequency: Long = 200

// Auto-resume across transient network drops.
// Re-prepare attempts used since the last successful prepare; reset on STATE_READY.
private var networkRetryCount = 0
// Max re-prepare attempts before giving up and tearing down.
private val maxNetworkRetries = 5
// Delay between re-prepare attempts.
private val networkRetryDelayMs = 3000L
// Pending delayed re-prepare; cancelled in stop().
private var retryRunnable: Runnable? = null

fun preparePlayer(
result: MethodChannel.Result,
path: String?,
Expand All @@ -38,6 +50,8 @@ class AudioPlayer(
}
val uri = Uri.parse(path)
val mediaItem = MediaItem.fromUri(uri)
hasReplied = false
networkRetryCount = 0
stop()
player?.clearMediaItems()
player = ExoPlayer.Builder(appContext).build()
Expand All @@ -47,15 +61,49 @@ class AudioPlayer(

override fun onPlayerError(error: PlaybackException) {
super.onPlayerError(error)
result.error(Constants.LOG_TAG, error.message, "Unable to load media source.")
if (!isPlayerPrepared) {
if (!hasReplied) {
hasReplied = true
result.error(Constants.LOG_TAG, error.message, "Unable to load media source.")
}
} else if (isRecoverableNetworkError(error) && networkRetryCount < maxNetworkRetries) {
// Transient network drop: keep the player and re-prepare. ExoPlayer retains the
// playback position, so it resumes once connectivity returns. Bounded retries +
// delay let permanent loss fall through to the teardown branch below.
networkRetryCount++
retryRunnable?.let { handler.removeCallbacks(it) }
// ExoPlayer retains playWhenReady across a re-prepare, so the user's current
// play/pause intent is honored even if they paused during the retry window.
// Don't snapshot it at error time — that would override a pause issued mid-wait.
retryRunnable = Runnable {
player?.prepare()
}
handler.postDelayed(retryRunnable!!, networkRetryDelayMs)
} else {
val args: MutableMap<String, Any?> = HashMap()
args[Constants.playerKey] = key
args[Constants.finishType] = 2
// Route through stop() so the listener and any pending retry are cleaned up too.
stop()
player?.release()
player = null
methodChannel.invokeMethod(Constants.onDidFinishPlayingAudio, args)
}
}

override fun onPlayerStateChanged(isReady: Boolean, state: Int) {
// Note: networkRetryCount is intentionally NOT reset here. Resetting on every
// STATE_READY lets a flapping connection (ready->drop->ready->drop) refill the
// budget endlessly, so maxNetworkRetries never bounds total retries. It is reset
// once per preparePlayer() call, making the cap a real per-prepare ceiling.
if (!isPlayerPrepared) {
if (state == Player.STATE_READY) {
player?.volume = volume ?: 1F
isPlayerPrepared = true
result.success(true)
if (!hasReplied) {
hasReplied = true
result.success(true)
}
}
}
if (state == Player.STATE_ENDED) {
Expand Down Expand Up @@ -132,6 +180,8 @@ class AudioPlayer(
}

fun stop() {
retryRunnable?.let { handler.removeCallbacks(it) }
retryRunnable = null
stopListening()
if (playerListener != null) {
player?.removeListener(playerListener!!)
Expand All @@ -140,6 +190,19 @@ class AudioPlayer(
player?.stop()
}

/**
* Network/HTTP IO errors that are worth retrying. ExoPlayer keeps the playback position across
* a re-prepare, so these recover seamlessly once connectivity is restored.
*/
private fun isRecoverableNetworkError(error: PlaybackException): Boolean {
return when (error.errorCode) {
PlaybackException.ERROR_CODE_IO_UNSPECIFIED,
PlaybackException.ERROR_CODE_IO_NETWORK_CONNECTION_FAILED,
PlaybackException.ERROR_CODE_IO_NETWORK_CONNECTION_TIMEOUT -> true
else -> false
}
}


fun pause() {
stopListening()
Expand All @@ -148,7 +211,12 @@ class AudioPlayer(

fun release(result: MethodChannel.Result) {
try {
// Route through stop() first so the listener and any pending re-prepare retry are
// cancelled; otherwise a delayed retryRunnable could call prepare() on a released
// player and crash with IllegalStateException.
stop()
player?.release()
player = null
result.success(true)
} catch (e: Exception) {
result.error(Constants.LOG_TAG, "Failed to release player resource", e.toString())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,9 @@ class AudioWaveformsPlugin : FlutterPlugin, MethodCallHandler, ActivityAware {
extractorCallBack = object : ExtractorCallBack {
override fun onProgress(value: Float) {
if (value == 1.0F) {
result.success(extractors[playerKey]?.sampleData)
// Route through the extractor's guarded, main-thread reply so success and
// any error path can never both reply to the same Result.
extractors[playerKey]?.submitWaveformData()
}
}

Expand Down
137 changes: 100 additions & 37 deletions android/src/main/kotlin/com/simform/audio_waveforms/WaveformExtractor.kt
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import android.media.MediaCodec
import android.media.MediaExtractor
import android.media.MediaFormat
import android.os.Build
import android.os.Handler
import android.os.Looper
import android.util.Log
import io.flutter.plugin.common.MethodChannel
import java.nio.ByteBuffer
Expand Down Expand Up @@ -66,8 +68,44 @@ class WaveformExtractor(
private var totalSamples = 0L
/** Number of audio samples per waveform data point */
private var perSamplePoints = 0L
/** Flag to prevent submitting multiple results */
/** Flag to prevent submitting multiple results. Guarded by [lock]; read/written from the decode and codec callback threads. */
@Volatile
private var isReplySubmitted = false
/** Background thread running the blocking decode setup (setDataSource can block on network sources) */
private var decodeThread: Thread? = null
/** Delivers MethodChannel replies/events on the platform (main) thread, since decode runs off it. */
private val mainHandler = Handler(Looper.getMainLooper())
/** Guards decoder/extractor teardown against the live MediaCodec callbacks */
private val lock = Any()
/** Set once stop() has released the codec/extractor so in-flight callbacks bail out */
@Volatile
private var released = false

/**
* Delivers the final waveform data to Flutter exactly once, on the main thread.
* Idempotent: the first reply (success or error) wins and every later call is a no-op.
*/
fun submitWaveformData() {
synchronized(lock) {
if (isReplySubmitted) return
isReplySubmitted = true
}
// Snapshot: the reply is posted async and sampleData may still be mutated by the decode thread.
val data = ArrayList(sampleData)
mainHandler.post { result.success(data) }
}

/**
* Delivers an error to Flutter exactly once, on the main thread.
* Idempotent: a no-op if a result (success or error) was already submitted.
*/
private fun submitError(message: String?, details: String) {
synchronized(lock) {
if (isReplySubmitted) return
isReplySubmitted = true
}
mainHandler.post { result.error(Constants.LOG_TAG, message, details) }
}

/**
* Retrieves the audio format from the given media file
Expand Down Expand Up @@ -114,15 +152,24 @@ class WaveformExtractor(
* 5. Reporting progress via the callback interface and method channel
*/
fun startDecode() {
// setDataSource()/getFormat() block synchronously and, for network sources with poor or no
// connectivity, can stall for tens of seconds (native NuCachedSource2 retries). Running on the
// platform main thread would freeze the UI and trigger an ANR, so do the setup off the main thread.
decodeThread = Thread {
decodeInternal()
}.also { it.start() }
}

private fun decodeInternal() {
try {
val format = getFormat(path) ?: error("No audio format found")
val mime = format.getString(MediaFormat.KEY_MIME) ?: error("No MIME type found")
decoder = MediaCodec.createDecoderByType(mime).also {
it.configure(format, null, null, 0)
it.setCallback(object : MediaCodec.Callback() {
override fun onInputBufferAvailable(codec: MediaCodec, index: Int) {
if (inputEof || index < 0) return
val extractor = extractor ?: return
override fun onInputBufferAvailable(codec: MediaCodec, index: Int): Unit = synchronized(lock) {
if (released || inputEof || index < 0) return@synchronized
val extractor = extractor ?: return@synchronized
codec.getInputBuffer(index)?.let { buf ->
val size = extractor.readSampleData(buf, 0)
val sampleTime = extractor.sampleTime
Expand All @@ -132,11 +179,7 @@ class WaveformExtractor(
extractor.advance()
} catch (e: Exception) {
inputEof = true
result.error(
Constants.LOG_TAG,
e.message,
"Invalid input buffer."
)
submitError(e.message, "Invalid input buffer.")
}
} else {
codec.queueInputBuffer(
Expand Down Expand Up @@ -173,24 +216,17 @@ class WaveformExtractor(
}

override fun onError(codec: MediaCodec, e: MediaCodec.CodecException) {
if (!isReplySubmitted) {
result.error(
Constants.LOG_TAG,
e.message,
"An error is thrown while decoding the audio file"
)
isReplySubmitted = true
finishCount.countDown()
}
submitError(e.message, "An error is thrown while decoding the audio file")
finishCount.countDown()
}

override fun onOutputBufferAvailable(
codec: MediaCodec,
index: Int,
info: MediaCodec.BufferInfo
) {
if (index < 0 || decoder == null) return
): Unit = synchronized(lock) {
if (released || index < 0 || decoder == null) return@synchronized

try {
if (info.size > 0) {
codec.getOutputBuffer(index)?.let { buf ->
Expand Down Expand Up @@ -232,6 +268,7 @@ class WaveformExtractor(
updateProgress()
val rms = sqrt(sampleSum / perSamplePoints).toFloat()
sendProgress(rms)
submitWaveformData()
stop()
}
}
Expand All @@ -241,14 +278,7 @@ class WaveformExtractor(
}

} catch (e: Exception) {
if (!isReplySubmitted) {
result.error(
Constants.LOG_TAG,
e.message,
"An error is thrown before decoding the audio file"
)
isReplySubmitted = true
}
submitError(e.message, "An error is thrown before decoding the audio file")
}


Expand Down Expand Up @@ -277,6 +307,7 @@ class WaveformExtractor(

// Discard redundant values and release resources
if (progress > 1.0F) {
submitWaveformData()
stop()
return
}
Expand Down Expand Up @@ -384,13 +415,16 @@ class WaveformExtractor(
sampleSum = 0.0

val args: MutableMap<String, Any?> = HashMap()
args[Constants.waveformData] = sampleData
args[Constants.waveformData] = ArrayList(sampleData)
args[Constants.progress] = progress
args[Constants.playerKey] = key
methodChannel.invokeMethod(
Constants.onCurrentExtractedWaveformData,
args
)
// Decode runs off the platform thread; MethodChannel must be invoked on the main thread.
mainHandler.post {
methodChannel.invokeMethod(
Constants.onCurrentExtractedWaveformData,
args
)
}
}

/**
Expand All @@ -402,9 +436,38 @@ class WaveformExtractor(
* 3. Signals completion via the countdown latch
*/
fun stop() {
decoder?.stop()
decoder?.release()
extractor?.release()
decodeThread?.interrupt()
decodeThread = null
var decoderToRelease: MediaCodec? = null
var extractorToRelease: MediaExtractor? = null
// Claim ownership of the codec/extractor under the lock and flip `released` so any
// callback that wins the lock afterwards bails out instead of touching freed objects.
synchronized(lock) {
if (released) return
released = true
decoderToRelease = decoder
extractorToRelease = extractor
decoder = null
extractor = null
}
// Release OUTSIDE the lock: MediaCodec.stop()/release() can block draining in-flight
// callbacks, and those callbacks contend for the same lock — releasing while holding it
// risks a deadlock. Wrap in try/catch so an already-released codec can't crash teardown.
try {
decoderToRelease?.stop()
} catch (e: Exception) {
Log.e(Constants.LOG_TAG, "Error stopping decoder: ${e.message}")
}
try {
decoderToRelease?.release()
} catch (e: Exception) {
Log.e(Constants.LOG_TAG, "Error releasing decoder: ${e.message}")
}
try {
extractorToRelease?.release()
} catch (e: Exception) {
Log.e(Constants.LOG_TAG, "Error releasing extractor: ${e.message}")
}
finishCount.countDown()
}
}
Expand Down
4 changes: 4 additions & 0 deletions lib/src/controllers/player_controller.dart
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,10 @@ class PlayerController extends ChangeNotifier {
..addAll(value);
notifyListeners();
},
).catchError(
(Object error) {
debugPrint('Waveform extraction failed: $error');
},
);
}
notifyListeners();
Expand Down