/* * Copyright (C) 2019 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package sushi.hardcore.droidfs.video_recording; import static androidx.camera.core.impl.ImageOutputConfig.OPTION_DEFAULT_RESOLUTION; import static androidx.camera.core.impl.ImageOutputConfig.OPTION_MAX_RESOLUTION; import static androidx.camera.core.impl.ImageOutputConfig.OPTION_SUPPORTED_RESOLUTIONS; import static androidx.camera.core.impl.ImageOutputConfig.OPTION_TARGET_ASPECT_RATIO; import static androidx.camera.core.impl.ImageOutputConfig.OPTION_TARGET_RESOLUTION; import static androidx.camera.core.impl.ImageOutputConfig.OPTION_TARGET_ROTATION; import static androidx.camera.core.impl.UseCaseConfig.OPTION_CAMERA_SELECTOR; import static androidx.camera.core.impl.UseCaseConfig.OPTION_CAPTURE_CONFIG_UNPACKER; import static androidx.camera.core.impl.UseCaseConfig.OPTION_DEFAULT_CAPTURE_CONFIG; import static androidx.camera.core.impl.UseCaseConfig.OPTION_DEFAULT_SESSION_CONFIG; import static androidx.camera.core.impl.UseCaseConfig.OPTION_SESSION_CONFIG_UNPACKER; import static androidx.camera.core.impl.UseCaseConfig.OPTION_SURFACE_OCCUPANCY_PRIORITY; import static androidx.camera.core.impl.VideoCaptureConfig.OPTION_AUDIO_BIT_RATE; import static androidx.camera.core.impl.VideoCaptureConfig.OPTION_AUDIO_CHANNEL_COUNT; import static androidx.camera.core.impl.VideoCaptureConfig.OPTION_AUDIO_MIN_BUFFER_SIZE; import static androidx.camera.core.impl.VideoCaptureConfig.OPTION_AUDIO_SAMPLE_RATE; import static androidx.camera.core.impl.VideoCaptureConfig.OPTION_BIT_RATE; import static androidx.camera.core.impl.VideoCaptureConfig.OPTION_INTRA_FRAME_INTERVAL; import static androidx.camera.core.impl.VideoCaptureConfig.OPTION_VIDEO_FRAME_RATE; import static androidx.camera.core.internal.TargetConfig.OPTION_TARGET_CLASS; import static androidx.camera.core.internal.TargetConfig.OPTION_TARGET_NAME; import static androidx.camera.core.internal.ThreadConfig.OPTION_BACKGROUND_EXECUTOR; import static androidx.camera.core.internal.UseCaseEventConfig.OPTION_USE_CASE_EVENT_CALLBACK; import android.Manifest; import android.media.AudioFormat; import android.media.AudioRecord; import android.media.CamcorderProfile; import android.media.MediaCodec; import android.media.MediaCodec.BufferInfo; import android.media.MediaCodecInfo; import android.media.MediaCodecInfo.CodecCapabilities; import android.media.MediaFormat; import android.media.MediaRecorder.AudioSource; import android.os.Bundle; import android.os.Handler; import android.os.HandlerThread; import android.os.Looper; import android.os.ParcelFileDescriptor; import android.provider.MediaStore; import android.util.Pair; import android.util.Size; import android.view.Display; import android.view.Surface; import androidx.annotation.DoNotInline; import androidx.annotation.GuardedBy; import androidx.annotation.IntDef; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import androidx.annotation.RequiresPermission; import androidx.annotation.RestrictTo; import androidx.annotation.RestrictTo.Scope; import androidx.annotation.UiThread; import androidx.annotation.VisibleForTesting; import androidx.camera.core.AspectRatio; import androidx.camera.core.CameraSelector; import androidx.camera.core.CameraXThreads; import androidx.camera.core.ImageCapture; import androidx.camera.core.Logger; import androidx.camera.core.UseCase; import androidx.camera.core.impl.CameraInternal; import androidx.camera.core.impl.CaptureConfig; import androidx.camera.core.impl.Config; import androidx.camera.core.impl.ConfigProvider; import androidx.camera.core.impl.DeferrableSurface; import androidx.camera.core.impl.ImageOutputConfig; import androidx.camera.core.impl.ImageOutputConfig.RotationValue; import androidx.camera.core.impl.ImmediateSurface; import androidx.camera.core.impl.MutableConfig; import androidx.camera.core.impl.MutableOptionsBundle; import androidx.camera.core.impl.OptionsBundle; import androidx.camera.core.impl.SessionConfig; import androidx.camera.core.impl.UseCaseConfig; import androidx.camera.core.impl.UseCaseConfigFactory; import androidx.camera.core.impl.utils.executor.CameraXExecutors; import androidx.camera.core.internal.ThreadConfig; import androidx.concurrent.futures.CallbackToFutureAdapter; import androidx.concurrent.futures.CallbackToFutureAdapter.Completer; import androidx.core.util.Preconditions; import com.google.common.util.concurrent.ListenableFuture; import java.io.File; import java.io.IOException; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.nio.ByteBuffer; import java.util.List; import java.util.UUID; import java.util.concurrent.Executor; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; /** * A use case for taking a video. * *
This class is designed for simple video capturing. It gives basic configuration of the * recorded video such as resolution and file format. * * @hide */ @RequiresApi(21) // TODO(b/200306659): Remove and replace with annotation on package-info.java public final class VideoCapture extends UseCase { //////////////////////////////////////////////////////////////////////////////////////////// // [UseCase lifetime constant] - Stays constant for the lifetime of the UseCase. Which means // they could be created in the constructor. //////////////////////////////////////////////////////////////////////////////////////////// /** * An unknown error occurred. * *
See message parameter in onError callback or log for more details. */ public static final int ERROR_UNKNOWN = 0; /** * An error occurred with encoder state, either when trying to change state or when an * unexpected state change occurred. */ public static final int ERROR_ENCODER = 1; /** An error with muxer state such as during creation or when stopping. */ public static final int ERROR_MUXER = 2; /** * An error indicating start recording was called when video recording is still in progress. */ public static final int ERROR_RECORDING_IN_PROGRESS = 3; /** * An error indicating the file saving operations. */ public static final int ERROR_FILE_IO = 4; /** * An error indicating this VideoCapture is not bound to a camera. */ public static final int ERROR_INVALID_CAMERA = 5; /** * An error indicating the video file is too short. *
The output file will be deleted if the OutputFileOptions is backed by File or uri.
*/
public static final int ERROR_RECORDING_TOO_SHORT = 6;
/**
* Provides a static configuration with implementation-agnostic options.
*
* @hide
*/
@RestrictTo(Scope.LIBRARY_GROUP)
public static final Defaults DEFAULT_CONFIG = new Defaults();
private static final String TAG = "VideoCapture";
/** Amount of time to wait for dequeuing a buffer from the videoEncoder. */
private static final int DEQUE_TIMEOUT_USEC = 10000;
/** Android preferred mime type for AVC video. */
private static final String VIDEO_MIME_TYPE = "video/avc";
private static final String AUDIO_MIME_TYPE = "audio/mp4a-latm";
/** Camcorder profiles quality list */
private static final int[] CamcorderQuality = {
CamcorderProfile.QUALITY_2160P,
CamcorderProfile.QUALITY_1080P,
CamcorderProfile.QUALITY_720P,
CamcorderProfile.QUALITY_480P
};
private final BufferInfo mVideoBufferInfo = new BufferInfo();
private final Object mMuxerLock = new Object();
private final AtomicBoolean mEndOfVideoStreamSignal = new AtomicBoolean(true);
private final AtomicBoolean mEndOfAudioStreamSignal = new AtomicBoolean(true);
private final AtomicBoolean mEndOfAudioVideoSignal = new AtomicBoolean(true);
private final BufferInfo mAudioBufferInfo = new BufferInfo();
/** For record the first sample written time. */
@VisibleForTesting(otherwise = VisibleForTesting.PRIVATE)
public final AtomicBoolean mIsFirstVideoKeyFrameWrite = new AtomicBoolean(false);
@VisibleForTesting(otherwise = VisibleForTesting.PRIVATE)
public final AtomicBoolean mIsFirstAudioSampleWrite = new AtomicBoolean(false);
////////////////////////////////////////////////////////////////////////////////////////////
// [UseCase attached constant] - Is only valid when the UseCase is attached to a camera.
////////////////////////////////////////////////////////////////////////////////////////////
/** Thread on which all encoding occurs. */
private HandlerThread mVideoHandlerThread;
private Handler mVideoHandler;
/** Thread on which audio encoding occurs. */
private HandlerThread mAudioHandlerThread;
private Handler mAudioHandler;
@NonNull
MediaCodec mVideoEncoder;
@NonNull
private MediaCodec mAudioEncoder;
@Nullable
private ListenableFuture StartRecording() is asynchronous. User needs to check if any error occurs by setting the
* {@link OnVideoSavedCallback#onError(int, String, Throwable)}.
*
* @param outputFileOptions Location to save the video capture
* @param executor The executor in which the callback methods will be run.
* @param callback Callback for when the recorded video saving completion or failure.
*/
@SuppressWarnings("ObjectToString")
@RequiresPermission(Manifest.permission.RECORD_AUDIO)
public void startRecording(
@NonNull OutputFileOptions outputFileOptions, @NonNull Executor executor,
@NonNull OnVideoSavedCallback callback) {
if (Looper.getMainLooper() != Looper.myLooper()) {
CameraXExecutors.mainThreadExecutor().execute(() -> startRecording(outputFileOptions,
executor, callback));
return;
}
Logger.i(TAG, "startRecording");
mIsFirstVideoKeyFrameWrite.set(false);
mIsFirstAudioSampleWrite.set(false);
OnVideoSavedCallback postListener = new VideoSavedListenerWrapper(executor, callback);
CameraInternal attachedCamera = getCamera();
if (attachedCamera == null) {
// Not bound. Notify callback.
postListener.onError(ERROR_INVALID_CAMERA,
"Not bound to a Camera [" + VideoCapture.this + "]", null);
return;
}
// Check video encoder initialization status, if there is any error happened
// return error callback directly.
if (mVideoEncoderInitStatus
== VideoEncoderInitStatus.VIDEO_ENCODER_INIT_STATUS_INSUFFICIENT_RESOURCE
|| mVideoEncoderInitStatus
== VideoEncoderInitStatus.VIDEO_ENCODER_INIT_STATUS_INITIALIZED_FAILED
|| mVideoEncoderInitStatus
== VideoEncoderInitStatus.VIDEO_ENCODER_INIT_STATUS_RESOURCE_RECLAIMED) {
postListener.onError(ERROR_ENCODER, "Video encoder initialization failed before start"
+ " recording ", mVideoEncoderErrorMessage);
return;
}
if (!mEndOfAudioVideoSignal.get()) {
postListener.onError(
ERROR_RECORDING_IN_PROGRESS, "It is still in video recording!",
null);
return;
}
if (mIsAudioEnabled.get()) {
try {
// Audio input start
if (mAudioRecorder.getState() == AudioRecord.STATE_INITIALIZED) {
mAudioRecorder.startRecording();
}
} catch (IllegalStateException e) {
// Disable the audio if the audio input cannot start. And Continue the recording
// without audio.
Logger.i(TAG,
"AudioRecorder cannot start recording, disable audio." + e.getMessage());
mIsAudioEnabled.set(false);
releaseAudioInputResource();
}
// Gets the AudioRecorder's state
if (mAudioRecorder.getRecordingState() != AudioRecord.RECORDSTATE_RECORDING) {
Logger.i(TAG,
"AudioRecorder startRecording failed - incorrect state: "
+ mAudioRecorder.getRecordingState());
mIsAudioEnabled.set(false);
releaseAudioInputResource();
}
}
AtomicReference stopRecording() is asynchronous API. User need to check if {@link
* OnVideoSavedCallback#onVideoSaved()} or
* {@link OnVideoSavedCallback#onError(int, String, Throwable)} be called
* before startRecording.
*/
public void stopRecording() {
if (Looper.getMainLooper() != Looper.myLooper()) {
CameraXExecutors.mainThreadExecutor().execute(() -> stopRecording());
return;
}
Logger.i(TAG, "stopRecording");
mSessionConfigBuilder.clearSurfaces();
mSessionConfigBuilder.addNonRepeatingSurface(mDeferrableSurface);
updateSessionConfig(mSessionConfigBuilder.build());
notifyUpdated();
if (mIsRecording) {
if (mIsAudioEnabled.get()) {
// Stop audio encoder thread, and wait video encoder and muxer stop.
mEndOfAudioStreamSignal.set(true);
} else {
// Audio is disabled, stop video encoder thread directly.
mEndOfVideoStreamSignal.set(true);
}
}
}
/**
* {@inheritDoc}
*
* @hide
*/
@RestrictTo(Scope.LIBRARY_GROUP)
@Override
public void onDetached() {
stopRecording();
if (mRecordingFuture != null) {
mRecordingFuture.addListener(() -> releaseResources(),
CameraXExecutors.mainThreadExecutor());
} else {
releaseResources();
}
}
private void releaseResources() {
mVideoHandlerThread.quitSafely();
// audio encoder release
releaseAudioInputResource();
if (mCameraSurface != null) {
releaseCameraSurface(true);
}
}
private void releaseAudioInputResource() {
mAudioHandlerThread.quitSafely();
if (mAudioEncoder != null) {
mAudioEncoder.release();
mAudioEncoder = null;
}
if (mAudioRecorder != null) {
mAudioRecorder.release();
mAudioRecorder = null;
}
}
/**
* {@inheritDoc}
*
* @hide
*/
@NonNull
@RestrictTo(Scope.LIBRARY_GROUP)
@Override
public UseCaseConfig.Builder, ?, ?> getUseCaseConfigBuilder(@NonNull Config config) {
return Builder.fromConfig(config);
}
/**
* {@inheritDoc}
*
* @hide
*/
@RestrictTo(Scope.LIBRARY_GROUP)
@UiThread
@Override
public void onStateDetached() {
stopRecording();
}
@UiThread
private void releaseCameraSurface(final boolean releaseVideoEncoder) {
if (mDeferrableSurface == null) {
return;
}
final MediaCodec videoEncoder = mVideoEncoder;
// Calling close should allow termination future to complete and close the surface with
// the listener that was added after constructing the DeferrableSurface.
mDeferrableSurface.close();
mDeferrableSurface.getTerminationFuture().addListener(
() -> {
if (releaseVideoEncoder && videoEncoder != null) {
videoEncoder.release();
}
}, CameraXExecutors.mainThreadExecutor());
if (releaseVideoEncoder) {
mVideoEncoder = null;
}
mCameraSurface = null;
mDeferrableSurface = null;
}
/**
* Sets the desired rotation of the output video.
*
* In most cases this should be set to the current rotation returned by {@link
* Display#getRotation()}.
*
* @param rotation Desired rotation of the output video.
*/
public void setTargetRotation(@RotationValue int rotation) {
setTargetRotationInternal(rotation);
}
/**
* Setup the {@link MediaCodec} for encoding video from a camera {@link Surface} and encoding
* audio from selected audio source.
*/
@UiThread
@SuppressWarnings("WeakerAccess") /* synthetic accessor */
@RequiresPermission(Manifest.permission.RECORD_AUDIO)
void setupEncoder(@NonNull String cameraId, @NonNull Size resolution) {
VideoCaptureConfig config = (VideoCaptureConfig) getCurrentConfig();
// video encoder setup
mVideoEncoder.reset();
mVideoEncoderInitStatus = VideoEncoderInitStatus.VIDEO_ENCODER_INIT_STATUS_UNINITIALIZED;
// Configures a Video encoder, if there is any exception, will abort follow up actions
try {
mVideoEncoder.configure(
createVideoMediaFormat(config, resolution), /*surface*/
null, /*crypto*/
null,
MediaCodec.CONFIGURE_FLAG_ENCODE);
} catch (MediaCodec.CodecException e) {
int errorCode = 0;
if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.M) {
errorCode = Api23Impl.getCodecExceptionErrorCode(e);
String diagnosticInfo = e.getDiagnosticInfo();
if (errorCode == MediaCodec.CodecException.ERROR_INSUFFICIENT_RESOURCE) {
Logger.i(TAG,
"CodecException: code: " + errorCode + " diagnostic: "
+ diagnosticInfo);
mVideoEncoderInitStatus =
VideoEncoderInitStatus.VIDEO_ENCODER_INIT_STATUS_INSUFFICIENT_RESOURCE;
} else if (errorCode == MediaCodec.CodecException.ERROR_RECLAIMED) {
Logger.i(TAG,
"CodecException: code: " + errorCode + " diagnostic: "
+ diagnosticInfo);
mVideoEncoderInitStatus =
VideoEncoderInitStatus.VIDEO_ENCODER_INIT_STATUS_RESOURCE_RECLAIMED;
}
} else {
mVideoEncoderInitStatus =
VideoEncoderInitStatus.VIDEO_ENCODER_INIT_STATUS_INITIALIZED_FAILED;
}
mVideoEncoderErrorMessage = e;
return;
} catch (IllegalArgumentException | IllegalStateException e) {
mVideoEncoderInitStatus =
VideoEncoderInitStatus.VIDEO_ENCODER_INIT_STATUS_INITIALIZED_FAILED;
mVideoEncoderErrorMessage = e;
return;
}
if (mCameraSurface != null) {
releaseCameraSurface(false);
}
Surface cameraSurface = mVideoEncoder.createInputSurface();
mCameraSurface = cameraSurface;
mSessionConfigBuilder = SessionConfig.Builder.createFrom(config);
if (mDeferrableSurface != null) {
mDeferrableSurface.close();
}
mDeferrableSurface = new ImmediateSurface(mCameraSurface, resolution, getImageFormat());
mDeferrableSurface.getTerminationFuture().addListener(
cameraSurface::release, CameraXExecutors.mainThreadExecutor()
);
mSessionConfigBuilder.addNonRepeatingSurface(mDeferrableSurface);
mSessionConfigBuilder.addErrorListener(new SessionConfig.ErrorListener() {
@Override
@RequiresPermission(Manifest.permission.RECORD_AUDIO)
public void onError(@NonNull SessionConfig sessionConfig,
@NonNull SessionConfig.SessionError error) {
// Ensure the attached camera has not changed before calling setupEncoder.
// TODO(b/143915543): Ensure this never gets called by a camera that is not attached
// to this use case so we don't need to do this check.
if (isCurrentCamera(cameraId)) {
// Only reset the pipeline when the bound camera is the same.
setupEncoder(cameraId, resolution);
notifyReset();
}
}
});
updateSessionConfig(mSessionConfigBuilder.build());
// audio encoder setup
// reset audio inout flag
mIsAudioEnabled.set(true);
setAudioParametersByCamcorderProfile(resolution, cameraId);
mAudioEncoder.reset();
mAudioEncoder.configure(
createAudioMediaFormat(), null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
if (mAudioRecorder != null) {
mAudioRecorder.release();
}
mAudioRecorder = autoConfigAudioRecordSource(config);
// check mAudioRecorder
if (mAudioRecorder == null) {
Logger.e(TAG, "AudioRecord object cannot initialized correctly!");
mIsAudioEnabled.set(false);
}
synchronized (mMuxerLock) {
mVideoTrackIndex = -1;
mAudioTrackIndex = -1;
}
mIsRecording = false;
}
/**
* Write a buffer that has been encoded to file.
*
* @param bufferIndex the index of the buffer in the videoEncoder that has available data
* @return returns true if this buffer is the end of the stream
*/
private boolean writeVideoEncodedBuffer(int bufferIndex) {
if (bufferIndex < 0) {
Logger.e(TAG, "Output buffer should not have negative index: " + bufferIndex);
return false;
}
// Get data from buffer
ByteBuffer outputBuffer = mVideoEncoder.getOutputBuffer(bufferIndex);
// Check if buffer is valid, if not then return
if (outputBuffer == null) {
Logger.d(TAG, "OutputBuffer was null.");
return false;
}
// Write data to mMuxer if available
if (mMuxerStarted.get()) {
if (mVideoBufferInfo.size > 0) {
outputBuffer.position(mVideoBufferInfo.offset);
outputBuffer.limit(mVideoBufferInfo.offset + mVideoBufferInfo.size);
mVideoBufferInfo.presentationTimeUs = (System.nanoTime() / 1000);
synchronized (mMuxerLock) {
if (!mIsFirstVideoKeyFrameWrite.get()) {
boolean isKeyFrame =
(mVideoBufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0;
if (isKeyFrame) {
Logger.i(TAG,
"First video key frame written.");
mIsFirstVideoKeyFrameWrite.set(true);
} else {
// Request a sync frame immediately
final Bundle syncFrame = new Bundle();
syncFrame.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0);
mVideoEncoder.setParameters(syncFrame);
}
}
mMuxer.writeSampleData(mVideoTrackIndex, outputBuffer, mVideoBufferInfo);
}
} else {
Logger.i(TAG, "mVideoBufferInfo.size <= 0, index " + bufferIndex);
}
}
// Release data
mVideoEncoder.releaseOutputBuffer(bufferIndex, false);
// Return true if EOS is set
return (mVideoBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
}
private boolean writeAudioEncodedBuffer(int bufferIndex) {
ByteBuffer buffer = getOutputBuffer(mAudioEncoder, bufferIndex);
buffer.position(mAudioBufferInfo.offset);
if (mMuxerStarted.get()) {
try {
if (mAudioBufferInfo.size > 0 && mAudioBufferInfo.presentationTimeUs > 0) {
synchronized (mMuxerLock) {
if (!mIsFirstAudioSampleWrite.get()) {
Logger.i(TAG, "First audio sample written.");
mIsFirstAudioSampleWrite.set(true);
}
mMuxer.writeSampleData(mAudioTrackIndex, buffer, mAudioBufferInfo);
}
} else {
Logger.i(TAG, "mAudioBufferInfo size: " + mAudioBufferInfo.size + " "
+ "presentationTimeUs: " + mAudioBufferInfo.presentationTimeUs);
}
} catch (Exception e) {
Logger.e(
TAG,
"audio error:size="
+ mAudioBufferInfo.size
+ "/offset="
+ mAudioBufferInfo.offset
+ "/timeUs="
+ mAudioBufferInfo.presentationTimeUs);
e.printStackTrace();
}
}
mAudioEncoder.releaseOutputBuffer(bufferIndex, false);
return (mAudioBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
}
/**
* Encoding which runs indefinitely until end of stream is signaled. This should not run on the
* main thread otherwise it will cause the application to block.
*
* @return returns {@code true} if an error condition occurred, otherwise returns {@code false}
*/
boolean videoEncode(@NonNull OnVideoSavedCallback videoSavedCallback, @NonNull String cameraId,
@NonNull Size resolution,
@NonNull OutputFileOptions outputFileOptions) {
// Main encoding loop. Exits on end of stream.
boolean errorOccurred = false;
boolean videoEos = false;
while (!videoEos && !errorOccurred) {
// Check for end of stream from main thread
if (mEndOfVideoStreamSignal.get()) {
mVideoEncoder.signalEndOfInputStream();
mEndOfVideoStreamSignal.set(false);
}
// Deque buffer to check for processing step
int outputBufferId =
mVideoEncoder.dequeueOutputBuffer(mVideoBufferInfo, DEQUE_TIMEOUT_USEC);
switch (outputBufferId) {
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
if (mMuxerStarted.get()) {
videoSavedCallback.onError(
ERROR_ENCODER,
"Unexpected change in video encoding format.",
null);
errorOccurred = true;
}
synchronized (mMuxerLock) {
mVideoTrackIndex = mMuxer.addTrack(mVideoEncoder.getOutputFormat());
if ((mIsAudioEnabled.get() && mAudioTrackIndex >= 0
&& mVideoTrackIndex >= 0)
|| (!mIsAudioEnabled.get() && mVideoTrackIndex >= 0)) {
Logger.i(TAG, "MediaMuxer started on video encode thread and audio "
+ "enabled: " + mIsAudioEnabled);
mMuxer.start();
mMuxerStarted.set(true);
}
}
break;
case MediaCodec.INFO_TRY_AGAIN_LATER:
// Timed out. Just wait until next attempt to deque.
break;
default:
videoEos = writeVideoEncodedBuffer(outputBufferId);
}
}
try {
Logger.i(TAG, "videoEncoder stop");
mVideoEncoder.stop();
} catch (IllegalStateException e) {
videoSavedCallback.onError(ERROR_ENCODER,
"Video encoder stop failed!", e);
errorOccurred = true;
}
try {
// new MediaMuxer instance required for each new file written, and release current one.
synchronized (mMuxerLock) {
if (mMuxer != null) {
if (mMuxerStarted.get()) {
Logger.i(TAG, "Muxer already started");
mMuxer.stop();
}
mMuxer.release();
mMuxer = null;
}
}
// A final checking for recording result, if the recorded file has no key
// frame, then the video file is not playable, needs to call
// onError() and will be removed.
boolean checkResult = removeRecordingResultIfNoVideoKeyFrameArrived(outputFileOptions);
if (!checkResult) {
videoSavedCallback.onError(ERROR_RECORDING_TOO_SHORT,
"The file has no video key frame.", null);
errorOccurred = true;
}
} catch (IllegalStateException e) {
// The video encoder has not got the key frame yet.
Logger.i(TAG, "muxer stop IllegalStateException: " + System.currentTimeMillis());
Logger.i(TAG,
"muxer stop exception, mIsFirstVideoKeyFrameWrite: "
+ mIsFirstVideoKeyFrameWrite.get());
if (mIsFirstVideoKeyFrameWrite.get()) {
// If muxer throws IllegalStateException at this moment and also the key frame
// has received, this will reported as a Muxer stop failed.
// Otherwise, this error will be ERROR_RECORDING_TOO_SHORT.
videoSavedCallback.onError(ERROR_MUXER, "Muxer stop failed!", e);
} else {
videoSavedCallback.onError(ERROR_RECORDING_TOO_SHORT,
"The file has no video key frame.", null);
}
errorOccurred = true;
}
if (mParcelFileDescriptor != null) {
try {
mParcelFileDescriptor.close();
mParcelFileDescriptor = null;
} catch (IOException e) {
videoSavedCallback.onError(ERROR_MUXER, "File descriptor close failed!", e);
errorOccurred = true;
}
}
mMuxerStarted.set(false);
// notify the UI thread that the video recording has finished
mEndOfAudioVideoSignal.set(true);
mIsFirstVideoKeyFrameWrite.set(false);
Logger.i(TAG, "Video encode thread end.");
return errorOccurred;
}
boolean audioEncode(OnVideoSavedCallback videoSavedCallback) {
// Audio encoding loop. Exits on end of stream.
boolean audioEos = false;
int outIndex;
long lastAudioTimestamp = 0;
while (!audioEos && mIsRecording) {
// Check for end of stream from main thread
if (mEndOfAudioStreamSignal.get()) {
mEndOfAudioStreamSignal.set(false);
mIsRecording = false;
}
// get audio deque input buffer
if (mAudioEncoder != null && mAudioRecorder != null) {
try {
int index = mAudioEncoder.dequeueInputBuffer(-1);
if (index >= 0) {
final ByteBuffer buffer = getInputBuffer(mAudioEncoder, index);
buffer.clear();
int length = mAudioRecorder.read(buffer, mAudioBufferSize);
if (length > 0) {
mAudioEncoder.queueInputBuffer(
index,
0,
length,
(System.nanoTime() / 1000),
mIsRecording ? 0 : MediaCodec.BUFFER_FLAG_END_OF_STREAM);
}
}
} catch (MediaCodec.CodecException e) {
Logger.i(TAG, "audio dequeueInputBuffer CodecException " + e.getMessage());
} catch (IllegalStateException e) {
Logger.i(TAG,
"audio dequeueInputBuffer IllegalStateException " + e.getMessage());
}
// start to dequeue audio output buffer
do {
outIndex = mAudioEncoder.dequeueOutputBuffer(mAudioBufferInfo, 0);
switch (outIndex) {
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
synchronized (mMuxerLock) {
mAudioTrackIndex = mMuxer.addTrack(mAudioEncoder.getOutputFormat());
if (mAudioTrackIndex >= 0 && mVideoTrackIndex >= 0) {
Logger.i(TAG, "MediaMuxer start on audio encoder thread.");
mMuxer.start();
mMuxerStarted.set(true);
}
}
break;
case MediaCodec.INFO_TRY_AGAIN_LATER:
break;
default:
// Drops out of order audio frame if the frame's earlier than last
// frame.
if (mAudioBufferInfo.presentationTimeUs > lastAudioTimestamp) {
audioEos = writeAudioEncodedBuffer(outIndex);
lastAudioTimestamp = mAudioBufferInfo.presentationTimeUs;
} else {
Logger.w(TAG,
"Drops frame, current frame's timestamp "
+ mAudioBufferInfo.presentationTimeUs
+ " is earlier that last frame "
+ lastAudioTimestamp);
// Releases this frame from output buffer
mAudioEncoder.releaseOutputBuffer(outIndex, false);
}
}
} while (outIndex >= 0 && !audioEos); // end of dequeue output buffer
}
} // end of while loop
// Audio Stop
try {
Logger.i(TAG, "audioRecorder stop");
mAudioRecorder.stop();
} catch (IllegalStateException e) {
videoSavedCallback.onError(
ERROR_ENCODER, "Audio recorder stop failed!", e);
}
try {
mAudioEncoder.stop();
} catch (IllegalStateException e) {
videoSavedCallback.onError(ERROR_ENCODER,
"Audio encoder stop failed!", e);
}
Logger.i(TAG, "Audio encode thread end");
// Use AtomicBoolean to signal because MediaCodec.signalEndOfInputStream() is not thread
// safe
mEndOfVideoStreamSignal.set(true);
return false;
}
private ByteBuffer getInputBuffer(MediaCodec codec, int index) {
return codec.getInputBuffer(index);
}
private ByteBuffer getOutputBuffer(MediaCodec codec, int index) {
return codec.getOutputBuffer(index);
}
/** Creates a {@link MediaFormat} using parameters for audio from the configuration */
private MediaFormat createAudioMediaFormat() {
MediaFormat format =
MediaFormat.createAudioFormat(AUDIO_MIME_TYPE, mAudioSampleRate,
mAudioChannelCount);
format.setInteger(
MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
format.setInteger(MediaFormat.KEY_BIT_RATE, mAudioBitRate);
return format;
}
/** Create a AudioRecord object to get raw data */
@RequiresPermission(Manifest.permission.RECORD_AUDIO)
private AudioRecord autoConfigAudioRecordSource(VideoCaptureConfig config) {
// Use channel count to determine stereo vs mono
int channelConfig =
mAudioChannelCount == 1
? AudioFormat.CHANNEL_IN_MONO
: AudioFormat.CHANNEL_IN_STEREO;
try {
// Use only ENCODING_PCM_16BIT because it mandatory supported.
int bufferSize =
AudioRecord.getMinBufferSize(mAudioSampleRate, channelConfig,
AudioFormat.ENCODING_PCM_16BIT);
if (bufferSize <= 0) {
bufferSize = config.getAudioMinBufferSize();
}
AudioRecord recorder =
new AudioRecord(
AudioSource.CAMCORDER,
mAudioSampleRate,
channelConfig,
AudioFormat.ENCODING_PCM_16BIT,
bufferSize * 2);
if (recorder.getState() == AudioRecord.STATE_INITIALIZED) {
mAudioBufferSize = bufferSize;
Logger.i(
TAG,
"source: "
+ AudioSource.CAMCORDER
+ " audioSampleRate: "
+ mAudioSampleRate
+ " channelConfig: "
+ channelConfig
+ " audioFormat: "
+ AudioFormat.ENCODING_PCM_16BIT
+ " bufferSize: "
+ bufferSize);
return recorder;
}
} catch (Exception e) {
Logger.e(TAG, "Exception, keep trying.", e);
}
return null;
}
/** Set audio record parameters by CamcorderProfile */
private void setAudioParametersByCamcorderProfile(Size currentResolution, String cameraId) {
CamcorderProfile profile;
boolean isCamcorderProfileFound = false;
try {
for (int quality : CamcorderQuality) {
if (CamcorderProfile.hasProfile(Integer.parseInt(cameraId), quality)) {
profile = CamcorderProfile.get(Integer.parseInt(cameraId), quality);
if (currentResolution.getWidth() == profile.videoFrameWidth
&& currentResolution.getHeight() == profile.videoFrameHeight) {
mAudioChannelCount = profile.audioChannels;
mAudioSampleRate = profile.audioSampleRate;
mAudioBitRate = profile.audioBitRate;
isCamcorderProfileFound = true;
break;
}
}
}
} catch (NumberFormatException e) {
Logger.i(TAG, "The camera Id is not an integer because the camera may be a removable "
+ "device. Use the default values for the audio related settings.");
}
// In case no corresponding camcorder profile can be founded, * get default value from
// VideoCaptureConfig.
if (!isCamcorderProfileFound) {
VideoCaptureConfig config = (VideoCaptureConfig) getCurrentConfig();
mAudioChannelCount = config.getAudioChannelCount();
mAudioSampleRate = config.getAudioSampleRate();
mAudioBitRate = config.getAudioBitRate();
}
}
private boolean removeRecordingResultIfNoVideoKeyFrameArrived(
@NonNull OutputFileOptions outputFileOptions) {
boolean checkKeyFrame;
// 1. There should be one video key frame at least.
Logger.i(TAG,
"check Recording Result First Video Key Frame Write: "
+ mIsFirstVideoKeyFrameWrite.get());
if (!mIsFirstVideoKeyFrameWrite.get()) {
Logger.i(TAG, "The recording result has no key frame.");
checkKeyFrame = false;
} else {
checkKeyFrame = true;
}
return checkKeyFrame;
}
/**
* Describes the error that occurred during video capture operations.
*
* This is a parameter sent to the error callback functions set in listeners such as {@link
* VideoCapture.OnVideoSavedCallback#onError(int, String, Throwable)}.
*
* See message parameter in onError callback or log for more details.
*
* @hide
*/
@IntDef({ERROR_UNKNOWN, ERROR_ENCODER, ERROR_MUXER, ERROR_RECORDING_IN_PROGRESS,
ERROR_FILE_IO, ERROR_INVALID_CAMERA, ERROR_RECORDING_TOO_SHORT})
@Retention(RetentionPolicy.SOURCE)
@RestrictTo(Scope.LIBRARY_GROUP)
public @interface VideoCaptureError {
}
enum VideoEncoderInitStatus {
VIDEO_ENCODER_INIT_STATUS_UNINITIALIZED,
VIDEO_ENCODER_INIT_STATUS_INITIALIZED_FAILED,
VIDEO_ENCODER_INIT_STATUS_INSUFFICIENT_RESOURCE,
VIDEO_ENCODER_INIT_STATUS_RESOURCE_RECLAIMED,
}
/** Listener containing callbacks for video file I/O events. */
public interface OnVideoSavedCallback {
/** Called when the video has been successfully saved. */
void onVideoSaved();
/** Called when an error occurs while attempting to save the video. */
void onError(@VideoCaptureError int videoCaptureError, @NonNull String message,
@Nullable Throwable cause);
}
/**
* Provides a base static default configuration for the VideoCapture
*
* These values may be overridden by the implementation. They only provide a minimum set of
* defaults that are implementation independent.
*
* @hide
*/
@RestrictTo(Scope.LIBRARY_GROUP)
public static final class Defaults
implements ConfigProvider The name should be a value that can uniquely identify an instance of the object being
* configured.
*
* If not set, the target name will default to an unique name automatically generated
* with the class canonical name and random UUID.
*
* @param targetName A unique string identifier for the instance of the class being
* configured.
* @return the current Builder.
*/
@Override
@NonNull
public Builder setTargetName(@NonNull String targetName) {
getMutableConfig().insertOption(OPTION_TARGET_NAME, targetName);
return this;
}
// Implementations of ImageOutputConfig.Builder default methods
/**
* Sets the aspect ratio of the intended target for images from this configuration.
*
* It is not allowed to set both target aspect ratio and target resolution on the same
* use case.
*
* The target aspect ratio is used as a hint when determining the resulting output aspect
* ratio which may differ from the request, possibly due to device constraints.
* Application code should check the resulting output's resolution.
*
* If not set, resolutions with aspect ratio 4:3 will be considered in higher
* priority.
*
* @param aspectRatio A {@link AspectRatio} representing the ratio of the
* target's width and height.
* @return The current Builder.
* @hide
*/
@RestrictTo(Scope.LIBRARY_GROUP)
@NonNull
@Override
public Builder setTargetAspectRatio(@AspectRatio.Ratio int aspectRatio) {
getMutableConfig().insertOption(OPTION_TARGET_ASPECT_RATIO, aspectRatio);
return this;
}
/**
* Sets the rotation of the intended target for images from this configuration.
*
* This is one of four valid values: {@link Surface#ROTATION_0}, {@link
* Surface#ROTATION_90}, {@link Surface#ROTATION_180}, {@link Surface#ROTATION_270}.
* Rotation values are relative to the "natural" rotation, {@link Surface#ROTATION_0}.
*
* If not set, the target rotation will default to the value of
* {@link Display#getRotation()} of the default display at the time the use case is
* created. The use case is fully created once it has been attached to a camera.
*
* @param rotation The rotation of the intended target.
* @return The current Builder.
* @hide
*/
@RestrictTo(Scope.LIBRARY_GROUP)
@NonNull
@Override
public Builder setTargetRotation(@RotationValue int rotation) {
getMutableConfig().insertOption(OPTION_TARGET_ROTATION, rotation);
return this;
}
/**
* Sets the resolution of the intended target from this configuration.
*
* The target resolution attempts to establish a minimum bound for the image resolution.
* The actual image resolution will be the closest available resolution in size that is not
* smaller than the target resolution, as determined by the Camera implementation. However,
* if no resolution exists that is equal to or larger than the target resolution, the
* nearest available resolution smaller than the target resolution will be chosen.
*
* It is not allowed to set both target aspect ratio and target resolution on the same
* use case.
*
* The target aspect ratio will also be set the same as the aspect ratio of the provided
* {@link Size}. Make sure to set the target resolution with the correct orientation.
*
* @param resolution The target resolution to choose from supported output sizes list.
* @return The current Builder.
* @hide
*/
@RestrictTo(Scope.LIBRARY_GROUP)
@NonNull
@Override
public Builder setTargetResolution(@NonNull Size resolution) {
getMutableConfig().insertOption(OPTION_TARGET_RESOLUTION, resolution);
return this;
}
/**
* Sets the default resolution of the intended target from this configuration.
*
* @param resolution The default resolution to choose from supported output sizes list.
* @return The current Builder.
* @hide
*/
@RestrictTo(Scope.LIBRARY_GROUP)
@NonNull
@Override
public Builder setDefaultResolution(@NonNull Size resolution) {
getMutableConfig().insertOption(OPTION_DEFAULT_RESOLUTION, resolution);
return this;
}
/** @hide */
@RestrictTo(Scope.LIBRARY_GROUP)
@NonNull
@Override
public Builder setMaxResolution(@NonNull Size resolution) {
getMutableConfig().insertOption(OPTION_MAX_RESOLUTION, resolution);
return this;
}
/** @hide */
@RestrictTo(Scope.LIBRARY_GROUP)
@Override
@NonNull
public Builder setSupportedResolutions(@NonNull List If not set, the background executor will default to an automatically generated
* {@link Executor}.
*
* @param executor The executor which will be used for background tasks.
* @return the current Builder.
* @hide
*/
@RestrictTo(Scope.LIBRARY_GROUP)
@Override
@NonNull
public Builder setBackgroundExecutor(@NonNull Executor executor) {
getMutableConfig().insertOption(OPTION_BACKGROUND_EXECUTOR, executor);
return this;
}
// Implementations of UseCaseConfig.Builder default methods
/** @hide */
@RestrictTo(Scope.LIBRARY_GROUP)
@Override
@NonNull
public Builder setDefaultSessionConfig(@NonNull SessionConfig sessionConfig) {
getMutableConfig().insertOption(OPTION_DEFAULT_SESSION_CONFIG, sessionConfig);
return this;
}
/** @hide */
@RestrictTo(Scope.LIBRARY_GROUP)
@Override
@NonNull
public Builder setDefaultCaptureConfig(@NonNull CaptureConfig captureConfig) {
getMutableConfig().insertOption(OPTION_DEFAULT_CAPTURE_CONFIG, captureConfig);
return this;
}
/** @hide */
@RestrictTo(Scope.LIBRARY_GROUP)
@Override
@NonNull
public Builder setSessionOptionUnpacker(
@NonNull SessionConfig.OptionUnpacker optionUnpacker) {
getMutableConfig().insertOption(OPTION_SESSION_CONFIG_UNPACKER, optionUnpacker);
return this;
}
/** @hide */
@RestrictTo(Scope.LIBRARY_GROUP)
@Override
@NonNull
public Builder setCaptureOptionUnpacker(
@NonNull CaptureConfig.OptionUnpacker optionUnpacker) {
getMutableConfig().insertOption(OPTION_CAPTURE_CONFIG_UNPACKER, optionUnpacker);
return this;
}
/** @hide */
@RestrictTo(Scope.LIBRARY_GROUP)
@Override
@NonNull
public Builder setSurfaceOccupancyPriority(int priority) {
getMutableConfig().insertOption(OPTION_SURFACE_OCCUPANCY_PRIORITY, priority);
return this;
}
/** @hide */
@RestrictTo(Scope.LIBRARY)
@Override
@NonNull
public Builder setCameraSelector(@NonNull CameraSelector cameraSelector) {
getMutableConfig().insertOption(OPTION_CAMERA_SELECTOR, cameraSelector);
return this;
}
/** @hide */
@RestrictTo(Scope.LIBRARY_GROUP)
@Override
@NonNull
public Builder setUseCaseEventCallback(
@NonNull UseCase.EventCallback useCaseEventCallback) {
getMutableConfig().insertOption(OPTION_USE_CASE_EVENT_CALLBACK, useCaseEventCallback);
return this;
}
}
/**
* Options for saving newly captured video.
*
* this class is used to configure save location and metadata. Save location can be
* either a {@link File}, {@link MediaStore}. The metadata will be
* stored with the saved video.
*/
public static final class OutputFileOptions {
private final SeekableWriter mWriter;
public OutputFileOptions(SeekableWriter writer) {
mWriter = writer;
}
}
/**
* Nested class to avoid verification errors for methods introduced in Android 6.0 (API 23).
*/
@RequiresApi(23)
private static class Api23Impl {
private Api23Impl() {
}
@DoNotInline
static int getCodecExceptionErrorCode(MediaCodec.CodecException e) {
return e.getErrorCode();
}
}
}