Android Camera原理之CameraDeviceCallbacks回调模块

时间 2019-11-06

标签 android camera 原理 cameradevicecallbacks 模块栏目 Android 繁體版

原文原文链接

在讲解《Android Camera原理之openCamera模块(二)》一文的时候提到了CameraDeviceCallbacks回调，当时没有详细展开，本文咱们详细展开讲解一下。
CameraDeviceCallbacks生成过程：
《Android Camera进程间通讯类总结》中2.ICameraDeviceCallbacks.aidl详细总结了CameraDeviceCallbacks的生成过程。
frameworks/av/camera/ndk/impl/ACameraDevice.h中回调接口以下。这个CameraDeviceCallbacks是openCamera的时候设置到camera service端的，后续HAL层有camera响应的话会调用ACameraDevice.h中的ServiceCallback接口来实现回调。咱们须要从流程上搞清楚这些回调是在什么场景下触发的，明白了这些，才真正明白camera capture的流程。java

// Callbacks from camera service
    class ServiceCallback : public hardware::camera2::BnCameraDeviceCallbacks {
      public:
        explicit ServiceCallback(CameraDevice* device) : mDevice(device) {}
        binder::Status onDeviceError(int32_t errorCode,
                           const CaptureResultExtras& resultExtras) override;
        binder::Status onDeviceIdle() override;
        binder::Status onCaptureStarted(const CaptureResultExtras& resultExtras,
                              int64_t timestamp) override;
        binder::Status onResultReceived(const CameraMetadata& metadata,
                              const CaptureResultExtras& resultExtras,
                              const std::vector<PhysicalCaptureResultInfo>& physicalResultInfos) override;
        binder::Status onPrepared(int streamId) override;
        binder::Status onRequestQueueEmpty() override;
        binder::Status onRepeatingRequestError(int64_t lastFrameNumber,
                int32_t stoppedSequenceId) override;
      private:
        const wp<CameraDevice> mDevice;
    };

1.onDeviceError

onDeviceError回调流程.jpgsession

connectHelper
在openCamera执行的时候会检查当前camera device是否正常。

binderDied
在camera service 死亡的时候，死亡回调中会通知上层当前的camera device可能有问题。

还有一个调用的地方是Camera3Device::notifyError--->从HAL传递上来的，关于当前device是否正常的信息，若是device存在问题执行回调--->listener->notifyError(errorCode, resultExtras);ide

void Camera3Device::notifyError(const camera3_error_msg_t &msg,
        sp<NotificationListener> listener) {
    ATRACE_CALL();
    // Map camera HAL error codes to ICameraDeviceCallback error codes
    // Index into this with the HAL error code
    static const int32_t halErrorMap[CAMERA3_MSG_NUM_ERRORS] = {
        // 0 = Unused error code
        hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_INVALID_ERROR,
        // 1 = CAMERA3_MSG_ERROR_DEVICE
        hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_DEVICE,
        // 2 = CAMERA3_MSG_ERROR_REQUEST
        hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_REQUEST,
        // 3 = CAMERA3_MSG_ERROR_RESULT
        hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT,
        // 4 = CAMERA3_MSG_ERROR_BUFFER
        hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER
    };

    int32_t errorCode =
            ((msg.error_code >= 0) &&
                    (msg.error_code < CAMERA3_MSG_NUM_ERRORS)) ?
            halErrorMap[msg.error_code] :
            hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_INVALID_ERROR;

    int streamId = 0;
    if (msg.error_stream != NULL) {
        Camera3Stream *stream =
                Camera3Stream::cast(msg.error_stream);
        streamId = stream->getId();
    }
    ALOGV("Camera %s: %s: HAL error, frame %d, stream %d: %d",
            mId.string(), __FUNCTION__, msg.frame_number,
            streamId, msg.error_code);

    CaptureResultExtras resultExtras;
    switch (errorCode) {
        case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_DEVICE:
            // SET_ERR calls notifyError
            SET_ERR("Camera HAL reported serious device error");
            break;
        case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_REQUEST:
        case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT:
        case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER:
            {
                Mutex::Autolock l(mInFlightLock);
                ssize_t idx = mInFlightMap.indexOfKey(msg.frame_number);
                if (idx >= 0) {
                    InFlightRequest &r = mInFlightMap.editValueAt(idx);
                    r.requestStatus = msg.error_code;
                    resultExtras = r.resultExtras;
                    if (hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT == errorCode
                            ||  hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_REQUEST ==
                            errorCode) {
                        r.skipResultMetadata = true;
                    }
                    if (hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT ==
                            errorCode) {
                        // In case of missing result check whether the buffers
                        // returned. If they returned, then remove inflight
                        // request.
                        removeInFlightRequestIfReadyLocked(idx);
                    }
                } else {
                    resultExtras.frameNumber = msg.frame_number;
                    ALOGE("Camera %s: %s: cannot find in-flight request on "
                            "frame %" PRId64 " error", mId.string(), __FUNCTION__,
                            resultExtras.frameNumber);
                }
            }
            resultExtras.errorStreamId = streamId;
            if (listener != NULL) {
                listener->notifyError(errorCode, resultExtras);
            } else {
                ALOGE("Camera %s: %s: no listener available", mId.string(), __FUNCTION__);
            }
            break;
        default:
            // SET_ERR calls notifyError
            SET_ERR("Unknown error message from HAL: %d", msg.error_code);
            break;
    }
}

2.onDeviceIdle

onDeviceIdle回调流程.jpg函数

openCamera执行以后开始调用到onDeviceIdle回调。这儿的调用过程须要讲讲Camera3Device::initializeCommonLockedoop

status_t Camera3Device::initializeCommonLocked() {
    /** Start up status tracker thread */
    mStatusTracker = new StatusTracker(this);
    status_t res = mStatusTracker->run(String8::format("C3Dev-%s-Status", mId.string()).string());
    if (res != OK) {
        SET_ERR_L("Unable to start status tracking thread: %s (%d)",
                strerror(-res), res);
        mInterface->close();
        mStatusTracker.clear();
        return res;
    }
//......
}

StatusTracker是C++中定义的线程，和java层有点相似，线程执行run会自动调用到threadLoop()，这应该很好理解。this

class StatusTracker: public Thread {
  public:
    explicit StatusTracker(wp<Camera3Device> parent);
    ~StatusTracker();
    virtual void requestExit();
  protected:
    virtual bool threadLoop();
}

接下来这个threadLoop()会一直执行，若是在执行过程当中发现当前的mStateTransitions[表示设备状态]处于IDLE，这时候回调到上层。spa

// Notify parent for all intermediate transitions
    if (mStateTransitions.size() > 0 && parent.get()) {
        for (size_t i = 0; i < mStateTransitions.size(); i++) {
            bool idle = (mStateTransitions[i] == IDLE);
            ALOGV("Camera device is now %s", idle ? "idle" : "active");
            parent->notifyStatus(idle);
        }
    }

3.onCaptureStarted

这个回调函数表示camera device已经准备好，能够开始调用camera获取capture frame数据了。回调的开始是camera HAL层获取底层camera device driver通知表示当前device已经准备好。通知的函数在Camera3Device::notify线程

Camera3Device::notify
底层msg通知当前camera 开门准备好了，能够随时准备拍照了。

void Camera3Device::notify(const camera3_notify_msg *msg) {
    ATRACE_CALL();
    sp<NotificationListener> listener;
    {
        Mutex::Autolock l(mOutputLock);
        listener = mListener.promote();
    }

    if (msg == NULL) {
        SET_ERR("HAL sent NULL notify message!");
        return;
    }

    switch (msg->type) {
        case CAMERA3_MSG_ERROR: {
            notifyError(msg->message.error, listener);
            break;
        }
        case CAMERA3_MSG_SHUTTER: {
            notifyShutter(msg->message.shutter, listener);
            break;
        }
        default:
            SET_ERR("Unknown notify message from HAL: %d",
                    msg->type);
    }
}

Camera3Device::notifyShutter
---> CameraDeviceClient::notifyShutter
此时回调到上层通知开发者当前的device shutter准备好了。

void CameraDeviceClient::notifyShutter(const CaptureResultExtras& resultExtras,
        nsecs_t timestamp) {
    // Thread safe. Don't bother locking.
    sp<hardware::camera2::ICameraDeviceCallbacks> remoteCb = getRemoteCallback();
    if (remoteCb != 0) {
        remoteCb->onCaptureStarted(resultExtras, timestamp);
    }
    Camera2ClientBase::notifyShutter(resultExtras, timestamp);
}

4.onResultReceived

预览的时候这个函数很重要，表示抓取的帧数据不断地返回，camera device正在不断消耗capture frame。code

CameraDeviceClient::initializeImpl--->CameraDeviceClient初始化执行的时候会启动FrameProcessorBase线程。
FrameProcessorBase继承一个线程，执行run以后，它的threadLoop开始启动。

template<typename TProviderPtr>
status_t CameraDeviceClient::initializeImpl(TProviderPtr providerPtr, const String8& monitorTags) {
//......
    String8 threadName;
    mFrameProcessor = new FrameProcessorBase(mDevice);
    threadName = String8::format("CDU-%s-FrameProc", mCameraIdStr.string());
    mFrameProcessor->run(threadName.string());

    mFrameProcessor->registerListener(FRAME_PROCESSOR_LISTENER_MIN_ID,
                                      FRAME_PROCESSOR_LISTENER_MAX_ID,
                                      /*listener*/this,
                                      /*sendPartials*/true);
//......
}

FrameProcessorBase::threadLoop
此时会不断到Camera HAL层取请求，是否有新的frame数据，若是有的话，处理capture frame数据。

bool FrameProcessorBase::threadLoop() {
    status_t res;

    sp<CameraDeviceBase> device;
    {
        device = mDevice.promote();
        if (device == 0) return false;
    }

    res = device->waitForNextFrame(kWaitDuration);
    if (res == OK) {
        processNewFrames(device);
    } else if (res != TIMED_OUT) {
        ALOGE("FrameProcessorBase: Error waiting for new "
                "frames: %s (%d)", strerror(-res), res);
    }

    return true;
}

FrameProcessorBase::processNewFrames
---> FrameProcessorBase::processSingleFrame
--->FrameProcessorBase::processListeners
--->CameraDeviceClient::onResultAvailable
经过CameraDeviceClient::onResultAvailable函数回调到上层，而后在回调到CameraCaptureSession.CaptureCallback--->onCaptureProgressed告知开发者当前正在不断捕获capture frame数据。

/** Device-related methods */
void CameraDeviceClient::onResultAvailable(const CaptureResult& result) {
    ATRACE_CALL();
    ALOGV("%s", __FUNCTION__);

    // Thread-safe. No lock necessary.
    sp<hardware::camera2::ICameraDeviceCallbacks> remoteCb = mRemoteCallback;
    if (remoteCb != NULL) {
        remoteCb->onResultReceived(result.mMetadata, result.mResultExtras,
                result.mPhysicalMetadatas);
    }
}

5.onPrepared

CameraCaptureSession中有两个接口：这个函数主要是用来给surface预分配内存，可是为了加速预览显示的速度。orm

public abstract void prepare(@NonNull Surface surface) throws CameraAccessException;

    public abstract void prepare(int maxCount, @NonNull Surface surface)
            throws CameraAccessException;

而CameraDeviceCallbacks->onPrepared回调就是在执行CameraCaptureSession->prepare以后回调执行的。若是向HAL申请camera device 内存分配成功，才会触发这个回调。

6.onRequestQueueEmpty

onRequestQueueEmpty回调是当前camera device的非预览流队列为空，开始准备capture 下一张图片了，只是一个中间状态。这儿不过重要，我列个调用流程就好了。

Camera3Device::RequestThread::threadLoop

bool Camera3Device::RequestThread::threadLoop() {
//......
    // Wait for the next batch of requests.
    waitForNextRequestBatch();
//......
}

Camera3Device::RequestThread::waitForNextRequestBatch

void Camera3Device::RequestThread::waitForNextRequestBatch() {
//......
    NextRequest nextRequest;
    nextRequest.captureRequest = waitForNextRequestLocked();
    if (nextRequest.captureRequest == nullptr) {
        return;
    }

    nextRequest.halRequest = camera3_capture_request_t();
    nextRequest.submitted = false;
    mNextRequests.add(nextRequest);

    // Wait for additional requests
    const size_t batchSize = nextRequest.captureRequest->mBatchSize;

    for (size_t i = 1; i < batchSize; i++) {
        NextRequest additionalRequest;
        additionalRequest.captureRequest = waitForNextRequestLocked();
        if (additionalRequest.captureRequest == nullptr) {
            break;
        }

        additionalRequest.halRequest = camera3_capture_request_t();
        additionalRequest.submitted = false;
        mNextRequests.add(additionalRequest);
    }
//......
}

Camera3Device::RequestThread::waitForNextRequestLocked

sp<Camera3Device::CaptureRequest>
        Camera3Device::RequestThread::waitForNextRequestLocked() {
//......
    if (nextRequest == NULL) {
        // Don't have a repeating request already in hand, so queue
        // must have an entry now.
        RequestList::iterator firstRequest =
                mRequestQueue.begin();
        nextRequest = *firstRequest;
        mRequestQueue.erase(firstRequest);
        if (mRequestQueue.empty() && !nextRequest->mRepeating) {
            sp<NotificationListener> listener = mListener.promote();
            if (listener != NULL) {
                listener->notifyRequestQueueEmpty();
            }
        }
    }
//......
}

CameraDeviceClient::notifyRequestQueueEmpty这儿会直接回调到上层。

void CameraDeviceClient::notifyRequestQueueEmpty() {
    // Thread safe. Don't bother locking.
    sp<hardware::camera2::ICameraDeviceCallbacks> remoteCb = getRemoteCallback();
    if (remoteCb != 0) {
        remoteCb->onRequestQueueEmpty();
    }
}

7.onRepeatingRequestError

Camera3Device.h中定义了一个RequestThread线程，用来管理capture request和HAL device之间的链接。

/**
     * Thread for managing capture request submission to HAL device.
     */
    class RequestThread : public Thread {
//......
      protected:
        virtual bool threadLoop();
    }

RequestThread启动的地方在Camera3Device::initializeCommonLocked中。

/** Start up request queue thread */
    mRequestThread = new RequestThread(this, mStatusTracker, mInterface, sessionParamKeys);
    res = mRequestThread->run(String8::format("C3Dev-%s-ReqQueue", mId.string()).string());
    if (res != OK) {
        SET_ERR_L("Unable to start request queue thread: %s (%d)",
                strerror(-res), res);
        mInterface->close();
        mRequestThread.clear();
        return res;
    }

线程核心的执行逻辑都在threadLoop函数中--->bool Camera3Device::RequestThread::threadLoop()

bool Camera3Device::RequestThread::threadLoop() {
//......
    // Prepare a batch of HAL requests and output buffers.
    res = prepareHalRequests();
    if (res == TIMED_OUT) {
        // Not a fatal error if getting output buffers time out.
        cleanUpFailedRequests(/*sendRequestError*/ true);
        // Check if any stream is abandoned.
        checkAndStopRepeatingRequest();
        return true;
    } else if (res != OK) {
        cleanUpFailedRequests(/*sendRequestError*/ false);
        return false;
    }
//......
}

请求HAL 层的device，若是请求超时，此时的预览是没法进行下去的。执行checkAndStopRepeatingRequest();

void Camera3Device::RequestThread::checkAndStopRepeatingRequest() {
//......
    if (listener != NULL && surfaceAbandoned) {
        listener->notifyRepeatingRequestError(lastFrameNumber);
    }
}

最终调用到CameraDeviceClient::notifyRepeatingRequestError，而后直接回调到上层，通知开发者当前的capture request请求失败了。

void CameraDeviceClient::notifyRepeatingRequestError(long lastFrameNumber) {
    sp<hardware::camera2::ICameraDeviceCallbacks> remoteCb = getRemoteCallback();

    if (remoteCb != 0) {
        remoteCb->onRepeatingRequestError(lastFrameNumber, mStreamingRequestId);
    }

    Mutex::Autolock idLock(mStreamingRequestIdLock);
    mStreamingRequestId = REQUEST_ID_NONE;
}