Android(安卓)Native层开发Camera应用的方法

现在大部分Android Camera 应用都是基于Android标准API1/API2进行开发的，但是其实Android Camera应用也是可以直接在Android Native层进行开发，而且存在多种方法：
我们认为至少存在三种方法

直接和CameraSerive通信进行开发(Android NDK libcamera2ndk就是使用该方案)
直接和CameraProvider通信进行开发(高通的QVRService就是这样做的)
直接使用camx接口(Hal3Test就是这样做的)

从android7.0（API level24）开始，Android NDK 集成了libcamera2ndk.so，其提供了Native层开发Camera应用的API，可以参考下
android_graphics_cts_CameraGpuCtsActivity.cpp，其实现原理就是直接和CameraSerive通信进行开发，由于libcamera2ndk.so不能修改，所以我们并没有直接使用该库。

直接在Android Native 层开发Camera应用的优缺点：

优点：

摒弃了Java层调用，能在性能方面存在一定的优化
由于高性能的算法大部分都是基于c/c++开发的，在Native层开发相机应用可以方便算法集成

缺点：
3. 参考资料少，开发难度稍高，会遇到很多bug，当然都是可以解决的。

我现在只针对第一种方式进行介绍下(需要Android源码环境)，第二、三种方式没有具体去实现，就不介绍了。

1. 直接和CameraSerive通信进行开发进行Native Camera开发

1.1 CameraSerive接口定义

CameraService所有对外接口都定义在ICameraService.aidl文件中。
主要定义的接口有：

    /**     * Open a camera device through the new camera API     * Only supported for device HAL versions >= 3.2     */    ICameraDeviceUser connectDevice(ICameraDeviceCallbacks callbacks,            String cameraId,            String opPackageName,            int clientUid);    CameraStatus[] addListener(ICameraServiceListener listener);                CameraMetadataNative getCameraCharacteristics(String cameraId);

1.2 CameraService 获取

首先需要从IServiceManager中获CameraService，代码如下：

    sp<IServiceManager> sm = defaultServiceManager();    sp<IBinder> binder = sm->getService(String16("media.camera"));    sp<ICameraService>mspService = interface_cast<ICameraService>(binder);

1.3 打开相机

CameraService打开相机的接口有好几个，我们使用了connectDevice，代码如下：

    sp<ICameraDeviceUser> device;    binder::Status res = mspService->connectDevice(callbacks, cameraId, clientPackageName,            ICameraService::USE_CALLING_UID, &device);

connectDevice会返回一个BpCameraDeviceUser 代理对象，表示一个相机设备，其继承自ICameraDeviceUser，ICameraDeviceUser定义如下：

   //ICameraDeviceUser.aidl   void disconnect();    const int NO_IN_FLIGHT_REPEATING_FRAMES = -1;    SubmitInfo submitRequest(in CaptureRequest request, boolean streaming);    SubmitInfo submitRequestList(in CaptureRequest[] requestList, boolean streaming);    /**     * Cancel the repeating request specified by requestId     * Returns the frame number of the last frame that will be produced from this     * repeating request, or NO_IN_FLIGHT_REPEATING_FRAMES if no frames were produced     * by this repeating request.     *     * Repeating request may be stopped by camera device due to an error. Canceling a stopped     * repeating request will trigger ERROR_ILLEGAL_ARGUMENT.     */    long cancelRequest(int requestId);    /**     * Begin the device configuration.     *     *      * beginConfigure must be called before any call to deleteStream, createStream,     * or endConfigure.  It is not valid to call this when the device is not idle.     * 
     */    void beginConfigure();    void endConfigure(int operatingMode);    void deleteStream(int streamId);    /**     * Create an output stream     *     * 
Create an output stream based on the given output configuration
     *     * @param outputConfiguration size, format, and other parameters for the stream     * @return new stream ID     */    int createStream(in OutputConfiguration outputConfiguration);

1.4 创建Stream流

在打开相机后就可以创建或者删除Stream了，在创建流Stream之前需要先准备Surface资源

1.4.1 Surface资源准备

如果预览数据不需要显示到真实的物理屏上，就可以通过BufferQueue::createBufferQueue创建出surface来，方法是：

void setupPreviewSurface(){    ALOGD("%s,%d E .", __FUNCTION__,__LINE__);    // Setup a buffer queue    BufferQueue::createBufferQueue(&mspGbProducer, &mspGbConsumer);    sp<BufferItemConsumer> consumer;    consumer = new BufferItemConsumer(mspGbConsumer, GRALLOC_USAGE_SW_READ_NEVER,            /*maxImages*/ MAX_BUFFER_NUM, /*controlledByApp*/ true);    String8 consumerName = String8::format("ImageReader-%dx%df%xm%d-%d",            m_iWidth, m_iHeight, m_iFormat, MAX_BUFFER_NUM, getpid());   //创建帧回调监听    mPreviewListener = new ConsumerBase::FrameAvailableListener();        consumer->setName(consumerName);    consumer->setDefaultBufferSize(m_iWidth, m_iHeight);    consumer->setDefaultBufferFormat(m_iFormat);    //给消费者注册帧回调监听    consumer->setFrameAvailableListener(mPreviewListener);    ALOGD("%s,%d X .", __FUNCTION__,__LINE__);}

如果预览帧数据需要显示到真实的物理屏上，就需要向SurfaceFlinger申请一个Surface

 sp<ANativeWindow> getSurfaceFromSF() {    status_t err;    sp<SurfaceComposerClient> surfaceComposerClient = new SurfaceComposerClient;    err = surfaceComposerClient->initCheck();    // Get main display parameters.    sp<IBinder> mainDpy = SurfaceComposerClient::getBuiltInDisplay(            ISurfaceComposer::eDisplayIdMain);    DisplayInfo mainDpyInfo;    err = SurfaceComposerClient::getDisplayInfo(mainDpy, &mainDpyInfo);    uint32_t width, height;    if (mainDpyInfo.orientation != DISPLAY_ORIENTATION_0 &&            mainDpyInfo.orientation != DISPLAY_ORIENTATION_180) {        // rotated        width = mainDpyInfo.h;        height = mainDpyInfo.w;    } else {        width = mainDpyInfo.w;        height = mainDpyInfo.h;    }    sp<SurfaceControl> sc = surfaceComposerClient->createSurface(            String8("mytest"), width, height,            PIXEL_FORMAT_RGBX_8888, ISurfaceComposerClient::eOpaque);    SurfaceComposerClient::openGlobalTransaction();    err = sc->setLayer(0x7FFFFFFF);     // always on top    err = sc->show();    SurfaceComposerClient::closeGlobalTransaction();    sp<ANativeWindow> anw = sc->getSurface();    return anw}

当然也存在其他方法，如从Activity中获取，欢迎补充

1.4.2 Stream创建

surface资源准备好后就可以创建Stream了，代码如下：

int configureDevice(){   //preview surface准备    setupPreviewSurface();   //capture surface准备    setupCaptureSurface();    ALOGD("waitUntilIdle E");    res = mspDevice->waitUntilIdle();    ALOGD("waitUntilIdle X");    res = mspDevice->beginConfigure();        ALOGD("create preview Stream E");    OutputConfiguration previewOutPut(mspGbProducer,          /*rotation*/ 0);    res = mspDevice->createStream(previewOutPut, &mPreviewStreamId);    ALOGD("create preview Stream mPreviewStreamId = %d X",mPreviewStreamId);    ALOGD("create capture Stream E");    OutputConfiguration jpegOutPut = OutputConfiguration(mspJpegGbpProducer,  /*rotation*/ 0);    res = mspDevice->createStream(jpegOutPut, &mCaptureStreamId);    ALOGD("create capture Stream  mCaptureStreamId = %d X",mCaptureStreamId);    ALOGD("endConfigure E .");    res = mspDevice->endConfigure(/*isConstrainedHighSpeed*/ false);    ALOGD("endConfigure X .");    return 0;}

1.4 预览申请

创建流成功后，就可以开启startpreview和takepicture了，在API2中是通过提交申请的方式来获取帧数据的，代码如下：

int startPreview() {    CameraMetadata    mRequestTemplate;    int templateId = camera2::ICameraDeviceUser::TEMPLATE_PREVIEW;    //createDefaultRequest获取的metadata只包含一些默认值    //如果想更新其里边的值，就需要更新metadata了    res = mspDevice->createDefaultRequest(templateId, /*out*/ &mRequestTemplate);    //for rbg camera, open qcom faceDetect feature defaultily.    if (m_iCameraId == CAMERA_RGB_ID) {        uint8_t faceDetectType  = TYPE_BYTE;        uint8_t faceDetectValue = ANDROID_STATISTICS_FACE_DETECT_MODE_SIMPLE;        updateMetaData("android.statistics.faceDetectMode",faceDetectType,&faceDetectValue,1);    }    std::vector< ::android::hardware::camera2::CaptureRequest>requestList;    sp<Surface> previewSurface = new Surface(mspGbProducer, /*controlledByApp*/ true);    //preview only    camera2::CaptureRequest previewTargetRequest;    previewTargetRequest.mMetadata    = requestMeta;    previewTargetRequest.mIsReprocess = false;    previewTargetRequest.mSurfaceList.add(previewSurface);    requestList.push_back(previewTargetRequest);    camera2::utils::SubmitInfo info;    //申请开启预览    res = mspDevice->submitRequestList(requestList, /*streaming*/ true, /*out*/ &info);    int32_t requestId       = info.mRequestId;    int64_t lastFrameNumber = info.mLastFrameNumber;    return 0;}

1.5 Metadata更新

在需要修改默认的MetaData时，就需要跟新CameraMetadata ，更新方法如下

int updateMetaData(CameraMetadata* metaData,char* key,uint8_t type,        void* data,int count){    status_t ret;    uint32_t tag = 0;    sp<VendorTagDescriptor> vTags;    sp<VendorTagDescriptorCache> cache = VendorTagDescriptorCache::getGlobalVendorTagCache();    if (cache.get()) {        const camera_metadata_t *metaBuffer = metaData->getAndLock();        metadata_vendor_id_t vendorId = get_camera_metadata_vendor_id(metaBuffer);        metaData->unlock(metaBuffer);        cache->getVendorTagDescriptor(vendorId, &vTags);    }    ret = CameraMetadata::getTagFromName(key,vTags.get(), &tag);    ret = metaData->update(tag,reinterpret_cast<const  T*>(data),count);    return ret;}

1.6 帧数据获取

在setupPreviewSurface时，创建了一对生产者消费者对象mspGbProducer，mspGbConsumer
其中mspGbProducer会封装成surface提供给CameraService使用，
同时会给mspGbConsumer注册了帧回调监听：

   //mPreviewListener是ConsumerBase::FrameAvailableListener的实现类    consumer->setFrameAvailableListener(mPreviewListener);

mPreviewListener实现了ConsumerBase::FrameAvailableListener接口，我们只介绍下
onFrameAvailable(const BufferItem &)实现。

void ConsumerBaseImple::onFrameAvailable(const BufferItem &){    //Part of ImageReader_imageSetup    BufferItem *buffer = new BufferItem;        BufferItemConsumer *bufferConsumer = getBufferConsumer();    //获取 graphicbuffer    status_t res = mspGbConsumer->acquireBuffer(buffer, 0);    //保存graphicbuffer    dumpImage(buffer);    // Part of ImageReader_imageRelease    //归还graphicbuffer    sp<Fence> releaseFence = Fence::NO_FENCE;    mspGbConsumer->releaseBuffer(*buffer, releaseFence);    returnBufferItem(buffer);}

上述流程graphicbuffer发生的状态变化为：
QUEUED->ACQUIRED->RELEASE
（应该还考虑下acquireFece和releaseFence的变化，这是之前实现的一个潜在问题！！！）
在acquireBuffer后就可以使用获取的graphicbuffer，我们这里只是保存了下，其他使用，如帧数据二次处理啥的都应该在这个地方实现。
我们直接介绍下如何保存graphicbuffer吧。

void dumpImage(BufferItem *buffer){    uint32_t width      = buffer->mGraphicBuffer->getWidth();    uint32_t height     = buffer->mGraphicBuffer->getHeight();    int      format     = buffer->mGraphicBuffer->getPixelFormat();    uint64_t Usage      = buffer->mGraphicBuffer->getUsage();    uint32_t stride     = buffer->mGraphicBuffer->getStride();    uint64_t frameNumber=buffer->mFrameNumber;    int readerFormat        = buffer->mGraphicBuffer->getPixelFormat();    LockedImage lockedImg   = LockedImage();    //从BufferItem中获取yuv数据或者raw16/raw10数据，jpeg数据等    lockImageFromBuffer(buffer.mGraphicBuffer,GRALLOC_USAGE_SW_READ_OFTEN,buffer->mCrop,buffer->mFence->dup(), &lockedImg);    uint8_t *pData      = NULL;    uint32_t dataSize   = 0;    int bytesPerPixel   = 0;//plant    int32_t fmt = lockedImg.flexFormat;    switch (fmt)    {        case HAL_PIXEL_FORMAT_YCbCr_420_888:        //dump nv12        dumpImage(lockedImg.data,lockedImg.dataCr,buffer->mFrameNumber,lockedImg.stride,lockedImg.height);        //dump nv21        dumpImage(lockedImg.data,lockedImg.dataCb,buffer->mFrameNumber,lockedImg.stride,lockedImg.height);            break;        case HAL_PIXEL_FORMAT_RAW16:            // Single plane 16bpp bayer data.            break;        case HAL_PIXEL_FORMAT_RAW10:            pData       = lockedImg.data;            dataSize    = lockedImg.stride * lockedImg.height;        case HAL_PIXEL_FORMAT_BLOB:        case HAL_PIXEL_FORMAT_JPEG:        case HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED:            pData       = lockedImg.data;            break;        default:;    }}

我们介绍下yuv数据获取方法

status_t lockImageFromBuffer(sp<GraphicBuffer> buffer, uint32_t inUsage,                             const Rect &rect, int fenceFd, LockedImage *outputImage){    if (HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED == buffer->getPixelFormat())    {        // This is the only opaque format exposed in the ImageFormat public API.        // Note that we do support CPU access for HAL_PIXEL_FORMAT_RAW_OPAQUE        // (ImageFormat#RAW_PRIVATE) so it doesn't count as opaque here.        ALOGE("Opaque format buffer is not lockable!");        return BAD_VALUE;    }    void *pData         = NULL;    android_ycbcr ycbcr = android_ycbcr();    status_t res;    int format          = buffer->getPixelFormat();    int flexFormat      = format;    if (isPossiblyYUV(format))    {   //从buff中读取yuv88 数据        res = buffer->lockAsyncYCbCr(inUsage, rect, &ycbcr, fenceFd);        pData = ycbcr.y;        flexFormat = HAL_PIXEL_FORMAT_YCbCr_420_888;    }    // lockAsyncYCbCr for YUV is unsuccessful.    if (pData == NULL)    {   //获取raw数据或者jpeg        res = buffer->lockAsync(inUsage, rect, &pData, fenceFd);    }    outputImage->data       = reinterpret_cast<uint8_t *>(pData);    outputImage->width      = buffer->getWidth();    outputImage->height     = buffer->getHeight();    outputImage->format     = format;    outputImage->flexFormat = flexFormat;    outputImage->stride     = (ycbcr.y != NULL) ? static_cast<uint32_t>(ycbcr.ystride) : buffer->getStride();    outputImage->dataCb         = reinterpret_cast<uint8_t *>(ycbcr.cb);    outputImage->dataCr         = reinterpret_cast<uint8_t *>(ycbcr.cr);    outputImage->chromaStride   = static_cast<uint32_t>(ycbcr.cstride);    outputImage->chromaStep     = static_cast<uint32_t>(ycbcr.chroma_step);    return OK;}

下边是dump方法，我们就不介绍了

void dumpImage(unsigned char* pY, unsigned char* pUV,        uint32_t frameid,uint32_t stride,uint32_t height) {    char buf[FILENAME_MAX];    memset(buf, 0, sizeof(buf));   snprintf(buf, sizeof(buf), CAMERA_DUMP_FRM_LOCATION "frame_%dx%d_%" PRIu64.nv12",        stride,height,frameid);   //snprintf(buf, sizeof(buf), CAMERA_DUMP_FRM_LOCATION "frame_%dx%d_%" PRIu64.nv21",        //stride,height,frameid);    ALOGE("%s,%d buf : %s", __FUNCTION__,__LINE__,buf);        int file_fd = open(buf, O_RDWR | O_CREAT, 0644);    uint32_t ySize = stride*height;    if (file_fd >= 0)    {        ssize_t written_yLen    = write(file_fd, pY, ySize);        ssize_t written_UVLen   = write(file_fd, pUV, ySize/2);    }}

至此完成了直接和CameraSerive通信进行开发进行Native Camera开发简单介绍。