新生代农民工的主页

简单了解 iOS CVPixelBuffer(下)

字数统计: 1.5k阅读时长: 7 min
2022/02/27

前言

「简单了解 iOS CVPixelBuffer (中)」中,我们了解了颜色空间RGBYUV的区别以及相关的背景知识,最后对CVPixelBuffer中的kCVPixelFormatType相关类型进行了解读。我们已经对CVPixelBuffer有了初步的了解,在这篇文章中,我们将继续聊聊CVPixelBuffer在使用过程中的一些格式转换;

RGBYUV格式转换

在很多场景下,我们需要将不同的颜色空间进行转换,以此来解决对应的工程性问题。
以下是转换公式:

YUV -> RGB

1
2
3
R = Y + 1.13983 * V
G = Y - 0.39465 * U - 0.58060 * V
B = Y + 2.03211 * U

RGB -> YUV

1
2
3
Y = 0.299 * R + 0.587 * G + 0.114 * B
U = -0.14713 * R - 0.28886 * G + 0.436 * B
V = 0.615 * R - 0.51499 * G - 0.10001 * B

iOS中常见格式转换

在iOS中RGBYUV互相转换的方法会使用到libyuv开源库,打开此链接需要梯子,目前国内也有,需要的自取libyuv开源库·国内仓库

iOS在CVPixelBuffer转换的上会很复杂,对buffer操作之前需要执行加锁方法CVPixelBufferLockBaseAddress进行保护,在处理完后,执行解锁buffer方法CVPixelBufferUnlockBaseAddress

以下的方法是我在项目中以及平常的开发中所整理,仅供参考;

NV12 to I420

核心NV12ToI420方法是使用了libyuv开源库

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
/// NV12 to I420
+ (CVPixelBufferRef)I420PixelBufferWithNV12:(CVImageBufferRef)cvpixelBufferRef {
    CVPixelBufferLockBaseAddress(cvpixelBufferRef, 0);
    //图像宽度(像素)
    size_t pixelWidth = CVPixelBufferGetWidth(cvpixelBufferRef);
    //图像高度(像素)
    size_t pixelHeight = CVPixelBufferGetHeight(cvpixelBufferRef);
    //获取CVPixelBufferRef中的y数据
    const uint8* y_frame = (uint8*)CVPixelBufferGetBaseAddressOfPlane(cvpixelBufferRef,0);
    //获取CMVImageBufferRef中的uv数据
    const uint8* uv_frame = (uint8*)CVPixelBufferGetBaseAddressOfPlane(cvpixelBufferRef,1);
    //y stride
    size_t plane1_stride = CVPixelBufferGetBytesPerRowOfPlane (cvpixelBufferRef, 0);
    //uv stride
    size_t plane2_stride = CVPixelBufferGetBytesPerRowOfPlane (cvpixelBufferRef, 1);
    //yuv_size(内存空间)
    size_t frame_size = pixelWidth*pixelHeight*3/2;
    //开辟frame_size大小的内存空间用于存放转换好的i420数据
    uint8* buffer = (unsigned char *)malloc(frame_size);
    //buffer为这段内存的首地址,plane1_size代表这一帧中y数据的长度
    uint8* dst_u = buffer + pixelWidth*pixelHeight;
    //dst_u为u数据的首地,plane1_size/4为u数据的长度
    uint8* dst_v = dst_u + pixelWidth*pixelHeight/4;
    //libyuv转换
    int ret = NV12ToI420(y_frame,
                        (int)plane1_stride,
                        uv_frame,
                        (int)plane2_stride,
                        buffer,
                        (int)pixelWidth,
                        dst_u,
                        (int)pixelWidth/2,
                        dst_v,
                        (int)pixelWidth/2,
                        (int)pixelWidth,
                        (int)pixelHeight
                        );
    if (ret) {
        return NULL;
    }
    NSDictionary *pixelAttributes = @{(id)kCVPixelBufferIOSurfacePropertiesKey : @{}};
    CVPixelBufferRef pixelBuffer = NULL;
    CVReturn result = CVPixelBufferCreate(kCFAllocatorDefault,
                                          pixelWidth,
                                          pixelHeight,
                                          kCVPixelFormatType_420YpCbCr8Planar,
                                          (__bridge CFDictionaryRef)(pixelAttributes),
                                          &pixelBuffer);
    CVPixelBufferLockBaseAddress(pixelBuffer, 0);
    size_t d = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
    size_t ud = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
    size_t vd = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 2);
    unsigned char* dsty = (unsigned char *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);
    unsigned char* dstu = (unsigned char *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);
    unsigned char* dstv = (unsigned char *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 2);
    unsigned char* srcy = buffer;
    for (unsigned int rIdx = 0; rIdx < pixelHeight; ++rIdx, srcy += pixelWidth, dsty += d) {
        memcpy(dsty, srcy, pixelWidth);
    }
    unsigned char* srcu = buffer + pixelHeight*pixelWidth;
    for (unsigned int rIdx = 0; rIdx < pixelHeight/2; ++rIdx, srcu += pixelWidth/2, dstu += ud) {
        memcpy(dstu, srcu, pixelWidth/2);
    }
    unsigned char* srcv = buffer + pixelHeight*pixelWidth*5/4;
    for (unsigned int rIdx = 0; rIdx < pixelHeight/2; ++rIdx, srcv += pixelWidth/2, dstv += vd) {
        memcpy(dstv, srcv, pixelWidth/2);
    }
    CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
    if (result != kCVReturnSuccess) {
        NSLog(@"Unable to create cvpixelbuffer %d", result);
    }
    free(buffer);
//    CVPixelBufferRelease(cvpixelBufferRef);
    return pixelBuffer;
}

NV12 to BGRA

核心NV12ToARGB方法同样使用了libyuv开源库

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
/// NV12 to BGRA
+ (CVPixelBufferRef)RGBAPixelBufferWithNV12:(CVImageBufferRef)pixelBufferNV12{
    CVPixelBufferLockBaseAddress(pixelBufferNV12, 0);
    //图像宽度(像素)
    size_t pixelWidth = CVPixelBufferGetWidth(pixelBufferNV12);
    //图像高度(像素)
    size_t pixelHeight = CVPixelBufferGetHeight(pixelBufferNV12);
    //y_stride
    size_t src_stride_y = CVPixelBufferGetBytesPerRowOfPlane(pixelBufferNV12, 0);
    //uv_stride
    size_t src_stride_uv = CVPixelBufferGetBytesPerRowOfPlane(pixelBufferNV12,1);
    //获取CVImageBufferRef中的y数据
    uint8_t *src_y = (unsigned char *)CVPixelBufferGetBaseAddressOfPlane(pixelBufferNV12, 0);
    //获取CMVImageBufferRef中的uv数据
    uint8_t *src_uv =(unsigned char *) CVPixelBufferGetBaseAddressOfPlane(pixelBufferNV12, 1);
    // 创建一个空的32BGRA格式的CVPixelBufferRef
    NSDictionary *pixelAttributes = @{(id)kCVPixelBufferIOSurfacePropertiesKey : @{}};
    CVPixelBufferRef pixelBufferRGBA = NULL;
    CVReturn result = CVPixelBufferCreate(kCFAllocatorDefault,
                                        pixelWidth,pixelHeight,kCVPixelFormatType_32BGRA,
                                          (__bridge CFDictionaryRef)pixelAttributes,&pixelBufferRGBA);//kCVPixelFormatType_32BGRA

    if (result != kCVReturnSuccess) {
        NSLog(@"Unable to create cvpixelbuffer %d", result);
        return NULL;
    }
    result = CVPixelBufferLockBaseAddress(pixelBufferRGBA, 0);
    if (result != kCVReturnSuccess) {
        CFRelease(pixelBufferRGBA);
        NSLog(@"Failed to lock base address: %d", result);
        return NULL;
    }
    // 得到新创建的CVPixelBufferRef中 rgb数据的首地址
    uint8_t *rgb_data = (uint8*)CVPixelBufferGetBaseAddress(pixelBufferRGBA);
    // 使用libyuv为rgb_data写入数据,将NV12转换为BGRA
    size_t bgraStride = CVPixelBufferGetBytesPerRowOfPlane(pixelBufferRGBA,0);
    int ret = NV12ToARGB(src_y, (int)src_stride_y, src_uv, (int)src_stride_uv, rgb_data,(int)bgraStride, (int)pixelWidth, (int)pixelHeight);
    if (ret) {
        NSLog(@"Error converting NV12 VideoFrame to BGRA: %d", result);
        CFRelease(pixelBufferRGBA);
        return NULL;
    }
    CVPixelBufferUnlockBaseAddress(pixelBufferRGBA, 0);
    CVPixelBufferUnlockBaseAddress(pixelBufferNV12, 0);
    return pixelBufferRGBA;
}

CVPixelBufferRef to UIImage

以下方法可以将视频帧转成单张图片(比较适用于间隔时间长的截图,高频的使用这个方法很可能会引起内存的问题)

1
2
3
4
5
6
7
8
9
10
11
/// buffer to image
+ (UIImage *)convert:(CVPixelBufferRef)pixelBuffer {
    CIImage *ciImage = [CIImage imageWithCVPixelBuffer:pixelBuffer];
    CIContext *temporaryContext = [CIContext contextWithOptions:nil];
    CGImageRef videoImage = [temporaryContext createCGImage:ciImage
             fromRect:CGRectMake(0, 0, CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer))];
    UIImage *uiImage = [UIImage imageWithCGImage:videoImage];
    CGImageRelease(videoImage);
    return uiImage;
}

CGImageRef to CVPixelBufferRef

以下方法会通过单张图片转成一个PixelBuffer (适用于将某一帧图片转成Buffer添加字幕或者美颜贴纸等等)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
/// image to buffer
+ (CVPixelBufferRef)pixelBufferFromCGImage:(CGImageRef)image {
    NSDictionary *options = @{
                              (NSString*)kCVPixelBufferCGImageCompatibilityKey : @YES,
                              (NSString*)kCVPixelBufferCGBitmapContextCompatibilityKey : @YES,
                              (NSString*)kCVPixelBufferIOSurfacePropertiesKey: [NSDictionary dictionary]
                              };
    CVPixelBufferRef pxbuffer = NULL;
    CGFloat frameWidth = CGImageGetWidth(image);
    CGFloat frameHeight = CGImageGetHeight(image);
    CVReturn status = CVPixelBufferCreate(kCFAllocatorDefault,
                                          frameWidth,
                                          frameHeight,
                                          kCVPixelFormatType_32BGRA,
                                          (__bridge CFDictionaryRef) options,
                                          &pxbuffer);
    NSParameterAssert(status == kCVReturnSuccess && pxbuffer != NULL);
    CVPixelBufferLockBaseAddress(pxbuffer, 0);
    void *pxdata = CVPixelBufferGetBaseAddress(pxbuffer);
    NSParameterAssert(pxdata != NULL);
    CGColorSpaceRef rgbColorSpace = CGColorSpaceCreateDeviceRGB();
    CGContextRef context = CGBitmapContextCreate(pxdata,
                                                 frameWidth,
                                                 frameHeight,
                                                 8,
                                                 CVPixelBufferGetBytesPerRow(pxbuffer),
                                                 rgbColorSpace,
(CGBitmapInfo)kCGImageAlphaNoneSkipFirst);
    NSParameterAssert(context);
    CGContextConcatCTM(context, CGAffineTransformIdentity);
    CGContextDrawImage(context, CGRectMake(0,
                                           0,
                                           frameWidth,
                                           frameHeight),
                       image);
    CGColorSpaceRelease(rgbColorSpace);
    CGContextRelease(context);
    CVPixelBufferUnlockBaseAddress(pxbuffer, 0);
    return pxbuffer;
}

Buffer Data to UIImage

以下方法会通过内存数据转成图片 (根据内存的地址去取出存储的buffer并生成图片,其实这里的内存的地址指向的就是Buffer)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
// NV12 to image
+ (UIImage *)YUVtoUIImage:(int)w h:(int)h buffer:(unsigned char *)buffer {
    //YUV(NV12)-->CIImage--->UIImage Conversion
    NSDictionary *pixelAttributes = @{(NSString*)kCVPixelBufferIOSurfacePropertiesKey:@{}};
    CVPixelBufferRef pixelBuffer = NULL;
    CVReturn result = CVPixelBufferCreate(kCFAllocatorDefault,
                                          w,
                                          h,
                                          kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange,
                                          (__bridge CFDictionaryRef)(pixelAttributes),
                                          &pixelBuffer);
    CVPixelBufferLockBaseAddress(pixelBuffer,0);
    void *yDestPlane = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);
    // Here y_ch0 is Y-Plane of YUV(NV12) data.
    unsigned char *y_ch0 = buffer;
    unsigned char *y_ch1 = buffer + w * h;
    memcpy(yDestPlane, y_ch0, w * h);
    void *uvDestPlane = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);
    // Here y_ch1 is UV-Plane of YUV(NV12) data.
    memcpy(uvDestPlane, y_ch1, w * h * 0.5);
    CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
    if (result != kCVReturnSuccess) {
        NSLog(@"Unable to create cvpixelbuffer %d", result);
    }
    // CIImage Conversion
    if (@available(iOS 13.0, *)) {
        CIImage *coreImage = [CIImage imageWithCVPixelBuffer:pixelBuffer];
        CIContext *temporaryContext = [CIContext contextWithOptions:nil];
        CGImageRef videoImage = [temporaryContext createCGImage:coreImage
                                                           fromRect:CGRectMake(0, 0, w, h)];
       
        UIImage *finalImage = [[UIImage alloc] initWithCGImage:videoImage];
        CVPixelBufferRelease(pixelBuffer);
        CGImageRelease(videoImage);
        return finalImage;
    }
    return nil;
};

iOS中格式转换涉及到C、OC、C++的一些方法,所以很多方法看起来会非常的冗余,需要一定的基础和持续的学习。还有很多是通过OpenGL来绘制图像的方法,更是难得看懂,所以初学者做好笔记,保持耐心,把常用的方法梳理起来,最后封装到工具类中。等到机缘巧合的时候再来深入了解。

参考文献

一文读懂 YUV 的采样与格式

CATALOG
  1. 1. 前言
  2. 2. RGB和YUV格式转换
    1. 2.1. YUV -> RGB
    2. 2.2. RGB -> YUV
  3. 3. iOS中常见格式转换
    1. 3.1. NV12 to I420
    2. 3.2. NV12 to BGRA
    3. 3.3. CVPixelBufferRef to UIImage
    4. 3.4. CGImageRef to CVPixelBufferRef
    5. 3.5. Buffer Data to UIImage
    6. 3.6. 参考文献