Pixel conversion in decompiled assembly from Ghidra - c

Question

I'm reversing an assembly function that I believe is converting an array of pixels (RGB) or photosites (RGGB) into a final array with an original (I think) encoding made of 10bits or 12bits per pixel. My goal is to understand in terms of c/c++ operations or natural language in the context of image format conversion what is happening, and in the end name in a meaningful way the variables.
There are few operation that I do not understand such as:
In the case of 10 bits per sample (1 sample ie., R|G|B):
uVar7 = (int64)iVar3 / 3 + ((int64)iVar3 >> 0x3f) & 0xffffffff; // 3f = 63
int iVar4 = (int)uVar7 + (int)(uVar7 >> 0x1f); // 1f = 31
puVar1 = (uint*)((int64)outData + (int64)iVar4 * 4);
*puVar1 = *puVar1 | (*inPixel & 0x3ff) <<
(((char)iVar3 + (char)iVar4 * -3) * '\n' + 2U & 0x1f);
The first two lines are considering the strongest bit (>> 0x3f and >> 0x1f); but why divide by 3 ?
why add the strongest bit ? I don't get the 4th line at all; * -3 * '\n' in particular, and 2U & 0x1f: why 31 as mask ?
and in the case of 12 bits per samples:
uint uVar8 = *inPixel & 0xfff; // considering an array of 3 bits (likely 4 times at some point)
uint uVar5 = surfaceSizeBits + position_ >> 0x1f & 7; // 7 = b111
& 7 is zeroing all but the 3 lowests bits, but why after considering the strongest bit just before ?
dataSizeBytes_ = surfaceSizeBits + position_ + uVar5;
iVar3 = ((dataSizeBytes_ & 7) - uVar5) * 12;
I don't get the purpose of - uVar5 and then *12; it's possible that dataSizeBytes is not data size in bytes here.
uint uVar6 = iVar3 >> 0x1f & 0x1f;
uVar5 = iVar3 + uVar6;
iVar3 = (uVar5 & 0x1f) - uVar6;
puVar1 = (uint*)((int64)outData +
((int64)((int)dataSizeBytes_ >> 3) * 3 +
(int64)((int)uVar5 >> 5)) * 4);
>>3 is "dropping the 2 weakest bits" but that's all I understand here.
byte bVar2 = (byte)iVar3;
*puVar1 = *puVar1 | uVar8 << (bVar2 & 0x1f);
if (0x14 < iVar3) // 0x14 = 20
puVar1[1] = puVar1[1] | uVar8 >> (0x20 - bVar2 & 0x1f); //0x20 = 32
Here I understand the bit operations but I can't move to a higher level.
End of 12 bits case.
For the decompiled code, I provided those definitions in order to help understanding:
typedef unsigned int uint;
typedef unsigned long long uint64;
typedef long long int64;
typedef int undefined; //may be wrong, may be 1 byte (int4) instead
typedef long int32;
typedef long undefined8;
typedef int int16;
typedef int int16;
typedef short int8;
typedef unsigned short uint8;
typedef unsigned short ushort;
typedef long code;
char LogBuffer[256];
The undefined* are from ghidra, I did not check yet in dynamic analysis if my translation is correct.
The decompiled code from ghidra is:
(modulo few renamings and type definitions)
undefined8
PixelConversionInOut
(undefined8 param_1, void* inData, void* outData, int height, int widthStride,
int bitsPerSample, int dstBufferSize, uint* bufferResult)
{
uint* puVar1;
int surfaceSizeBits;
undefined8 status;
int iVar3;
uint64 uVar7;
int64 currentPosition;
uint dataSizeBytes_;
GetCurrentThreadId();
if (inData != (void*)nullptr && outData != (void*)nullptr)
{
int heightPadding = 0;
*bufferResult = 0;
if (bitsPerSample == 10)
{
if ((height == 0x818) || (height == 0x1010))
heightPadding = 4;
else if (height == 0x1018)
heightPadding = 2;
}
int heightPadded = heightPadding + height;
if (bitsPerSample == 10)
{
currentPosition = (int64)(heightPadded * widthStride * 4);
uVar7 = currentPosition / 3 + (currentPosition >> 0x3f) & 0xffffffff;
*bufferResult = (int)uVar7 + (int)(uVar7 >> 0x1f);
}
else
{
if (bitsPerSample == 0xc)
{
surfaceSizeBits = heightPadded * widthStride * 0xc;
dataSizeBytes_ = (int)(surfaceSizeBits + (surfaceSizeBits >> 0x1f & 7U)) >> 3;
}
else
dataSizeBytes_ = heightPadded * widthStride * 4 / 2;
*bufferResult = dataSizeBytes_;
}
dataSizeBytes_ = *bufferResult;
if (dstBufferSize < (int)dataSizeBytes_)
{
sprintf(LogBuffer, "dstBuffer must be allocated at least %d bytes.", (uint64)dataSizeBytes_);
status = 0x80000003;
}
else
{
if (bitsPerSample == 0x10)
{
memcpy(outData, inData, (int64)(int)dataSizeBytes_);
GetCurrentThreadId();
status = 0;
}
else
{
memset(outData, 0, (int64)(int)dataSizeBytes_);
currentPosition = 0;
if (0 < widthStride)
{
surfaceSizeBits = 0;
do
{
int position_ = 0;
if (0 < (int64)(heightPadded - heightPadding))
{
ushort* inPixel = (ushort*)((int64)inData + height * currentPosition * 2);
int64 rowsRemaining = (int64)(heightPadded - heightPadding);
do
{
if (bitsPerSample == 10)
{
iVar3 = surfaceSizeBits + position_;
uVar7 = (int64)iVar3 / 3 + ((int64)iVar3 >> 0x3f) & 0xffffffff;
int iVar4 = (int)uVar7 + (int)(uVar7 >> 0x1f);
puVar1 = (uint*)((int64)outData + (int64)iVar4 * 4);
*puVar1 = *puVar1 | (*inPixel & 0x3ff) <<
(((char)iVar3 + (char)iVar4 * -3) * '\n' + 2U & 0x1f);
}
else
{
if (bitsPerSample == 12)
{
uint uVar8 = *inPixel & 0xfff;
uint uVar5 = surfaceSizeBits + position_ >> 0x1f & 7;
dataSizeBytes_ = surfaceSizeBits + position_ + uVar5;
iVar3 = ((dataSizeBytes_ & 7) - uVar5) * 12;
uint uVar6 = iVar3 >> 0x1f & 0x1f;
uVar5 = iVar3 + uVar6;
iVar3 = (uVar5 & 0x1f) - uVar6;
puVar1 = (uint*)((int64)outData +
((int64)((int)dataSizeBytes_ >> 3) * 3 +
(int64)((int)uVar5 >> 5)) * 4);
byte bVar2 = (byte)iVar3;
*puVar1 = *puVar1 | uVar8 << (bVar2 & 0x1f);
if (0x14 < iVar3)
puVar1[1] = puVar1[1] | uVar8 >> (0x20 - bVar2 & 0x1f);
}
}
position_ = position_ + 1;
inPixel = inPixel + 1;
rowsRemaining = rowsRemaining + -1;
}
while (rowsRemaining != 0);
}
currentPosition = currentPosition + 1;
surfaceSizeBits = surfaceSizeBits + heightPadded;
}
while (currentPosition < widthStride);
}
GetCurrentThreadId();
status = 0;
}
}
return status;
}
return 0x80000023;
}
Here is the original assembly but ghidra did not complain or warn about the result.
**************************************************************
* FUNCTION *
**************************************************************
undefined8 __fastcall PixelConversionInOut(undefined8 pa
undefined8 RAX:8 <RETURN> XREF[1]: 180037b9c(W)
undefined8 RCX:8 param_1
void * RDX:8 inData
void * R8:8 outData
int R9D:4 height
int Stack[0x28]:4 widthStride XREF[1]: 180037b66(R)
int Stack[0x30]:4 bitsPerSample XREF[1]: 180037b26(R)
int Stack[0x38]:4 dstBufferSize XREF[1]: 180037bb8(R)
uint * Stack[0x40]:8 bufferResult XREF[1]: 180037b19(R)
undefined4 EDI:4 heightPadding XREF[2]: 180037b62(W),
180037d1e(W)
undefined4 EAX:4 surfaceSizeBits XREF[2]: 180037b9c(W),
180037bd7(W)
undefined8 RAX:8 status XREF[1]: 180037bd7(W)
undefined8 R10:8 inPixel XREF[1]: 180037c71(W)
undefined4 R11D:4 position_ XREF[1]: 180037d17(W)
undefined8 RDI:8 rowsRemaining XREF[1]: 180037d1e(W)
undefined8 RBP:8 currentPosition XREF[1]: 180037d2c(W)
undefined4 Stack[0x20]:4 local_res20 XREF[2]: 180037ae0(W),
180037c5c(R)
undefined8 Stack[0x18]:8 local_res18 XREF[2]: 180037b21(W),
180037be1(R)
undefined8 Stack[0x10]:8 local_res10 XREF[2]: 180037ae5(W),
180037c61(R)
undefined8 Stack[0x8]:8 local_res8 XREF[4]: 180037aea(W),
180037c23(W),
180037c40(R),
180037d27(R)
undefined8 Stack[-0x20]:8 local_20 XREF[2]: 180037c14(W),
180037d40(R)
undefined8 Stack[-0x28]:8 local_28 XREF[2]: 180037b2d(W),
180037bdc(R)
undefined8 Stack[-0x30]:8 local_30 XREF[2]: 180037b34(W),
180037be6(R)
undefined8 Stack[-0x38]:8 local_38 XREF[2]: 180037c33(W),
180037d3b(R)
undefined4 HASH:27f0811 heightPadded
undefined4 HASH:5fcc647 dataSizeBytes_
PixelConversionInOut XREF[2]: GetDecompressedData:1800384fa(c),
FUN_180038630:18003875a(c)
180037ae0 44 89 4c 24 20 MOV dword ptr [RSP + local_res20],height
180037ae5 48 89 54 24 10 MOV qword ptr [RSP + local_res10],inData
180037aea 48 89 4c 24 08 MOV qword ptr [RSP + local_res8],param_1
180037aef 56 PUSH RSI
180037af0 41 56 PUSH R14
180037af2 41 57 PUSH R15
180037af4 48 83 ec 40 SUB RSP,0x40
180037af8 41 8b f1 MOV ESI,height
180037afb 4d 8b f8 MOV R15,outData
180037afe 4c 8b f2 MOV R14,inData
180037b01 ff 15 29 7d CALL qword ptr [->KERNEL32.DLL::GetCurrentThreadId]
9d 00
180037b07 4d 85 f6 TEST R14,R14
180037b0a 0f 84 42 02 JZ LAB_180037d52
00 00
180037b10 4d 85 ff TEST R15,R15
180037b13 0f 84 39 02 JZ LAB_180037d52
00 00
180037b19 4c 8b 94 24 MOV R10,qword ptr [RSP + bufferResult]
98 00 00 00
180037b21 48 89 5c 24 70 MOV qword ptr [RSP + local_res18],RBX
180037b26 8b 9c 24 88 MOV EBX,dword ptr [RSP + bitsPerSample]
00 00 00
180037b2d 48 89 7c 24 30 MOV qword ptr [RSP + local_28],RDI
180037b32 33 ff XOR EDI,EDI
180037b34 4c 89 64 24 28 MOV qword ptr [RSP + local_30],R12
180037b39 41 89 3a MOV dword ptr [R10],EDI
180037b3c 83 fb 0a CMP EBX,0xa
180037b3f 75 21 JNZ LAB_180037b62
180037b41 8b ce MOV param_1,ESI
180037b43 81 e9 18 08 SUB param_1,0x818
00 00
180037b49 74 12 JZ LAB_180037b5d
180037b4b 81 e9 f8 07 SUB param_1,0x7f8
00 00
180037b51 74 0a JZ LAB_180037b5d
180037b53 83 f9 08 CMP param_1,0x8
180037b56 75 0a JNZ LAB_180037b62
180037b58 8d 7b f8 LEA EDI,[RBX + -0x8]
180037b5b eb 05 JMP LAB_180037b62
LAB_180037b5d XREF[2]: 180037b49(j), 180037b51(j)
180037b5d bf 04 00 00 00 MOV EDI,0x4
LAB_180037b62 XREF[3]: 180037b3f(j), 180037b56(j),
180037b5b(j)
180037b62 44 8d 24 37 LEA R12D,[heightPadding + RSI*0x1]
180037b66 8b b4 24 80 MOV ESI,dword ptr [RSP + widthStride]
00 00 00
180037b6d 83 fb 0a CMP EBX,0xa
180037b70 75 1c JNZ LAB_180037b8e
180037b72 41 8b cc MOV param_1,R12D
180037b75 b8 56 55 55 55 MOV EAX,0x55555556
180037b7a 0f af ce IMUL param_1,ESI
180037b7d c1 e1 02 SHL param_1,0x2
180037b80 f7 e9 IMUL param_1
180037b82 8b c2 MOV EAX,inData
180037b84 c1 e8 1f SHR EAX,0x1f
180037b87 03 d0 ADD inData,EAX
180037b89 41 89 12 MOV dword ptr [R10],inData
180037b8c eb 27 JMP LAB_180037bb5
LAB_180037b8e XREF[1]: 180037b70(j)
180037b8e 41 8b c4 MOV EAX,R12D
180037b91 0f af c6 IMUL EAX,ESI
180037b94 83 fb 0c CMP EBX,0xc
180037b97 75 11 JNZ LAB_180037baa
180037b99 8d 04 40 LEA EAX,[RAX + RAX*0x2]
180037b9c c1 e0 02 SHL surfaceSizeBits,0x2
180037b9f 99 CDQ
180037ba0 83 e2 07 AND inData,0x7
180037ba3 03 c2 ADD surfaceSizeBits,inData
180037ba5 c1 f8 03 SAR surfaceSizeBits,0x3
180037ba8 eb 08 JMP LAB_180037bb2
LAB_180037baa XREF[1]: 180037b97(j)
180037baa c1 e0 02 SHL surfaceSizeBits,0x2
180037bad 99 CDQ
180037bae 2b c2 SUB surfaceSizeBits,inData
180037bb0 d1 f8 SAR surfaceSizeBits,1
LAB_180037bb2 XREF[1]: 180037ba8(j)
180037bb2 41 89 02 MOV dword ptr [R10],surfaceSizeBits
LAB_180037bb5 XREF[1]: 180037b8c(j)
180037bb5 49 63 02 MOVSXD surfaceSizeBits,dword ptr [R10]
180037bb8 3b 84 24 90 CMP surfaceSizeBits,dword ptr [RSP + dstBufferSize]
00 00 00
180037bbf 7e 34 JLE LAB_180037bf5
180037bc1 48 8d 15 30 LEA inData,[s_dstBuffer_must_be_allocated_at_l_180b675f8] = "dstBuffer must be allocated at least %d bytes."
fa b2 00
180037bc8 48 8d 0d 81 LEA param_1,[LogBuffer] = ??
db bc 00
180037bcf 44 8b c0 MOV outData,surfaceSizeBits
180037bd2 e8 bd 28 1c 00 CALL sprintf int sprintf(char * _Dest, char *
180037bd7 b8 03 00 00 80 MOV status,0x80000003
LAB_180037bdc XREF[2]: 180037c10(j), 180037d4d(j)
180037bdc 48 8b 7c 24 30 MOV heightPadding,qword ptr [RSP + local_28]
180037be1 48 8b 5c 24 70 MOV RBX,qword ptr [RSP + local_res18]
180037be6 4c 8b 64 24 28 MOV R12,qword ptr [RSP + local_30]
180037beb 48 83 c4 40 ADD RSP,0x40
180037bef 41 5f POP R15
180037bf1 41 5e POP R14
180037bf3 5e POP RSI
180037bf4 c3 RET
LAB_180037bf5 XREF[1]: 180037bbf(j)
180037bf5 4c 8b c0 MOV outData,status
180037bf8 49 8b cf MOV param_1,R15
180037bfb 83 fb 10 CMP EBX,0x10
180037bfe 75 12 JNZ LAB_180037c12
180037c00 49 8b d6 MOV inData,R14
180037c03 e8 f8 d9 1b 00 CALL memcpy void * memcpy(void * _Dst, void
180037c08 ff 15 22 7c CALL qword ptr [->KERNEL32.DLL::GetCurrentThreadId]
9d 00
180037c0e 33 c0 XOR status,status
180037c10 eb ca JMP LAB_180037bdc
LAB_180037c12 XREF[1]: 180037bfe(j)
180037c12 33 d2 XOR inData,inData
180037c14 48 89 6c 24 38 MOV qword ptr [RSP + local_20],RBP
180037c19 e8 d2 17 1c 00 CALL memset void * memset(void * _Dst, int _
180037c1e 48 63 c6 MOVSXD status,ESI
180037c21 33 ed XOR EBP,EBP
180037c23 48 89 44 24 60 MOV qword ptr [RSP + local_res8],status
180037c28 85 f6 TEST ESI,ESI
180037c2a 0f 8e 10 01 JLE LAB_180037d40
00 00
180037c30 41 8b c4 MOV status,R12D
180037c33 4c 89 6c 24 20 MOV qword ptr [RSP + local_38],R13
180037c38 2b c7 SUB status,heightPadding
180037c3a 45 33 f6 XOR R14D,R14D
180037c3d 4c 63 e8 MOVSXD R13,status
180037c40 48 8b 44 24 60 MOV status,qword ptr [RSP + local_res8]
180037c45 66 66 66 0f NOP word ptr [RAX + RAX*0x1]
1f 84 00 00
00 00 00
LAB_180037c50 XREF[1]: 180037d35(j)
180037c50 45 33 db XOR R11D,R11D
180037c53 4d 85 ed TEST R13,R13
180037c56 0f 8e d0 00 JLE LAB_180037d2c
00 00
180037c5c 48 63 44 24 78 MOVSXD status,dword ptr [RSP + local_res20]
180037c61 48 8b 4c 24 68 MOV param_1,qword ptr [RSP + local_res10]
180037c66 49 8b fd MOV heightPadding,R13
180037c69 48 0f af c5 IMUL status,RBP
180037c6d 4c 8d 14 41 LEA R10,[param_1 + status*0x2]
LAB_180037c71 XREF[1]: 180037d21(j)
180037c71 45 0f b7 02 MOVZX outData,word ptr [inPixel]
180037c75 83 fb 0a CMP EBX,0xa
180037c78 75 37 JNZ LAB_180037cb1
180037c7a 43 8d 0c 1e LEA param_1,[R14 + R11*0x1]
180037c7e b8 56 55 55 55 MOV status,0x55555556
180037c83 41 81 e0 ff AND outData,0x3ff
03 00 00
180037c8a f7 e9 IMUL param_1
180037c8c 8b c2 MOV status,inData
180037c8e c1 e8 1f SHR status,0x1f
180037c91 03 d0 ADD inData,status
180037c93 48 63 c2 MOVSXD status,inData
180037c96 4d 8d 0c 87 LEA height,[R15 + status*0x4]
180037c9a 8d 04 52 LEA status,[RDX + RDX*0x2]
180037c9d 2b c8 SUB param_1,status
180037c9f 8d 0c 89 LEA param_1,[RCX + RCX*0x4]
180037ca2 8d 0c 4d 02 LEA param_1,[0x2 + param_1*0x2]
00 00 00
180037ca9 41 d3 e0 SHL outData,param_1
180037cac 45 09 01 OR dword ptr [height],outData
180037caf eb 66 JMP LAB_180037d17
LAB_180037cb1 XREF[1]: 180037c78(j)
180037cb1 83 fb 0c CMP EBX,0xc
180037cb4 75 61 JNZ LAB_180037d17
180037cb6 43 8d 04 1e LEA status,[R14 + R11*0x1]
180037cba 41 81 e0 ff AND outData,0xfff
0f 00 00
180037cc1 99 CDQ
180037cc2 83 e2 07 AND inData,0x7
180037cc5 03 c2 ADD status,inData
180037cc7 8b c8 MOV param_1,status
180037cc9 83 e0 07 AND status,0x7
180037ccc 2b c2 SUB status,inData
180037cce c1 f9 03 SAR param_1,0x3
180037cd1 8d 04 40 LEA status,[RAX + RAX*0x2]
180037cd4 48 63 c9 MOVSXD param_1,param_1
180037cd7 c1 e0 02 SHL status,0x2
180037cda 99 CDQ
180037cdb 83 e2 1f AND inData,0x1f
180037cde 03 c2 ADD status,inData
180037ce0 44 8b c8 MOV height,status
180037ce3 83 e0 1f AND status,0x1f
180037ce6 2b c2 SUB status,inData
180037ce8 41 c1 f9 05 SAR height,0x5
180037cec 48 8d 14 49 LEA inData,[RCX + RCX*0x2]
180037cf0 49 63 c9 MOVSXD param_1,height
180037cf3 48 03 d1 ADD inData,param_1
180037cf6 8b c8 MOV param_1,status
180037cf8 4d 8d 0c 97 LEA height,[R15 + inData*0x4]
180037cfc 41 8b d0 MOV inData,outData
180037cff d3 e2 SHL inData,param_1
180037d01 41 09 11 OR dword ptr [height],inData
180037d04 83 f8 14 CMP status,0x14
180037d07 7e 0e JLE LAB_180037d17
180037d09 b9 20 00 00 00 MOV param_1,0x20
180037d0e 2b c8 SUB param_1,status
180037d10 41 d3 e8 SHR outData,param_1
180037d13 45 09 41 04 OR dword ptr [height + 0x4],outData
LAB_180037d17 XREF[3]: 180037caf(j), 180037cb4(j),
180037d07(j)
180037d17 41 ff c3 INC position_
180037d1a 49 83 c2 02 ADD inPixel,0x2
180037d1e 48 ff cf DEC rowsRemaining
180037d21 0f 85 4a ff JNZ LAB_180037c71
ff ff
180037d27 48 8b 44 24 60 MOV status,qword ptr [RSP + local_res8]
LAB_180037d2c XREF[1]: 180037c56(j)
180037d2c 48 ff c5 INC currentPosition
180037d2f 45 03 f4 ADD R14D,R12D
180037d32 48 3b e8 CMP currentPosition,status
180037d35 0f 8c 15 ff JL LAB_180037c50
ff ff
180037d3b 4c 8b 6c 24 20 MOV R13,qword ptr [RSP + local_38]
LAB_180037d40 XREF[1]: 180037c2a(j)
180037d40 48 8b 6c 24 38 MOV currentPosition,qword ptr [RSP + local_20]
180037d45 ff 15 e5 7a CALL qword ptr [->KERNEL32.DLL::GetCurrentThreadId]
9d 00
180037d4b 33 c0 XOR status,status
180037d4d e9 8a fe ff ff JMP LAB_180037bdc
LAB_180037d52 XREF[2]: 180037b0a(j), 180037b13(j)
180037d52 b8 23 00 00 80 MOV status,0x80000023
180037d57 48 83 c4 40 ADD RSP,0x40
180037d5b 41 5f POP R15
180037d5d 41 5e POP R14
180037d5f 5e POP RSI
180037d60 c3 RET
180037d61 cc cc cc cc align align(15)
cc cc cc cc
cc cc cc cc
I may add that it's an AMD64/Intelx64 64bits assembly for Windows 10.