How to copy an OleVariant array efficiently to my own structure? - arrays

Question

I am trying to copy an OleVariant array to my own structure. I received the OleVariant from an external COM call.
The size is 1000 x 500 elements (I don't know if its the same as this Structure Definition: array of array of OleVariant).
Currently, I am trying to achieve something like:
result := Copy(Source, Amount)
But the OleVariant structure is in my way.
If I use a "classical" loop, it works, but it is slow (very slow).
aResult is currently defined as TData = array of array of string;
procedure CopyResult(aResultCount: Integer; var aResult: TData; aSource: Variant);
var
i, j: Integer;
bVariantConversion: boolean;
begin
SetLength(aResult, aResultCount, VarArrayHighBound(aSource[0], 1));
bVariantConversion := NullStrictConvert; // settings to manage how string conversion for Variant is handled.
NullStrictConvert := False;
try
for i := VarArrayLowBound(aSource, 1) to VarArrayHighBound(aSource, 1) do
begin
for j := VarArrayLowBound(aSource[i], 1) to pred(VarArrayHighBound(aSource[i], 1)) do
begin
//nearly every execution pause is somewhere in this String Conversion or Array Function.
aResult[i][j] := aSource[i][j]; //implicit conversion to string ...
end;
end;
finally
NullStrictConvert := bVariantConversion;
end;
end;
As #Remy Lebau mentioned the bounds Check for the Vararray[x][y] access is the routine my source burns its time. I am trying to eliminate this kind of acces by going directly to the OleVariantArray Elements.
Aftermath...
trying to Determine my Structure i think i found the Root.
tmyVarType := VarType(aSource); //8204 => Array(VT_ARRAY = 0x2000 = 8192) + variant(VT_VARIANT = 0x000C = 12)
tmyVarType := VarType(aSource[0]); //8204
tmyVarType := VarType(aSource[0][0]); //3 VT_I4 = 0x0003 = 3 is integer and this is correctly changin for the fields.
So i try to acess the Source without the build in functions to avoid the bounds check.

Answer 1

The biggest bottleneck in this code is the bounds checking performed by the [] operator on each Variant array, and potentially on your aResult array, too. Since you are already handling the bounds in each loop, there is no need to verify the bounds inside of the loops as well.
So, if performance is an issue for you, then you can use VarArrayLock() to access the underlying Variant elements in each array, using pointer arithmetic to move between them, eliminating those redundant bounds checks.
You should also reduce the redundant calls to VarArray(Low|High)Bound(aSource[i], 1) on each iteration of the outer array, since you claim the inner arrays all have the same length. So you can calculate that up front before entering the loops.
Try something like this:
type
TStrArr = array of string;
PStrArr = ^TStrArr;
TData = array of TStrArr;
procedure CopyResult(aResultCount: Integer; var aResult: TData; aSource: Variant);
var
i, j,
OuterLBound, OuterHBound, OuterCount,
InnerLBound, InnerHBound, InnerCount: Integer;
pOuterVarArr, pInnerVarArr: PVariant;
pOuterDynArr: PStrArr;
pInnerDynArr: PString;
bVariantConversion: boolean;
begin
aResult := nil;
Assert(VarIsType(aSource, varArray or varVariant));
Assert(VarArrayDimCount(aSource) = 1);
OuterLBound := VarArrayLowBound(aSource, 1);
OuterHBound := VarArrayHighBound(aSource, 1);
OuterCount := {aResultCount} OuterHBound - OuterLBound + 1;
if OuterCount < 1 then Exit;
Assert(VarIsType(aSource[0], varArray or varVariant));
Assert(VarArrayDimCount(aSource[0]) = 1);
InnerLBound := VarArrayLowBound(aSource[0], 1);
InnerHBound := VarArrayHighBound(aSource[0], 1);
InnerCount := InnerHBound - InnerLBound + 1;
SetLength(aResult, {aResultCount} OuterCount, InnerCount);
bVariantConversion := NullStrictConvert; // settings to manage how string conversion for Variant is handled.
NullStrictConvert := False;
try
pOuterDynArr := PStrArr(aResult);
pOuterVarArr := PVariant(VarArrayLock(aSource));
try
for i := OuterLBound to OuterHBound do
begin
pInnerDynArr := PString(pOuterDynArr^);
pInnerVarArr := PVariant(VarArrayLock(pOuterVarArr^));
try
//System.Variants.DynArrayFromVariant(pOuterDynArr^, pInnerVarArr^, TypeInfo(String));
for j := InnerLBound to InnerHBound do
begin
pInnerDynArr^ := pInnerVarArr^; //implicit conversion to string ...
Inc(pInnerDynArr);
Inc(pInnerVarArr);
end;
finally
VarArrayUnlock(pOuterVarArr^);
end;
Inc(pOuterDynArr);
Inc(pOuterVarArr);
end;
finally
VarArrayUnlock(aSource);
end;
finally
NullStrictConvert := bVariantConversion;
end;
end;
On the other hand, if there is ever a chance that the inner arrays have different lengths, then you can try this adjustment instead:
type
TStrArr = array of string;
PStrArr = ^TStrArr;
TData = array of TStrArr;
procedure CopyResult(aResultCount: Integer; var aResult: TData; aSource: Variant);
var
i, j,
OuterLBound, OuterHBound, OuterCount,
InnerLBound, InnerHBound, InnerCount: Integer;
pOuterVarArr, pInnerVarArr: PVariant;
pOuterDynArr: PStrArr;
pInnerDynArry: PString;
bVariantConversion: boolean;
begin
aResult := nil;
Assert(VarIsType(aSource, varArray or varVariant);
Assert(VarArrayDimCount(aSource) = 1);
OuterLBound := VarArrayLowBound(aSource, 1);
OuterHBound := VarArrayHighBound(aSource, 1);
OuterCount := {aResultCount} OuterHBound - OuterLBound + 1;
if OuterCount < 1 then Exit;
SetLength(aResult, {aResultCount} OuterCount);
bVariantConversion := NullStrictConvert; // settings to manage how string conversion for Variant is handled.
NullStrictConvert := False;
try
pOuterDynArr := PStrArr(aResult);
pOuterVarArr := PVariant(VarArrayLock(aSource));
try
for i := OuterLBound to OuterHBound do
begin
pInnerVarArr := PVariant(VarArrayLock(pOuterVarArr^));
try
//System.Variants.DynArrayFromVariant(pOuterDynArr^, pInnerVarArr^, TypeInfo(String));
Assert(VarIsType(pInnerVarArr^, varArray or varVariant);
Assert(VarArrayDimCount(pInnerVarArr^) = 1);
InnerLBound := VarArrayLowBound(pInnerVarArr^, 1);
InnerHBound := VarArrayHighBound(pInnerVarArr^, 1);
InnerCount := InnerHBound - InnerLBound + 1;
SetLength(pOuterDynArr^, InnerCount);
pInnerDynArr := PString(pOuterDynArr^);
for j := InnerLBound to InnerHBound do
begin
pInnerDynArr^ := pInnerVarArr^; //implicit conversion to string ...
Inc(pInnerDynArr);
Inc(pInnerVarArr);
end;
finally
VarArrayUnlock(pOuterVarArr^);
end;
Inc(pOuterDynArr);
Inc(pOuterVarArr);
end;
finally
VarArrayUnlock(aSource);
end;
finally
NullStrictConvert := bVariantConversion;
end;
end;
Edit: I Only tested the Source version for all Entrys the same length but it works my own partial [] free Version used ~5 Million cycles with Tstopwatch ElapsedTicks and this one only took around ~2 Millon (more like 1.6) Thanks