Get pos and extract string in TFileStream - file

I am trying to find posisition of mykeyword and extract strings from a loaded file (up to 200 MB).
procedure TForm5.Button4Click(Sender: TObject);
var
Stream: TFileStream;
Buffer: array [0 .. 1023] of AnsiChar;
i: Integer;
myKeyword: string;
pullStr: AnsiString;
begin
myKeyword :='anything';
Stream := TFileStream.Create(edtTarget.Text, fmOpenRead);
while Stream.Position < Stream.Size do
begin
Stream.Read(Buffer, 1024);
m1.Lines.Add(Buffer); // no need, just display to evaluate
(* 1. Get address of given keyword *)
// i := Stream.PositionOf(myKeyword); < how to do this?
(* 2. Stream Exract *)
// pullStr := Stream.copy(i,1000); < how to do this ?
end;
end;
I have read other topics regarding file and string. I found a very good answer from here. And i think i want to expand those features.
Something like
TFileSearchReplace.GetStrPos(const KeyWord: string) : Integer;
TFileSearchReplace.ExtractStr (const KeyWord: string; Len : Integer) ;

procedure TForm5.Button4Click(Sender: TObject);
var
Stream: TFileStream;
Buffer: AnsiString;
i, BytesRead, SearchPos: Integer;
myKeyword: string;
pullStr: AnsiString;
Found: Boolean;
begin
myKeyword :='anything';
Found := False;
SetLength(Buffer, 1024);
Stream := TFileStream.Create(edtTarget.Text, fmOpenRead);
while Stream.Position < Stream.Size do
begin
// read some bytes and remember, how many bytes been read actually
BytesRead := Stream.Read(Buffer[1], 1024);
// glue new bytes to the end of the pullStr
pullStr := pullStr + copy(Buffer, 1, BytesRead);
// file is divided to two parts: before myKeyword, and after
// if myKeyword alreay found, there is nothing to do, just repeat reading to pullStr
if Found then
continue;
// if myKeyword is not found yet, pullStr acts like temporary buffer
// search for myKeyword in buffer
SearchPos := Pos(myKeyword, pullStr);
if SearchPos > 0 then
begin //keyword was found, delete from beginning up to and icluding myKeyword
// from now on, pullStr is not tmp buffer, but result
Found := True;
Delete(pullStr, 1, SearchPos + Length(myKeyWord) - 1);
continue;
end;
// myKeyword still not found. Find last line end in buffer
SearchPos := LastDelimiter(#13#10, pullStr);
// and delete everything before it
if SearchPos > 0 then
Delete(pullStr, 1, SearchPos);
// so if myKeyword spans across two reads, it still will be found in next iteration
end;
// if there is no myKeyword in file, clear buffer
if not Found then
pullStr := '';
end;

Related

Array to Stream in Delphi 10.4

How can i save and load an 'Array[0..9] of TJpegImage' to a MemoryStream.
There are many samples to save one Image to a Stream but no one save an Array.
I need it to save the Stream to a Database Blob field and later load the BlobField back to my Array.
All Images in the Array has the same Size.
Maybe a Stream of TByte can do this ?
But how to convert a TByte-Stream back to an Array?
Update:
Thank you very much MBo for your example.
I tried to use it and get some Problems.
I use a Paintbox to Draw the Picture and get an access violation at 0x0062b9ab:read of address 0x000....
This is the Code i try:
unit PicToMemStream;
interface
uses
Winapi.Windows, Winapi.Messages, System.SysUtils, System.Variants, System.Classes, Vcl.Graphics,
Vcl.Controls, Vcl.Forms, Vcl.Dialogs, Vcl.Imaging.jpeg, Vcl.ExtCtrls;
type
TForm1 = class(TForm)
Image1: TImage;
PaintBox: TPaintBox;
private
{ Private-Deklarationen }
public
{ Public-Deklarationen }
end;
var
Form1: TForm1;
implementation
{$R *.dfm}
var
ms, temp: TMemoryStream;
jps: array of TJpegImage;
i, n, sz: Integer;
ProgPath: String;
begin
ProgPath := ExtractFilePath(Application.ExeName);
n := 3;
SetLength(jps, n);
ms := TMemoryStream.Create;
temp := TMemoryStream.Create;
for i := 0 to n - 1 do
begin
jps[i] := TJpegImage.Create;
jps[i].LoadFromFile(Format(ProgPath + '%s.jpg', [Chr(Ord('a') + i)]));
// in my case loads files a.jpg, b.jpg, c.jpg
end;
// Save Pictures to Stream and later in BlobField (100kByte)
try
ms.Write(n, SizeOf(n));
for i := 0 to n - 1 do
begin
temp.Clear;
// Save Image Data first here
jps[i].SaveToStream(temp); // jpeg data here
sz := temp.Size;
// Save length second here
ms.Write(sz, SizeOf(sz)); // size of image
temp.Position := 0;
ms.CopyFrom(temp, sz); // image data to final destination
end;
ms.SaveToFile(ProgPath + 'base.dat'); // for control // later here Write Data to BLob
// Read Picture and show it in a Paintbox or Image
// revert to empty stream
ms.Clear;
ms.LoadFromFile(ProgPath + 'base.dat'); // load from "base" // later here Read Data from Blob
ms.Read(n, SizeOf(n));
// here some actions to setup array
for i := 0 to n - 1 do // Read 3 times a picture
begin
ms.Read(sz, SizeOf(sz)); // Read Lenth of Picturedata first
temp.Clear; // Read Picturedata in jps[i]
temp.CopyFrom(ms, sz);
temp.Position := 0;
jps[i].LoadFromStream(temp);
Form1.PaintBox.Canvas.Draw(0, 0, jps[i]); // Show Picture with Error
//Form1.Image1.Picture.Bitmap.LoadFromStream(temp);
Sleep(2000); // To see the Picture for 2 seconds (Test)
end;
finally
ms.Free;
temp.Free;
for i := 0 to n - 1 do
jps[i].Free;
end;
end.
Update2:
I put the code in a ButtomClick procedure and change:
temp.Position := 0;
jps[i].LoadFromStream(temp);
Form1.PaintBox.Canvas.Draw(0, 0, jps[i]); // Show Picture in Paintbox
Form1.Image1.Picture.Assign(jps[i]); // Show Picture in TImage
Application.ProcessMessages;
Sleep(500); // To see the Picture for 2 seconds (Test)
end;
I will also test the other Ideas with using Master-Detail Table or TFDMemTable or Encoding to Text, to see what is the best for me.
Thanks to all Helpers.
All Images in the Array has the same Size - perhaps dimensions (width/height) are the same, but size of jpeg image in bytes varies depending on picture and compression. So I assume you need to store both image data and data size.
Possible approach:
write integer value - length of array - to memory stream ms
for every array element save it to temporary memory stream temp_ms
write integer value sz = ms.size to main memory stream ms
reset temp_ms.Position to 0
copy (CopyFrom) sz bytes from temp_ms to ms
clear temp_ms
repeat for all array elements
Now ms contents looks like:
10 14212 FFD8....D9 31234 FFD8....D9
10 images
1st image size
1st image contents
2nd image size
2nd image contents
To retrieve image, perform reverse actions:
reset ms position (if needed)
read number of images n from ms
set arrray size (if array is dynamic)
n times:
read size of image sz
copy sz bytes to temp_ms
reset temp_ms.Position
load array[i] from temp_ms
working example:
var
ms, temp: TMemoryStream;
jps: array of TJpegImage;
i, n, sz: Integer;
begin
n := 3;
SetLength(jps, n);
ms := TMemoryStream.Create;
temp := TMemoryStream.Create;
for i := 0 to n - 1 do begin
jps[i] := TJpegImage.Create;
jps[i].LoadFromFile(Format('h:\%s.jpg', [Chr(Ord('a') + i)]));
//in my case loads files a.jpg, b.jpg, c.jpg
end;
try
ms.Write(n, SizeOf(n));
for i :=0 to n - 1 do begin
temp.Clear;
jps[i].SaveToStream(temp); //jpeg data here
sz := temp.Size;
ms.Write(sz, SizeOf(sz)); //size of image
temp.Position := 0;
ms.CopyFrom(temp, sz); //image data to final destination
end;
ms.SaveToFile('h:\base.dat'); //for control
//revert to empty stream
ms.Clear;
ms.LoadFromFile('h:\base.dat'); //load from "base"
ms.Read(n, SizeOf(n));
//here some actions to setup array
for i :=0 to n - 1 do begin
ms.Read(sz, SizeOf(sz));
temp.Clear;
temp.CopyFrom(ms, sz);
temp.Position := 0;
jps[i].LoadFromStream(temp);
Canvas.Draw(i * 200, 0, jps[i]); //control shot
end;
finally
ms.Free;
temp.Free;
for i := 0 to n - 1 do
jps[i].Free;
end;
Why don't you use a TFDMemTable?
Another option can be, TStringList using System.NetEncoding.TNetEncoding.Base64 encoding
since TStringList is very much like a superpowered vector

How get correct total number of elements of a PByteArray?

On following code i'm trying convert a binary file to your correspondent bytes representation in Delphi.
I'm with difficulty to get the correct total number of elements of array (declared as PByteArray) and this returns a limited number (32767) like already discussed here. Then exists a alternative way to achieve it?
This is code:
type
TForm1 = class(TForm)
btn1: TButton;
procedure btn1Click(Sender: TObject);
private
{ Private declarations }
public
{ Public declarations }
end;
var
Form1: TForm1;
implementation
{$R *.dfm}
function File2ByteArray(const FileName: string; out ByteArray: PByteArray): Boolean;
var
hFile: HWND;
dwSize, dwRead: DWORD;
begin
Result := False;
hFile := CreateFile(PChar(FileName), GENERIC_READ, FILE_SHARE_READ, nil, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
if hFile = INVALID_HANDLE_VALUE then
Exit;
dwSize := GetFileSize(hFile, nil);
ByteArray := AllocMem(dwSize);
ReadFile(hFile, ByteArray^, dwSize, dwRead, nil);
Result := True;
CloseHandle(hFile);
end;
procedure TForm1.btn1Click(Sender: TObject);
var
ByteArray: PByteArray;
MyList: TStringList;
I: Integer;
begin
if File2ByteArray(ParamStr(0), ByteArray) then
begin
try
MyList := TStringList.Create;
MyList.Add('Const');
MyList.Add(' aSize = ' + IntToStr(High(ByteArray^)) + ';');
MyList.Add('stub_: Array [' + IntToStr(Low(ByteArray^)) + ' .. ' + IntToStr(High(ByteArray^) - 1) + '] of Byte =');
MyList.Add('(');
MyList.Add('');
for I := Low(ByteArray^) to High(ByteArray^) - 1 do
MyList.Add('$' + IntToHex(ByteArray[I], 2) + ',');
MyList.Add(');');
MyList.SaveToFile('C:\File2ByteArray.txt');
finally
MyList.Free;
end;
end;
end;
end.
A pointer to an array allocated as you have done so, does not contain length information. You cannot query that pointer for the length of the array. Instead you need to modify File2ByteArray so that it returns that length, in addition to the pointer.
We can't see what type PByteArray is, but I would normally expect to see PByte used to represent a pointer to an array of bytes.
It's also worth noting that your code leaks the array. I would resolve that, and resolve the issue of length, by returning a TBytes dynamic array instead of a manually allocated array.

"Operator is not overloaded" when checking for a byte in an array of bytes

I have an array[0..2] of byte. I need to check if a byte is in that array or not. However, when using if ($52 in byteArray) then, I run into the error "Operator is not overloaded". I've tried setting an additional variable as a byte and then using that in the statement, but still get the error. Here's an incredibly simple program showing this:
program overloaded;
var
byteArray: array[0..2] of Byte;
begin
byteArray[0] := $00;
byteArray[1] := $69;
byteArray[2] := $52;
if ($52 in byteArray) then
writeLn('We will not get to this point');
end.
This will fail to compile with the above error in FPC 3.0.2.
You have two alternatives, overloading the operator or using for.
program ByteArrayIteration;
var
ByteArray: array[0..2] of Byte = ($00, $69, $52);
SomeByte : Byte = $52;
begin
for SomeByte in byteArray do
if SomeByte = $52 then
begin
WriteLn('We will get to this point');
Break;
end;
end.
And the overload alternative:
program OverloadInOperator;
uses SysUtils;
operator in(const A: Byte; const B: TBytes): Boolean;
var
i: Integer;
begin
Result := True;
for i := Low(B) to High(B) do if A = B[i] then Exit;
Result := False;
end;
var
Bytes : TBytes;
AByte : Byte = $52;
begin
Bytes := TBytes.Create($41, $52, $90);
if AByte in Bytes then WriteLn('Done');
end.
If your use case is limited to 255 items per array, consider using sets instead.
program SetOfByte;
var
Bytes : set of Byte = [$41, $52, $90];
AByte : Byte = $52;
begin
if AByte in Bytes then WriteLn('Done');
end.

How do I speedup the copy of a generic array?

I have a simple array:
Myarray<T> = array[0..100] of T;
How do I copy this as fast as possible into another array.
Note that the source is always a plain generic array and the dest can be a dynamic array. (but I doubt that matters).
I tried
move(Source[0], dest[0], count * sizeof(T));
But that does not play nice with managed types.
I'm hoping to get something faster than:
for i:= 0 to count -1 do D[i]:= S[i];
What options do I have?
Background
The array is part of a BTree structure and there is a lot of copying going on.
I need the speed.
I asked the same question on Google+ - https://plus.google.com/u/0/116430453567926016001/posts/b1u1shzkmrW
The code is by Stefan Glienke and I have re-posted the code here with to minor changes to two variable types, so it is 64 bit compatible.
uses
TypInfo,
SysUtils;
type
TArray = record
class function Clone<T>(const source: T): T; static;
end;
function DynArrayLength(const A: Pointer): NativeInt;
type
PDynArrayRec = ^TDynArrayRec;
TDynArrayRec = packed record
{$IFDEF CPUX64}
_Padding: LongInt;
{$ENDIF}
RefCnt: LongInt;
Length: NativeInt;
end;
begin
Result := 0;
if A <> nil then
Result := PDynArrayRec(PByte(A) - SizeOf(TDynArrayRec))^.Length;
end;
{$POINTERMATH ON}
type
PArray = ^Pointer;
{$POINTERMATH OFF}
procedure CopyArrayDeep(Dest, Source: PArray; TypeInfo: Pointer; Count: NativeInt);
var
typeData: PTypeData;
elType: PPTypeInfo;
i: Integer;
len : NativeInt;
begin
if (Source = nil) or (Count = 0) then
Exit;
typeData := GetTypeData(TypeInfo);
elType := typeData.elType;
if (elType <> nil) and (elType^.kind = tkDynArray) then
begin
for i := 0 to Count-1 do
begin
len := DynArrayLength(Source[i]);
DynArraySetLength(Dest[i], elType^, 1,#len);
CopyArrayDeep(Dest[i], Source[i], elType^, len);
end;
end
else
if elType <> nil then
CopyArray(Dest, Source, elType^, Count)
else
Move(Source^, Dest^, Count * typeData.elSize);
end;
class function TArray.Clone<T>(const source: T): T;
var
p: PTypeInfo;
count: NativeInt;
i: Integer;
begin
p := TypeInfo(T);
if p.Kind <> tkDynArray then
raise ENotSupportedException.Create('type not supported');
count := DynArrayLength(PArray(#source)^);
DynArraySetLength(PArray(#Result)^, p, 1,#count);
CopyArrayDeep(PArray(#Result)^, PArray(#source)^, p, count);
end;

Sharing data array between two applications in Delphi

I want to share array data between two applications. In my mind, first program create the array and the second program can read the array from already allocated memory area. The array is not a dynamic array.
I found a way to share pointer using OpenFileMapping and MapViewOfFile. I have no luck to implement array sharing and I think i don't want to use IPC method yet.
Is it possible to plan a scheme like this (sharing array)? My purpose is to minimize memory usage and reading data quickly.
Scratched my head thinking of what a short-but-complete example of sharing memory between two applications might be. The only option is a console application, GUI applications require a minimum of 3 files (DPR + PAS + DFM). So I cooked up a small example where one integers array is shared using a memory mapped file (backed by the page file so I don't need to have a phisical file on disk for this to work). The console application responds to 3 commands:
EXIT
SET NUM VALUE Changes the value at index NUM in the array to VALUE
DUMP NUM displays the value in the array at index NUM
DUMP ALL displays the whole array
Of course, the command processing code takes up about 80% of the whole application. To test this compile the following console application, find the executable and start it twice. Go to the first window and enter:
SET 1 100
SET 2 50
Go to the second console and enter this:
DUMP 1
DUMP 2
DUMP 3
SET 1 150
Go to the first console and enter this:
DUMP 1
There you have it, you've just witnessed sharing memory between two applications.
program Project2;
{$APPTYPE CONSOLE}
uses
SysUtils, Windows, Classes;
type
TSharedArray = array[0..10] of Integer;
PSharedArray = ^TSharedArray;
var
hFileMapping: THandle; // Mapping handle obtained using CreateFileMapping
SharedArray: PSharedArray; // Pointer to the shared array
cmd, s: string;
num, value, i: Integer;
L_CMD: TStringList;
function ReadNextCommand: string;
begin
WriteLn('Please enter command (one of EXIT, SET NUM VALUE, DUMP NUM, DUMP ALL)');
WriteLn;
ReadLn(Result);
end;
begin
try
hFileMapping := CreateFileMapping(0, nil, PAGE_READWRITE, 0, SizeOf(TSharedArray), '{C616DDE6-23E2-425C-B871-9E0DA54D96DF}');
if hFileMapping = 0 then
RaiseLastOSError
else
try
SharedArray := MapViewOfFile(hFileMapping, FILE_MAP_READ or FILE_MAP_WRITE, 0, 0, SizeOf(TSharedArray));
if SharedArray = nil then
RaiseLastOSError
else
try
WriteLn('Connected to the shared view of the file.');
cmd := ReadNextCommand;
while UpperCase(cmd) <> 'EXIT' do
begin
L_CMD := TStringList.Create;
try
L_CMD.DelimitedText := cmd;
for i:=0 to L_CMD.Count-1 do
L_CMD[i] := UpperCase(L_CMD[i]);
if (L_CMD.Count = 2) and (L_CMD[0] = 'DUMP') and TryStrToInt(L_CMD[1], num) then
WriteLn('SharedArray[', num, ']=', SharedArray^[num])
else if (L_CMD.Count = 2) and (L_CMD[0] = 'DUMP') and (L_CMD[1] = 'ALL') then
begin
for i:= Low(SharedArray^) to High(SharedArray^) do
WriteLn('SharedArray[', i, ']=', SharedArray^[i]);
end
else if (L_CMD.Count = 3) and (L_CMD[0] = 'SET') and TryStrToInt(L_CMD[1], num) and TryStrToInt(L_CMD[2], value) then
begin
SharedArray^[num] := Value;
WriteLn('SharedArray[', num, ']=', SharedArray^[num]);
end
else
WriteLn('Error processing command: ' + cmd);
finally L_CMD.Free;
end;
// Requst next command
cmd := ReadNextCommand;
end;
finally UnmapViewOfFile(SharedArray);
end;
finally CloseHandle(hFileMapping);
end;
except
on E: Exception do
Writeln(E.ClassName, ': ', E.Message);
end;
end.
A Named File Mapping would be the easiest solution, here is some short example code.
In this sample there is a main program that writes some data and reader(s) that only read from it.
Main:
type
TSharedData = record
Handle: THandle;
end;
PSharedData = ^TSharedData;
const
BUF_SIZE = 256;
var
SharedData: PSharedData;
hFileMapping: THandle; // Don't forget to close when you're done
function CreateNamedFileMapping(const Name: String): THandle;
begin
Result := CreateFileMapping(INVALID_HANDLE_VALUE, nil, PAGE_READWRITE, 0,
BUF_SIZE, PChar(Name));
Win32Check(Result > 0);
SharedData := MapViewOfFile(Result, FILE_MAP_ALL_ACCESS, 0, 0, BUF_SIZE);
Win32Check(Assigned(SharedData));
end;
procedure TForm1.Button1Click(Sender: TObject);
begin
hFileMapping := CreateNamedFileMapping('MySharedMemory');
Win32Check(hFileMapping > 0);
SharedData^.Handle := CreateHiddenWindow;
end;
reader:
var
hMapFile: THandle; // Don't forget to close
function GetSharedData: PSharedData;
begin
hMapFile := OpenFileMapping(FILE_MAP_ALL_ACCESS, False, 'MySharedMemory');
Win32Check(hMapFile > 0);
Result := MapViewOfFile(hMapFile, FILE_MAP_ALL_ACCESS, 0, 0, BUF_SIZE);
Win32Check(Assigned(Result));
end;

Resources