I am trying to write a remapping target for usage with DM.
I followed instructions from several places (including this Answer) all essentially giving the same code.
This is ok, but not enough for me.
I need to modify "in transit" data of struct bio being remapped.
This means I need to make a deep-clone of the bio, including the data; apparently the provided functions (e.g.: bio_clone_bioset()) do not copy data at all, but point iovec's to the original pages/offsets.
I tried some variations of the following scheme:
void
mt_copy(struct bio *dst, struct bio *src) {
struct bvec_iter src_iter, dst_iter;
struct bio_vec src_bv, dst_bv;
void *src_p, *dst_p;
unsigned bytes;
unsigned salt;
src_iter = src->bi_iter;
dst_iter = dst->bi_iter;
salt = src_iter.bi_sector;
while (1) {
if (!src_iter.bi_size) {
break;
}
if (!dst_iter.bi_size) {
break;
}
src_bv = bio_iter_iovec(src, src_iter);
dst_bv = bio_iter_iovec(dst, dst_iter);
bytes = min(src_bv.bv_len, dst_bv.bv_len);
src_p = kmap_atomic(src_bv.bv_page);
dst_p = kmap_atomic(dst_bv.bv_page);
memcpy(dst_p + dst_bv.bv_offset, src_p + src_bv.bv_offset, bytes);
kunmap_atomic(dst_p);
kunmap_atomic(src_p);
bio_advance_iter(src, &src_iter, bytes);
bio_advance_iter(dst, &dst_iter, bytes);
}
}
static struct bio *
mt_clone(struct bio *bio) {
struct bio *clone;
clone = bio_clone_bioset(bio, GFP_KERNEL, NULL);
if (!clone) {
return NULL;
}
if (bio_alloc_pages(clone, GFP_KERNEL)) {
bio_put(clone);
return NULL;
}
clone->bi_private = bio;
if (bio_data_dir(bio) == WRITE) {
mt_copy(clone, bio);
}
return clone;
}
static int
mt_map(struct dm_target *ti, struct bio *bio) {
struct mt_private *mdt = (struct mt_private *) ti->private;
bio->bi_bdev = mdt->dev->bdev;
bio = mt_clone(bio);
submit_bio(bio->bi_rw, bio);
return DM_MAPIO_SUBMITTED;
}
This, however, does not work.
When I submit_bio() using the cloned bio I do not get the .end_io call and the calling task becomes blocked ("INFO: task mount:488 blocked for more than 120 seconds."). This with a READ request consisting of a single iovec (1024 bytes). In this case, of course the in buffers do not need copying because they should be overwritten; I need to copy back the incoming data unto the original buffers after the request has completed... but I don't get there.
I'm quite evidently missing some piece, but I'm unable to understand what.
Note: I didn't do any optimization (e.g.: use smarter allocation strategies) specifically because I need to get the basics first.
Note: I corrected a mistake (thanks #RuslanRLaishev), unfortunately ininfluent; see my own answer.
It's correct ?
if (bio_alloc_pages(**bio**, GFP_KERNEL)) {
bio_put(clone);
return NULL;
}
or
if (bio_alloc_pages(**clone**, GFP_KERNEL)) {
bio_put(bio);
return NULL;
}
It turns out bio_clone_bioset() and friends do not copy the callback address to call when request is over.
Trivial solution is to add clone->bi_end_io = bio->bi_end_io; before the end of mt_clone().
Unfortunately this is not enough to make the code functional because it turns out upper layers can spawn thousands of inflight requests (i.e.: requests queued and preprocessed before the previous ones complete) leading to memory starvation. Trying to slow upper layers by returning DM_MAPIO_REQUEUE does not seem to work (see: https://unix.stackexchange.com/q/410525/130498). This has nothing to do with current question, however.
Related
When creating a vfs using the tcl api how do you get the current filesystem in Tcl_Filesystem.pathInFilesystemProc
My code looks something like this:
typedef struct {
FILE* dbFile;
/*...*/
} FSBackend;
void createFS(const char* dbFile)
{
FSBackend* fsback = (FSBackend*)malloc(sizeof(FSBackend));
initDb(fsback,dbFile);
Tcl_Filesystem tfs;
tfs.typeName="Db Fs";
tfs.structureLength = sizeof(Tcl_Filesystem);
tfs.version = TCL_FILESYSTEM_VERSION_1;
tfs.pathInFilesystemProc = inFsProc;
/*...*/
Tcl_FSRegister((void*),tfs);
}
int inFsProc(Tcl_Obj* pathPtr,ClientData* cd)
{
/* How do I get my FSBackend struct here */
FSBackend* bk = /* ? */
int len;
const char* searchPath = Tcl_GetStringFromObj(pathPtr,&len);
char* foundPath = findFileInDb(searchPath,bk);
if (foundPath == 0) {
return -1;
}
cd = buildInternalRep(foundPath,bk);
return TCL_OK;
}
/**
...
*/
int main()
{
createFS("db1.db");
createFS("db2.db");
}
How do I, in inFsProc get back the struct I passed into Tcl_FSRegister?
The Tcl_FSData function says it can get it but I would then need to get a Tcl_Filesystem pointer
That's a weird one. The clientData handle there is not used to specify a mount point, but rather a separate capability of the filesystem type. Tcl's internal use of Tcl_FSRegister doesn't use it at all. The code which is as close as anything to a canonical use of it is the tclvfs package.
https://github.com/tcl-mirror/tclvfs/blob/master/generic/vfs.c#L385 shows us the use:
static void
Vfs_RegisterWithInterp(interp)
Tcl_Interp *interp;
{
ClientData vfsAlreadyRegistered;
/*
* We need to know if the interpreter is deleted, so we can
* remove all interp-specific mounts.
*/
Tcl_SetAssocData(interp, "vfs::inUse", (Tcl_InterpDeleteProc*)
Vfs_UnregisterWithInterp, (ClientData) 1);
/*
* Perform one-off registering of our filesystem if that
* has not happened before.
*/
vfsAlreadyRegistered = Tcl_FSData(&vfsFilesystem);
if (vfsAlreadyRegistered == NULL) {
Tcl_FSRegister((ClientData)1, &vfsFilesystem);
Tcl_CreateExitHandler(VfsExitProc, (ClientData)NULL);
Tcl_CreateThreadExitHandler(VfsThreadExitProc, NULL);
}
}
As you can see, the clientData there is really just being used as a marker so the code knows whether to do one-time initialisation.
To discover what the mount mapping is, you'll need to keep internal structures. You're strongly recommended to make the Tcl_Filesystem structure instance itself be global (or rather static at file scope) in your code.
I am implementing FIFO in C. One thread is writing in FIFO and other is reading from it.
#define BUFFER_LENGTH 1000
struct Frame
{
char data[1024];
unsigned int data_len;
struct Frame* frame;
};
struct Frame * get_from_fifo ()
{
if (!fifo_length)
{
first = last = NULL;
return NULL;
}
struct Frame* frame = first;
first = first->frame;
fifo_length--;
return frame;
}
int add_to_fifo (const char* data, unsigned int frame_size)
{
if (fifo_length >= BUFFER_LENGTH)
{
ast_log(LOG_ERROR, "Buffer full\n");
return SURESH_ERROR;
}
struct Frame* frame = malloc(sizeof (struct Frame));
frame->data_len = frame_size;
memcpy(frame->data, data, frame_size);
if (last)
{
last->frame = frame;
last = frame;
}
if (!first)
{
first = last = frame;
}
fifo_length++;
return SURESH_SUCCESS;
}
how can I prevent functions *add_to_fifo* and *get_from_fifo* to be called at the same time by different threads. i.e. *get_from_fifo* should only be called when the other thread is not executing *add_to_fifo* and vice verca.
As you are implementing FIFO stack the only really concurrent operation you have is changing the stack size (fifo_length).
You are adding entries to the tail of the stack and removing entries from the head of the stack so these two operation will never interfere with each other. So the only part you will need to worry about is changing the stack size (fifo_length), I would put it into separate function synchronised by mutex or flag (as mentioned by "Joey" above) and call it from both add_to_fifo() and get_from_fifo() functions.
You need to use a mutex (mutual exclusion) variable. The pthread library has everything you will need. Here's a good place to start looking at the available functions:
http://pubs.opengroup.org/onlinepubs/009695399/basedefs/pthread.h.html
You'll need to init a mutex variable that each thread will have access to:
http://pubs.opengroup.org/onlinepubs/009695399/functions/pthread_mutex_init.html
Then your threads will need to lock it when they need access to shared memory, and then unlock it when they are done using the shared memory:
http://pubs.opengroup.org/onlinepubs/009695399/functions/pthread_mutex_lock.html
Here's a simple example:
http://publib.boulder.ibm.com/infocenter/iseries/v5r3/index.jsp?topic=%2Frzahw%2Frzahwe18rx.htm
Good luck!
I'm need to implement a few functions that read messages from different devices that have different interface possibilities and different message structure. (but the messages have pretty much the same data)
Eg
Device_A {
message type: A
iface 1: tcp
}
Device_B {
message type: B
iface 1: serial
iface 2: tcp
}
... and so on
In my main...
struct msg_data;
while(user_wants_to_read) {
read_msg(); // reads and sets data in msg_data
do_work(msg_data);
}
In an OO Language I would use the strategy pattern. I think I could do this with a void* read_func;?
I'm inexperienced in C and I want to learn to program this like a good C programmer would do. What sort of design pattern/functions should I implement?
It sounds like you got two or more different abstractions to solve for:
Different stream sources (TCP vs. Serial). Is the the TCP protocol the same for device A and device B?
Different message types that are structurally different but semantically the same.
Different device classes (device A vs Device B)
I would focus on a strategy pattern with factories for reading from a stream. And then perhaps an adapter or strategy pattern for getting more data into message objects. But I wouldn't get held up on "which design pattern". More likely, just think in terms of interfaces.
So to start, perhaps abstracting out the serial and TCP streaming into different implementations with the same interface. One implementation that knows how connect and read bytes from a TCP socket without regard to the message contents. Another that knows how to read from a serial port. They should have the same "interface". Here's a lightweight example of a a "byte stream interface" with some hacked up socket code thrown. Forgive me if this doesn't compile. I might have a typo valid in C++ by wrong in C. In any case, it's just an example demonstrating interfaces through function table pointers.
My thinking on suggesting this is, "how would I implement this in C++?" And then I'm transposing my answer to pure "C". (Note: I'm likely making some declaration mistakes below.)
struct ByteStreamer;
typedef int (*ReadFunc)(ByteStreamer*, char* buffer, int count);
typedef int (*OpenFunc)(ByteStreamer*, char* url); // maybe 'open' isn't needed if it's handled by the factory
typedef int (*CloseFunc)(ByteStreamer*);
typedef void (*DisposeFunc)(ByteStreamer*);
typedef struct _ByteStreamer
{
ReadFunc readfunc;
OpenFunc openfunc;
CloseFunc closefunc;
DisposeFunc dispose;
// private data meant for the "class"
void* instancedata;
} ByteStreamer;
struct _tcpconnection
{
int socket;
sockaddr_in addrRemote;
} TCPConnection;
struct _serialconnection
{
int filehandle;
int baud;
} SerialConnection;
// ---------------------------------------
ByteStream* CreateStreamForTCP(const sockaddr_in *pAddr) // pass additional parameter as needed
{
ByteStreamer* pStream = (ByteStreamre*)malloc(sizeof(ByteStreamer));
TCPConnection* pTCPConnection = (TCPConnection*)malloc(sizeof(TCPConnection*));
pTCPConnection->socket = -1;
pTCPConnection->addrRemote = *pAddr;
pStream->instancedata = pTCPConnection;
pStream->ReadFunc = TCPRead;
pStream->OpenFunc = TCPOpen;
pStream->CloseFunc = TCPClose;
pStream->DisposeFunc = TCPDispose;
pStream->type = STREAM_TYPE_TCP;
return pStream;
}
int TCPRead(ByteStream* pStream, char* buffer, int count)
{
return recv(((TCPConnection*)pStream->instancedata)->socket, buffer, count, 0);
}
int TCPOpen(ByteStream* pStream, char* url)
{
// it's up to you if you want to encapsulate the socket address in url or in the instance data
TCPConnection* pConn = (TCPConnection*)(pStream->instancedata);
int sock = socket(AF_INET, SOCK_STREAM, 0);
connect(&pConn->addrRemote, sizeof(pConn->addrRemote));
return (pConn->sock >= 0); // true/false return;
}
void TCPClose(ByteStream* pStream)
{
TCPConnection* pConn = (TCPConnection*)(pStream->instancedata);
close(pConn->sock);
}
void TCPDispose(ByteStream* pStream)
{
free(pStream->instancedata);
free(pStream);
}
Now replace all the TCP code above with an equivalent serial port implementation. It would also be a good idea to implement a "file stream" (or "in memory stream") version of the ByteStream struct. Because it will be very useful in unit tests for higher level code.
So after you get all the byte stream implementations worked out, then move onto parsing device specific messages.
typedef struct _Message_A
{
// A specific data fields
} Message_A;
struct _Message_B
{
// B specific data fields
} Message_B;
struct Message
{
// commonality between Message_A and Message_B
};
typedef (*ReadMessageFromStream)(MessageReader* pReader, Message* pMsg); // pStream is an in-param, pMSg is an out-param.
typedef (*MessageReaderDispose)();
struct MessageReader
{
ReadMessageFromStream reader;
MessageReaderDispose dispose;
// -----------------------------
ByteStream* pStream;
void *instancedata;
};
// function to read a "Message_A" from a stream - and then transpose it to the generic Message type
int ReadMessage_A(ByteStream* pStream, Message* pMsg);
// function to read a "Message_B" from a stream - and then transpose it to the generic Message type
int ReadMessage_B(ByteStream* pStream, Message* pMsg);
So what's really cool about implementing ReadMessage_A and ReadMessage_B is that you can pass that "file stream" implementation of ByteStream and make some really good unit tests. So when you plug in the TCP or serial version, it has a high chance of just working (assuming your TCP and serial code are tested seperately).
And then perhaps a factory method off each class for creating the uber ReadMessageFromStream:
MessageReader* CreateTCPReaderForDeviceA(DeviceA* pA, sockaddr_in* pAddr)
{
MessageReader *pMR = (vMessageReader*)malloc(sizeof(MessageReader));
pMR->pStream = CreateStreamForTCP(pAddr);
pMR->pStream->Open();
pMR->reader = ReadMessage_A;
return pMR;
}
MessageReader* CreateSerialReaderForDeviceB(DeviceB* pB, int comport)
{
MessageReader *pMR = (vMessageReader*)malloc(sizeof(MessageReader));
pMR->pStream = CreateStreamForSerial(comport);
pMR->pStream->Open();
pMR->reader = ReadMessage_B;
return pMR;
}
And then your main loop looks something like the following:
if ((type == DEVICE_A) && (source == TCP))
pReader = CreateTCPReaderForDeviceA(pDevice, &addr)
else if ((type == DEVICE_B) && (source == SERIAL))
pReader = CreateSerialReaderForDeviceB(pDeviceB, 1);
// read the message
Message msg;
pReader->reader(pReader, &msg);
pReader->Dispose(); // free all the data allocated and close connections/files
Wooh.... I'm tired of typing this point. hope this helps.
I would agree with #rsaxvc. Function pointers are probably the best way to go about this. A google search turned up this: Strategy pattern in C
And for your message struct, you could use nested struct to emulate OO class inheritance
struct base {
// common members
}
struct child1 {
struct base;
// other data members
}
or simplely:
struct child2 {
// same data members as base
// other data members
}
use a base* parameter
gcc 4.4.4 c89
I have always thought of using malloc for the life of the project for being the scope.
But I am just wondering if my idea is the best practice. For example, I initalize an instance of the struct in main. And create 2 functions for creating and destroying. I am just wondering if this is the right thing to do.
I have some skeleton code below.
Many thanks for any advice,
typedef struct Network_dev_t {
size_t id;
char *name;
} Network_dev;
Network_dev* create_network_device(Network_dev *network)
{
network = malloc(sizeof *network);
if(network == NULL) {
return NULL;
}
return network;
}
void do_something(Network_dev *network)
{
/* Do something with the network device */
}
void destroy_network_device(Network_dev *network)
{
free(network);
}
int main(void)
{
Network_dev *network = NULL;
network = create_network_device(network);
/* Do something with the network device */
do_something(network);
destroy_network_device(network);
return 0;
}
Looks good.
I have a point or 2 about create_network_device
Network_dev* create_network_device(Network_dev *network)
no need to pass in a pointer; I'd rather have Network_dev* create_network_device(void)
{
network = malloc(sizeof *network);
the if is not really necessary; if malloc failed the return network at the end of the function is the same as return NULL.
if(network == NULL) {
return NULL;
}
If the allocation succeeded you might want to insure the struct members are in a know state here
/* if (network) { */
/* id = 0; */
/* name = NULL; */
/* } */
return network;
}
This code looks fine to me. I agree with pmg that your create_network_device could use a little work. Just to pull together what he said and make things clearer, here is exactly how I would write the function:
Network_dev *create_network_device()
{
Network_dev *network = malloc(sizeof(*network));
if (network) {
network->id = 0;
network->name = NULL;
}
return network;
}
It is best to allocate memory and free memory in the same function. Just like you open and close files in the same function. You did this by creating and destroying a Network_dev in the main() function, which is good. This makes it easy to confirm that all malloced locations are also freed.
It is best to malloc() something as late as possible and free() it as soon as possible. That is, hold the memory for as short as possible. If your program's job is to do something with Network_dev, you did all right. If your program does a lot of other things, you should do them before malloc() or after free().
I am getting an error when I try to run a c file which does some basic writes to a serial port. I am trying to run it asynchronously because the writes sometimes take a long time to transfer. My original version had it running synchronously with WriteFile() commands which worked fine. I am new to using OVERLAPPED and would appreciate and input concerning it.
The error I am getting is:
Debug Assertion Failed!
<path to dbgheap.c>
Line: 1317
Expression: _CrtIsValidHeapPointer(pUserData)
when the second write function is called.
In main:
{
//initialized port (with overlapped), DBC, and timeouts
result = write_port(outPortHandle, 128);
result = write_port(outPortHandle, 131);
}
static void CALLBACK write_compl(DWORD dwErrorCode, DWORD dwNumberOfBytesTransfered, LPOVERLAPPED lpOverlapped) {
//write completed. check for errors? if so throw an exception maybe?
printf("write completed--and made it to callback function\n");
}
int write_port(HANDLE hComm,BYTE* lpBuf) {
OVERLAPPED osWrite = {0};
// Create this write operation's OVERLAPPED structure's hEvent.
osWrite.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
if (osWrite.hEvent == NULL)
// error creating overlapped event handle
return 0;
// Issue write.
if (!WriteFileEx(hComm, &lpBuf, 1, &osWrite, &write_compl )) {
if (GetLastError() != ERROR_IO_PENDING) {
// WriteFile failed, but isn't delayed. Report error and abort.
printf("last error: %ld",GetLastError());
return 0; //failed, return false;
}
else {
// Write is pending.
WaitForSingleObjectEx(osWrite.hEvent, 50, TRUE); //50 ms timeout
return -1; //pending
}
}
else {
return 1; //finished
}
}
That was not the full code, sorry. I was using an array of BYTEs as well, not constants. But system("pause")'s were causing my debug assertion failed errors, and after carefully looking through my code, when the WriteFileEx() was successful, it was never setting an alert/timeout on the event in the overlapped structure, so the callback function would never get called. I fixed these problems though.
I just need help with the handling/accessing a single BYTE in a structure which is allocated when a ReadFileEx() function is called (for storing the BYTE that is read so it can be handled). I need to know how to access that BYTE storage using an offset and make the overlapped structure null. Would making the overlapped structure null be as simple as setting the handle in it to INVALID_HANDLE_VALUE?
I think you have a couple of issues:
You are passing an integer as a pointer (your compiler should warn against this or preferably refuse to compile the code):
result = write_port(outPortHandle, 128);
Compare this to the definition of write_port:
int write_port(HANDLE hComm,BYTE* lpBuf) {
The above statements doesn't match. Later on you then pass a pointer to the lpBuf pointer to the WriteFileEx function by taking the address of the BYTE* -> "&lpBuf". This will not result in what you think it will do.
Even if you fix this, you will still have potential lifetime issues whenever the write is successfully queued but won't complete within the 50 ms timeout.
When using overlapped I/O, you need to make sure that the read/write buffer and the overlapped structure remain valid until the I/O is completed, cancelled or the associated device is closed. In your code above you use a pointer to an OVERLAPPED struct that lives on the stack in your call to WriteFileEx. If WriteFileEx does not complete within 50 ms, the pending I/O will have a reference to a non-existing OVERLAPPED struct and you will (hopefully) have an access violation (or worse, silently corrupted stack data somewhere in your app).
The canonical way of handling these lifetime issues (if performance is not a big issue), is to use a custom struct that includes an OVERLAPPED struct and some storage for the data to be read/written. Allocate the struct when posting the write and deallocate the struct from the I/O completion routine. Pass the address of the included OVERLAPPED struct to WriteFileEx, and use e.g. offsetof to get the address to the custom struct from the OVERLAPPED address in the completion routine.
Also note that WriteFileEx does not actually use the hEvent member, IIRC.
EDIT: Added code sample, please note:
I haven't actually tried to compile the code, there might be typos or other problems with the code.
It's not the most efficient way of sending data (allocating/deallocating a memory block for each byte that is sent). It should be easy to improve, though.
#include <stddef.h>
#include <assert.h>
#include <windows.h>
// ...
typedef struct _MYOVERLAPPED
{
OVERLAPPED ol;
BYTE buffer;
} MYOVERLAPPED, *LPMYOVERLAPPED;
// ...
static void CALLBACK write_compl(DWORD dwErrorCode, DWORD dwNumberOfBytesTransfered, LPOVERLAPPED lpOverlapped)
{
if (NULL == lpOverlapped)
{
assert(!"Should never happen");
return;
}
LPBYTE pOlAsBytes = (LPBYTE)lpOverlapped;
LPBYTE pMyOlAsBytes = pOlAsBytes - offsetof(MYOVERLAPPED, ol);
LPMYOVERLAPPED pMyOl = (LPMYOVERLAPPED)pOlAsBytes;
if ((ERROR_SUCCESS == dwErrorCode) &&
(sizeof(BYTE) == dwNumberOfBytesTransfered))
{
printf("written %uc\n", pMyOl->buffer);
}
else
{
// handle error
}
free(pMyOl);
}
int write_port(HANDLE hComm, BYTE byte) {
LPMYOVERLAPPED pMyOl = (LPMYOVERLAPPED)malloc(sizeof(MYOVERLAPPED));
ZeroMemory(pMyOl, sizeof(MYOVERLAPPED));
pMyOl->buffer = byte;
// Issue write.
if (!WriteFileEx(hComm, &pMyOl->buffer, sizeof(BYTE), pMyOl, &write_compl )) {
if (GetLastError() != ERROR_IO_PENDING) {
// WriteFile failed, but isn't delayed. Report error and abort.
free(pMyOl);
printf("last error: %ld",GetLastError());
return 0; //failed, return false;
}
else {
return -1; //pending
}
}
else {
free(pMyOl);
return 1; //finished
}
}
result = write_port(outPortHandle, 128);
result = write_port(outPortHandle, 131);
The lpBuf argument have to be pointers to buffers, not constants.
e.g.
char buffer;
buffer = 128;
result = write_port(outPortHandle, &buffer);
buffer = 131;
result = write_port(outPortHandle, &buffer);
What you really want to do is also pass a buffer length.
e.g.
char buffer[] = { 128, 131 };
result = write_port(outPortHandle, &buffer, sizeof(buffer));
int write_port(HANDLE hComm,BYTE* lpBuf, size_t length) {
...
// Issue write.
if (!WriteFileEx(hComm, &lpBuf, length, &osWrite, &write_compl )) {
...