I am trying to read a character array that contains the contents of many large files. The character array is going to be quite large, because the files are large, so I want to do it using multithreading (pthread). I want the user to be able to designate how many threads they want to run. I have something working, but increasing the number of threads does nothing to affect performance (i.e. 1 thread finishes just as fast as 10). In fact, it seems to be just the opposite: telling the program to use 10 threads runs much slower than telling it to use 1.
Here is the method for slicing up the character array according to the number of threads the user passes to the program. I know this is wrong, I could use some advice here.
//Universal variables
int numThreads;
size_t sizeOfAllFiles; // Size, in bytes, of allFiles
char* allFiles; // Where all of the files are stored, together
void *zip(void *nthread);
void *zip(void *nThread) {
int currentThread = *(int*)nThread;
int remainder = sizeOfAllFiles % currentThread;
int slice = (sizeOfAllFiles-remainder) / currentThread;
// I subtracted the remainder for my testing
// because I didn't want to worry about whether
// the char array's size is evenly divisible by numThreads
int i = (slice * (currentThread-1));
char currentChar = allFiles[i]; //Used for iterating
while(i<(slice * currentThread) && i>=(slice * (currentThread-1))) {
i++;
// Do things with the respective thread's
// 'slice' of the array.
.....
}
return 0;
}
And here is how I am spawning the threads, which I am almost positive that I am doing correctly:
for (int j = 1; j <= threadNum; j++) {
k = malloc(sizeof(int));
*k = j;
if (pthread_create (&thread[j], NULL, zip, k) != 0) {
printf("Error\n");
free(thread);
exit(EXIT_FAILURE);
}
}
for (int i = 1; i <= threadNum; i++)
pthread_join (thread[i], NULL);
This is all really confusing for me so if I could get some help on this, I'd greatly appreciate it. I specifically am struggling with the slicing part (cutting it up correctly), and with not seeing performance gains by using more than one thread. Thanks in advance.
I'm starting by throwing a test program at you:
#include <assert.h>
#include <stdbool.h>
#include <stdlib.h>
#include <stdio.h>
#include <stddef.h>
#include <time.h>
bool
EnlargeBuffer(char ** const buffer_pointer,
size_t * const buffer_size)
{
char * larger_buffer = realloc(*buffer_pointer,
2 * *buffer_size);
if (! larger_buffer) {
larger_buffer = realloc(*buffer_pointer,
*buffer_size + 100);
if (! larger_buffer) {
return false;
}
*buffer_size += 100;
} else {
*buffer_size *= 2;
}
*buffer_pointer = larger_buffer;
printf("(Buffer size now at %zu)\n", *buffer_size);
return true;
}
bool
ReadAll(FILE * const source,
char ** pbuffer,
size_t * pbuffer_size,
size_t * pwrite_index)
{
int c;
while ((c = fgetc(source)) != EOF) {
assert(*pwrite_index < *pbuffer_size);
(*pbuffer)[(*pwrite_index)++] = c;
if (*pwrite_index == *pbuffer_size) {
if (! EnlargeBuffer(pbuffer, pbuffer_size)) {
free(*pbuffer);
return false;
}
}
}
if (ferror(source)) {
free(*pbuffer);
return false;
}
return true;
}
unsigned
CountAs(char const * const buffer,
size_t size)
{
unsigned count = 0;
while (size--)
{
if (buffer[size] == 'A') ++count;
}
return count;
}
int
main(int argc, char ** argv)
{
char * buffer = malloc(100);
if (! buffer) return 1;
size_t buffer_size = 100;
size_t write_index = 0;
clock_t begin = clock();
for (int i = 1; i < argc; ++i)
{
printf("Reading %s now ... \n", argv[i]);
FILE * const file = fopen(argv[i], "r");
if (! file) return 1;
if (! ReadAll(file, &buffer, &buffer_size, &write_index))
{
return 1;
}
fclose(file);
}
clock_t end = clock();
printf("Reading done, took %f seconds\n",
(double)(end - begin) / CLOCKS_PER_SEC);
begin = clock();
unsigned const as = CountAs(buffer, write_index);
end = clock();
printf("All files have %u 'A's, counting took %f seconds\n",
as,
(double)(end - begin) / CLOCKS_PER_SEC);
}
This program reads all files (passed as command line arguments) into one big large char * buffer, and then counts all bytes which are == 'A'. It also times both of these steps.
Example run with (shortened) output on my system:
# gcc -Wall -Wextra -std=c11 -pedantic allthefiles.c
# dd if=/dev/zero of=large_file bs=1M count=1000
# ./a.out allthefiles.c large_file
Reading allthefiles.c now ...
(Buffer size now at 200)
...
(Buffer size now at 3200)
Reading large_file now ...
(Buffer size now at 6400)
(Buffer size now at 12800)
...
(Buffer size now at 1677721600)
Reading done, took 4.828559 seconds
All files have 7 'A's, counting took 0.764503 seconds
Reading took almost 5 seconds, but counting (= iterating once, in a single thread, over all bytes) took a bit less than 1 second.
You're optimizing at the wrong place!
Using 1 thread to read all files, and then using N threads to operate on that one buffer isn't going to bring you places. The fastest way to read 1 file is to use 1 thread. For multiple files, use 1 thread per file!
So, in order to achieve the speedup that you need to show for your assignment:
Create a pool of threads with variable size.
Have a pool of tasks, where each task consists of
read one file
compute it's run-length encoding
store the run-length encoded file
let the threads take tasks from your task pool.
Things to consider: How do you combine the results of each task? Without requiring (costly) synchronization.
I have a string creating function in C which accepts an array of structs as it's argument and outputs a string based on a predefined format (like a list of list in python).
Here's the function
typedef struct
{
PacketInfo_t PacketInfo;
char Gnss60[1900];
//and other stuff...
} Track_json_t;
typedef struct
{
double latitude;
double longitude;
} GPSPoint_t;
typedef struct
{
UInt16 GPS_StatusCode;
UInt32 fixtime;
GPSPoint_t point;
double altitude;
unsigned char GPS_Satilite_Num;
} GPS_periodic_t;
unsigned short SendTrack()
{
Track_json_t i_sTrack_S;
memset(&i_sTrack_S, 0x00, sizeof(Track_json_t));
getEvent_Track(&i_sTrack_S);
//Many other stuff added to the i_sTrack_S struct...
//Make a JSON format out of it
BuildTrackPacket_json(&i_sTrack_S, XPORT_MODE_GPRS);
}
Track_json_t *getEvent_Track(Track_json_t *trk)
{
GPS_periodic_t l_gps_60Sec[60];
memset(&l_gps_60Sec, 0x00,
sizeof(GPS_periodic_t) * GPS_PERIODIC_ARRAY_SIZE);
getLastMinGPSdata(l_gps_60Sec, o_gps_base);
get_gps60secString(l_gps_60Sec, trk->Gnss60);
return trk;
}
void get_gps60secString(GPS_periodic_t input[60], char *output)
{
int i = 0;
memcpy(output, "[", 1); ///< Copy the first char as [
char temp[31];
for (i = 0; i < 59; i++) { //Run for n-1 elements
memset(temp, 0, sizeof(temp));
snprintf(temp, sizeof(temp), "[%0.8f,%0.8f],",
input[i].point.latitude, input[i].point.longitude);
strncat(output, temp, sizeof(temp));
}
memset(temp, 0, sizeof(temp)); //assign last element
snprintf(temp, sizeof(temp), "[%0.8f,%0.8f]]",
input[i].point.latitude, input[i].point.longitude);
strncat(output, temp, sizeof(temp));
}
So the output of the function must be a string of format
[[12.12345678,12.12345678],[12.12345678,12.12345678],...]
But at times I get a string which looks like
[[12.12345678,12.12345678],[55.01[12.12345678,12.12345678],...]
[[21.28211567,84.13454083],[21.28211533,21.22[21.28211517,84.13454000],..]
Previously, I had a buffer overflow at the function get_gps60secString, I fixed that by using snprintf and strncat.
Note: This is an embedded application and this error occur once or twice a day (out of 1440 packets)
Question
1. Could this be caused by an interrupt during the snprintf/strncat process?
2. Could this be caused by a memory leak, overwriting the stack or some other segmentation issue caused else where?
Basically I would like to understand what might be causing a corrupt string.
Having a hard time finding the cause and fixing this bug.
EDIT:
I used chux's function. Below is the Minimal, Complete, and Verifiable Example
/*
* Test code for SO question https://stackoverflow.com/questions/5216413
* A Minimal, Complete, and Verifiable Example
*/
#include <stdio.h>
#include <stdlib.h>
#include <memory.h>
#include <stdbool.h>
#include <signal.h>
#include <unistd.h>
typedef unsigned short UInt16;
typedef unsigned long UInt32;
#define GPS_PERIODIC_ARRAY_SIZE 60
#define GPS_STRING_SIZE 1900
/* ---------------------- Data Structs --------------------------*/
typedef struct
{
char Gnss60[GPS_STRING_SIZE];
} Track_json_t;
typedef struct
{
double latitude;
double longitude;
} GPSPoint_t;
typedef struct
{
UInt16 GPS_StatusCode;
UInt32 fixtime;
GPSPoint_t point;
double altitude;
unsigned char GPS_Satilite_Num;
} GPS_periodic_t;
/* ----------------------- Global --------------------------------*/
FILE *fptr; //Global file pointer
int res = 0;
int g_last = 0;
GPS_periodic_t l_gps_60Sec[GPS_PERIODIC_ARRAY_SIZE];
/* ----------------------- Function defs --------------------------*/
/* At signal interrupt this function is called.
* Flush and close the file. And safly exit the program */
void userSignalInterrupt()
{
fflush(fptr);
fclose(fptr);
res = 1;
exit(0);
}
/* #brief From the array of GPS structs we create a string of the format
* [[lat,long],[lat,long],..]
* #param input The input array of GPS structs
* #param output The output string which will contain lat, long
* #param sz Size left in the output buffer
* #return 0 Successfully completed operation
* 1 Failed / Error
*/
int get_gps60secString(GPS_periodic_t input[GPS_PERIODIC_ARRAY_SIZE],
char *output, size_t sz)
{
int cnt = snprintf(output, sz, "[");
if (cnt < 0 || cnt >= sz)
return 1;
output += cnt;
sz -= cnt;
int i = 0;
for (i = 0; i < GPS_PERIODIC_ARRAY_SIZE; i++) {
cnt = snprintf(output, sz, "[%0.8f,%0.8f]%s",
input[i].point.latitude, input[i].point.longitude,
i + 1 == GPS_PERIODIC_ARRAY_SIZE ? "" : ",");
if (cnt < 0 || cnt >= sz)
return 1;
output += cnt;
sz -= cnt;
}
cnt = snprintf(output, sz, "]");
if (cnt < 0 || cnt >= sz)
return 1;
return 0; // no error
}
/* #brief Create a GPS struct with data for testing. It will populate the
* point field of GPS_periodic_t. Lat starts from 0.0 and increases by 1*10^(-8)
* and Long will dstart at 99.99999999 and dec by 1*10^(-8)
*
* #param o_gps_60sec Output array of GPS structs
*/
void getLastMinGPSdata(GPS_periodic_t *o_gps_60sec)
{
//Fill in GPS related data here
int i = 0;
double latitude = o_gps_60sec[0].point.latitude;
double longitude = o_gps_60sec[0].point.longitude;
for (i = 0; i < 60; i++)
{
o_gps_60sec[i].point.latitude = latitude + (0.00000001 * (float)g_last +
0.00000001 * (float)i);
o_gps_60sec[i].point.longitude = longitude - (0.00000001 * (float)g_last +
0.00000001 * (float)i);
}
g_last = 60;
}
/* #brief Get the GPS data and convert it into a string
* #param trk Track structure with GPS string
*/
int getEvent_Track(Track_json_t *trk)
{
getLastMinGPSdata(l_gps_60Sec);
get_gps60secString(l_gps_60Sec, trk->Gnss60, GPS_STRING_SIZE);
return 0;
}
int main()
{
fptr = fopen("gpsAno.txt", "a");
if (fptr == NULL) {
printf("Error!!\n");
exit(1);
}
//Quit at signal interrupt
signal(SIGINT, userSignalInterrupt);
Track_json_t trk;
memset(&l_gps_60Sec, 0x00, sizeof(GPS_periodic_t) * GPS_PERIODIC_ARRAY_SIZE);
//Init Points to be zero and 99.99999999
int i = 0;
for (i = 0; i < 60; i++) {
l_gps_60Sec[i].point.latitude = 00.00000000;
l_gps_60Sec[i].point.longitude = 99.99999999;
}
do {
memset(&trk, 0, sizeof(Track_json_t));
getEvent_Track(&trk);
//Write to file
fprintf(fptr, "%s", trk.Gnss60);
fflush(fptr);
sleep(1);
} while (res == 0);
//close and exit
fclose(fptr);
return 0;
}
Note: Error was not recreated in the above code.
Because this doesn't have the strcat pitfalls.
I tested this function in the embedded application.
Through this I was able to find that the snprintf returns an error and the string created ended up to be:
[17.42401750,78.46098717],[17.42402083,53.62
It ended there (because of the return 1).
Does this mean that the data which was passed to snprints corrupted? It's a float value. How can it get corrupted?
Solution
The error have not been seen since I changed the sprintf function with one that doesn't directly deal with 64 bits of data.
Here's the function modp_dtoa2
/** \brief convert a floating point number to char buffer with a
* variable-precision format, and no trailing zeros
*
* This is similar to "%.[0-9]f" in the printf style, except it will
* NOT include trailing zeros after the decimal point. This type
* of format oddly does not exists with printf.
*
* If the input value is greater than 1<<31, then the output format
* will be switched exponential format.
*
* \param[in] value
* \param[out] buf The allocated output buffer. Should be 32 chars or more.
* \param[in] precision Number of digits to the right of the decimal point.
* Can only be 0-9.
*/
void modp_dtoa2(double value, char* str, int prec)
{
/* if input is larger than thres_max, revert to exponential */
const double thres_max = (double)(0x7FFFFFFF);
int count;
double diff = 0.0;
char* wstr = str;
int neg= 0;
int whole;
double tmp;
uint32_t frac;
/* Hacky test for NaN
* under -fast-math this won't work, but then you also won't
* have correct nan values anyways. The alternative is
* to link with libmath (bad) or hack IEEE double bits (bad)
*/
if (! (value == value)) {
str[0] = 'n'; str[1] = 'a'; str[2] = 'n'; str[3] = '\0';
return;
}
if (prec < 0) {
prec = 0;
} else if (prec > 9) {
/* precision of >= 10 can lead to overflow errors */
prec = 9;
}
/* we'll work in positive values and deal with the
negative sign issue later */
if (value < 0) {
neg = 1;
value = -value;
}
whole = (int) value;
tmp = (value - whole) * pow10[prec];
frac = (uint32_t)(tmp);
diff = tmp - frac;
if (diff > 0.5) {
++frac;
/* handle rollover, e.g. case 0.99 with prec 1 is 1.0 */
if (frac >= pow10[prec]) {
frac = 0;
++whole;
}
} else if (diff == 0.5 && ((frac == 0) || (frac & 1))) {
/* if halfway, round up if odd, OR
if last digit is 0. That last part is strange */
++frac;
}
/* for very large numbers switch back to native sprintf for exponentials.
anyone want to write code to replace this? */
/*
normal printf behavior is to print EVERY whole number digit
which can be 100s of characters overflowing your buffers == bad
*/
if (value > thres_max) {
sprintf(str, "%e", neg ? -value : value);
return;
}
if (prec == 0) {
diff = value - whole;
if (diff > 0.5) {
/* greater than 0.5, round up, e.g. 1.6 -> 2 */
++whole;
} else if (diff == 0.5 && (whole & 1)) {
/* exactly 0.5 and ODD, then round up */
/* 1.5 -> 2, but 2.5 -> 2 */
++whole;
}
//vvvvvvvvvvvvvvvvvvv Diff from modp_dto2
} else if (frac) {
count = prec;
// now do fractional part, as an unsigned number
// we know it is not 0 but we can have leading zeros, these
// should be removed
while (!(frac % 10)) {
--count;
frac /= 10;
}
//^^^^^^^^^^^^^^^^^^^ Diff from modp_dto2
// now do fractional part, as an unsigned number
do {
--count;
*wstr++ = (char)(48 + (frac % 10));
} while (frac /= 10);
// add extra 0s
while (count-- > 0) *wstr++ = '0';
// add decimal
*wstr++ = '.';
}
// do whole part
// Take care of sign
// Conversion. Number is reversed.
do *wstr++ = (char)(48 + (whole % 10)); while (whole /= 10);
if (neg) {
*wstr++ = '-';
}
*wstr='\0';
strreverse(str, wstr-1);
}
Here's (part of) my unabashedly opinionated guide on safe string handling in C. Normally, I would promote dynamic memory allocation instead of fixed-length strings, but in this case I'm assuming that in the embedded environment that might be problematic. (Although assumptions like that should always be checked.)
So, first things first:
Any function which creates a string in a buffer must be told explicitly how long the buffer is. This is non-negotiable.
As should be obvious, it's impossible for a function filling a buffer to check for buffer overflow unless it knows where the buffer ends. "Hope that the buffer is long enough" is not a viable strategy. "Document the needed buffer length" would be fine if everyone carefully read the documentation (they don't) and if the required length never changes (it will). The only thing that's left is an extra argument, which should be of type size_t (because that's the type of buffer lengths in the C library functions which require lengths).
Forget that strncpy and strncat exist. Also forget about strcat. They are not your friends.
strncpy is designed for a specific use case: ensuring that an entire fixed-length buffer is initialised. It is not designed for normal strings, and since it doesn't guarantee that the output is NUL-terminated, it doesn't produce a string.
If you're going to NUL-terminate yourself anyway, you might as well use memmove, or memcpy if you know that the source and destination don't overlap, which should almost always be the case. Since you'll want the memmove to stop at the end of the string for short strings (which strncpy does not do), measure the string length first with strnlen: strnlen takes a maximum length, which is precisely what you want in the case that you are going move a maximum number of characters.
Sample code:
/* Safely copy src to dst where dst has capacity dstlen. */
if (dstlen) {
/* Adjust to_move will have maximum value dstlen - 1 */
size_t to_move = strnlen(src, dstlen - 1);
/* copy the characters */
memmove(dst, src, to_move);
/* NUL-terminate the string */
dst[to_move] = 0;
}
strncat has a slightly more sensible semantic, but it's practically never useful because in order to use it, you already have to know how many bytes you could copy. In order to know that, in practice, you need to know how much space is left in your output buffer, and to know that you need to know where in the output buffer the copy will start. [Note 1]. But if you already know where the copy will start, what's the point of searching through the buffer from the beginning to find the copy point? And if you do let strncat do the search, how sure are you that your previously computed start point is correct?
In the above code snippet, we already computed the length of the copy. We can extend that to do an append without rescanning:
/* Safely copy src1 and then src2 to dst where dst has capacity dstlen. */
/* Assumes that src1 and src2 are not contained in dst. */
if (dstlen) {
/* Adjust to_move will have maximum value dstlen - 1 */
size_t to_move = strnlen(src1, dstlen - 1);
/* Copy the characters from src1 */
memcpy(dst, src1, to_move);
/* Adjust the output pointer and length */
dst += to_move;
dstlen -= to_move;
/* Now safely copy src2 to just after src1. */
to_move = strnlen(src2, dstlen - 1);
memcpy(dst, src2, to_move);
/* NUL-terminate the string */
dst[to_move] = 0;
}
It might be that we want the original values of dst and dstlen after creating the string, and it might also be that we want to know how many bytes we inserted into dst in all. In that case, we would probably want to make copies of those variables before doing the copies, and save the cumulative sum of moves.
The above assumes that we're starting with an empty output buffer, but perhaps that isn't the case. Since we still need to know where the copy will start in order to know how many characters we can put at the end, we can still use memcpy; we just need to scan the output buffer first to find the copy point. (Only do this if there is no alternative. Doing it in a loop instead of recording the next copy point is Shlemiel the Painter's algorithm.)
/* Safely append src to dst where dst has capacity dstlen and starts
* with a string of unknown length.
*/
if (dstlen) {
/* The following code will "work" even if the existing string
* is not correctly NUL-terminated; the code will not copy anything
* from src, but it will put a NUL terminator at the end of the
* output buffer.
*/
/* Figure out where the existing string ends. */
size_t prefixlen = strnlen(dst, dstlen - 1);
/* Update dst and dstlen */
dst += prefixlen;
dstlen -= prefixlen;
/* Proceed with the append, as above. */
size_t to_move = strnlen(src, dstlen - 1);
memmove(dst, src, to_move);
dst[to_move] = 0;
}
Embrace snprintf. It really is your friend. But always check its return value.
Using memmove, as above, is slightly awkward. It requires you to manually check that the buffer's length is not zero (otherwise subtracting one would be disastrous since the length is unsigned), and it requires you to manually NUL-terminate the output buffer, which is easy to forget and the source of many bugs. It is very efficient, but sometimes it's worth sacrificing a little efficiency so that your code is easier to write and easier to read and verify.
And that leads us directly to snprintf. For example, you can replace:
if (dstlen) {
size_t to_move = strnlen(src, dstlen - 1);
memcpy(dst, src, to_move);
dst[to_move] = 0;
}
with the much simpler
int copylen = snprintf(dst, dstlen, "%s", src);
That does everything: checks that dstlen is not 0; only copies the characters from src which can fit in dst, and correctly NUL-terminates dst (unless dstlen was 0). And the cost is minimal; it takes very little time to parse the format string "%s" and most implementations are pretty well optimised for this case. [Note 2]
But snprintf is not a panacea. There are still a couple of really important warnings.
First, the documentation for snprintf makes clear that it is not permitted for any input argument to overlap the output range. (So it replaces memcpy but not memmove.) Remember that overlap includes NUL-terminators, so the following code which attempts to double the string in str instead leads to Undefined Behaviour:
char str[BUFLEN];
/* Put something into str */
get_some_data(str, BUFLEN);
/* DO NOT DO THIS: input overlaps output */
int result = snprintf(str, BUFLEN, "%s%s", str, str);
/* DO NOT DO THIS EITHER; IT IS STILL UB */
size_t len = strnlen(str, cap - 1);
int result = snprintf(str + len, cap - len, "%s", str);
The problem with the second invocation of snprintf is that the NUL which terminates str is precisely at str + len, the first byte of the output buffer. That's an overlap, so it's illegal.
The second important note about snprintf is that it returns a value, which must not be ignored. The value returned is not the length of the string created by snprintf. It's the length the string would have been had it not been truncated to fit in the output buffer.
If no truncation occurred, then the result is the length of the result, which must be strictly less than the size of the output buffer (because there must be room for a NUL terminator, which is not considered part of the length of the result.) You can use this fact to check whether truncation occurred:
if (result >= dstlen) /* Output was truncated */
This can be used, for example, to redo the snprintf with a larger, dynamically-allocated buffer (of size result + 1; never forget the need to NUL-terminate).
But remember that the result is an int -- that is, a signed value. That means that snprintf cannot cope with very long strings. That's not likely to be an issue in embedded code, but on systems where it's conceivable that strings exceed 2GB, you may not be able to safely use %s formats in snprintf. It also means that snprintf is allowed to return a negative value to indicate an error. Very old implementations of snprintf returned -1 to indicate truncation, or in response to being called with buffer length 0. That's not standard behaviour according to C99 (nor recent versions of Posix), but you should be prepared for it.
Standard-compliant implementations of snprintf will return a negative value if the buffer length argument is too big to fit in a (signed) int; it's not clear to me what the expected return value is if the buffer length is OK but the untruncated length is too big for an int. A negative value will also be returned if you used a conversion which resulted in an encoding error; for example, a %lc conversion whose corresponding argument contains an integer which cannot be converted to a multibyte (typically UTF-8) sequence.
In short, you should always check the return value of snprintf (recent gcc/glibc versions will produce a warning if you do not), and you should be prepared for it to be negative.
So, with all that behind us, let's write a function which produces a string of co-ordinate pairs:
/* Arguments:
* buf the output buffer.
* buflen the capacity of buf (including room for trailing NUL).
* points a vector of struct Point pairs.
* npoints the number of objects in points.
* Description:
* buf is overwritten with a comma-separated list of points enclosed in
* square brackets. Each point is output as a comma-separated pair of
* decimal floating point numbers enclosed in square brackets. No more
* than buflen - 1 characters are written. Unless buflen is 0, a NUL is
* written following the (possibly-truncated) output.
* Return value:
* If the output buffer contains the full output, the number of characters
* written to the output buffer, not including the NUL terminator.
* If the output was truncated, (size_t)(-1) is returned.
*/
size_t sprint_points(char* buf, size_t buflen,
struct Point const* points, size_t npoints)
{
if (buflen == 0) return (size_t)(-1);
size_t avail = buflen;
char delim = '['
while (npoints) {
int res = snprintf(buf, avail, "%c[%f,%f]",
delim, points->lat, points->lon);
if (res < 0 || res >= avail) return (size_t)(-1);
buf += res; avail -= res;
++points; --npoints;
delim = ',';
}
if (avail <= 1) return (size_t)(-1);
strcpy(buf, "]");
return buflen - (avail - 1);
}
Notes
You will often see code like this:
strncat(dst, src, sizeof(src)); /* NEVER EVER DO THIS! */
Telling strncat not to append more characters from src than can fit in src is obviously pointless (unless src is not correctly NUL-terminated, in which case you have a bigger problem). More importantly, it does absolutely nothing to protect you from writing beyond the end of the output buffer, since you have not done anything to check that dst has room for all those characters. So about all it does is get rid of compiler warnings about the unsafety of strcat. Since this code is exactly as unsafe as strcat was, you probably would be better off with the warning.
You might even find a compiler which understands snprintf will enough to parse the format string at compile time, so the convenience comes at no cost at all. (And if your current compiler doesn't do this, no doubt a future version will.) As with any use of the *printf family, you should never try to economize keystrokes by
leaving out the format string (snprintf(dst, dstlen, src) instead of snprintf(dst, dstlen, "%s", src).) That's unsafe (it has undefined behaviour if src contains an unduplicated %). And it's much slower because the library function has to parse the entire string to be copied looking for percent signs, instead of just copying it to the output.
Code is using functions that expect pointers to string, yet not always passing pointers to strings as arguments.
Stray characters seen at output of snprintf
A string must have a terminating null character.
strncat(char *, .... expects the first parameter to be a pointer to a string. memcpy(output, "[",1); does not insure that. #Jeremy
memcpy(output, "[",1);
...
strncat(output, temp,sizeof(temp));
This is a candidate source of stray characters.
strncat(...., ..., size_t size). itself is a problem as the size is the amount of space available for concatenating (minus the null character). The size available to char * output is not passed in. #Jonathan Leffler. Might as well do strcat() here.
Instead, pass in the size available to output to prevent buffer overflow.
#define N 60
int get_gps60secString(GPS_periodic_t input[N], char *output, size_t sz) {
int cnt = snprintf(output, sz, "[");
if (cnt < 0 || cnt >= sz)
return 1;
output += cnt;
sz -= cnt;
int i = 0;
for (i = 0; i < N; i++) {
cnt = snprintf(output, size, "[%0.8f,%0.8f]%s", input[i].point.latitude,
input[i].point.longitude, i + 1 == N ? "" : ",");
if (cnt < 0 || cnt >= sz)
return 1;
output += cnt;
sz -= cnt;
}
cnt = snprintf(output, sz, "]");
if (cnt < 0 || cnt >= sz)
return 1;
return 0; // no error
}
OP has posted more code - will review.
Apparently the buffer char *output is pre-filled with 0 before the get_gps60secString() so the missing null character from memcpy(output, "[",1); should not cause the issue - hmmmmmm
unsigned short SendTrack() does not return a value. 1) Using its result value is UB. 2) Enable all compiler warnings.
Closed. This question needs details or clarity. It is not currently accepting answers.
Want to improve this question? Add details and clarify the problem by editing this post.
Closed 5 years ago.
Improve this question
I understand the general idea of C and how making a log file would go. Reading/writing to a file and such.
My concern is the following format that is desired:
[![enter image description here][1]][1]
I've gotten a good chunk done now but am concerned with how to append to my log file after the first record. I increment the file's record count (in the top 2 bytes) and write the first record after it. How would I then setup to add the 2nd/3rd/etc records to showup after each other?
//confirm a file exists in the directory
bool fileExists(const char* file)
{
struct stat buf;
return (stat(file, &buf) == 0);
}
int rightBitShift(int val, int space)
{
return ((val >> space) & 0xFF);
}
int leftBitShift(int val, int space)
{
return (val << space);
}
int determineRecordCount(char * logName)
{
unsigned char record[2];
FILE *fp = fopen(logName, "rb");
fread(record, sizeof(record), 1, fp);
//display the record number
int recordNum = (record[0] << 8) | record[1];
recordNum = recordNum +1;
return (recordNum);
}
void createRecord(int argc, char **argv)
{
int recordNum;
int aux = 0;
int dst;
char* logName;
char message[30];
memset(message,' ',30);
//check argument count and validation
if (argc == 7 && strcmp("-a", argv[2]) ==0 && strcmp("-f", argv[3]) ==0 && strcmp("-t", argv[5]) ==0)
{
//aux flag on
aux = 1;
logName = argv[4];
strncpy(message, argv[6],strlen(argv[6]));
}
else if (argc == 6 && strcmp("-f", argv[2]) ==0 && strcmp("-t", argv[4]) ==0)
{
logName = argv[3];
strncpy(message, argv[5],strlen(argv[5]));
}
else
{
printf("Invalid Arguments\n");
exit(0);
}
//check if log exists to get latest recordNum
if (fileExists(logName))
{
recordNum = determineRecordCount(logName);
printf("%i\n",recordNum);
}
else
{
printf("Logfile %s not found\n", logName);
recordNum = 1;
}
//Begin creating record
unsigned char record[40]; /* One record takes up 40 bytes of space */
memset(record, 0, sizeof(record));
//recordCount---------------------------------------------------------------------
record[0] = rightBitShift (recordNum, 8); /* Upper byte of sequence number */
record[1] = rightBitShift (recordNum, 0); /* Lower byte of sequence number */
//get aux/dst flags---------------------------------------------------------------
//get date and time
time_t timeStamp = time(NULL);
struct tm *date = localtime( &timeStamp );
if (date->tm_isdst)
dst = 1;
record[2] |= aux << 7; //set 7th bit
record[2] |= dst << 6; //set 6th
//timeStamp-----------------------------------------------------------------------
record[3] |= rightBitShift(timeStamp, 24);//high byte
record[4] |= rightBitShift(timeStamp, 16);
record[5] |= rightBitShift(timeStamp, 8);
record[6] |= rightBitShift(timeStamp, 0); //low byte
//leave bytes 7-8, set to 0 -----------------------------------------
record[7] = 0;
record[8] = 0;
//store message--------------------------------------------
strncpy(&record[9], message, strlen(message));
//write record to log-----------------------------------------------------------------
FILE *fp = fopen(logName, "w+");
unsigned char recordCount[4];
recordCount[0] = rightBitShift (recordNum, 8); /* Upper byte of sequence number */
recordCount[1] = rightBitShift (recordNum, 0); /* Lower byte of sequence number */
recordCount[2] = 0;
recordCount[3] = 0;
fwrite(recordCount, sizeof(recordCount), 1, fp);
fwrite(record, sizeof(record), 1, fp);
fclose(fp);
printf("Record saved successfully\n");
}
NOTE: I've never had to do this before in C, take it with a grain of salt.
This is a very specific binary formatting where each bit is precisely accounted for. It's using the Least-Significant-Bit numbering scheme (LSB 0) where the bits are numbered from 7 to 0.
Specifying that the "upper byte" comes first means this format is big-endian. The most significant bits come first. This is like how we write our numbers, four thousand, three hundred, and twenty one is 4321. 1234 would be little-endian. For example, the Number Of Records and Sequence are both 16 bit big-endian numbers.
Finally, the checksum is a number calculated from the rest of the record to verify there were no mistakes in transmission. The spec defines how to make the checksum.
Your job is to precisely reproduce this format, probably using the fixed-sized types found in stdint.h or unsigned char. For example, the sequence would be a uint16_t or unsigned char[2].
The function to produce a record might have a signature like this:
unsigned char *make_record( const char *message, bool aux );
The user only has to supply you with the message and the aux flag. The rest you can be figured out by the function. You might decide to let them pass in the timestamp and sequence. Point is, the function needs to be passed just the data, it takes care of the formatting.
This byte-ordering means you can't just write out integers, they might be the wrong size or the wrong byte order. That means any multi-byte integers must be serialized before you can write them to the record. This answer covers ways to do that and I'll be using the ones from this answer because they proved a bit more convenient.
#include <stdio.h>
#include <stdint.h>
#include <time.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
unsigned char *make_record( const char *message, bool aux ) {
// Allocate and zero memory for the buffer.
// Zeroing means no risk of accidentally sending garbage.
unsigned char *buffer = calloc( 40, sizeof(unsigned char) );
// As we add to the buffer, pos will track the next byte to be written.
unsigned char *pos = buffer;
// I decided not make the user responsible for
// the sequence number. YMMV.
static uint16_t sequence = 1;
pos = serialize_uint16( pos, sequence );
// Get the timestamp and DST.
time_t timestamp = time(NULL);
struct tm *date = localtime( ×tamp );
// 2nd row is all flags and a bunch of 0s. Start with them all off.
uint8_t flags = 0;
if( aux ) {
// Flip the 7th bit on.
flags |= 0x80;
}
if( date->tm_isdst ) {
// Flip the 6th bit on.
flags |= 0x40;
}
// That an 8 bit integer has no endianness, this is to ensure
// pos is consistently incremented.
pos = serialize_uint8(pos, flags);
// I don't know what their timestamp format is.
// This is just a guess. It's probably wrong.
pos = serialize_uint32(pos, (uint32_t)timestamp);
// "Spare" is all zeros.
// The spec says this is 3 bytes, but only gives it bytes
// 7 and 8. I'm going with 2 bytes.
pos = serialize_uint16(pos, 0);
// Copy the message in, 30 bytes.
// strncpy() does not guarantee the message will be null
// terminated. This is probably fine as the field is fixed width.
// More info about the format would be necessary to know for sure.
strncpy( pos, message, 30 );
pos += 30;
// Checksum the first 39 bytes.
// Sorry, I don't know how to do 1's compliment sums.
pos = serialize_uint8( pos, record_checksum( buffer, 39 ) );
// pos has moved around, but buffer remains at the start
return buffer;
}
int main() {
unsigned char *record = make_record("Basset hounds got long ears", true);
fwrite(record, sizeof(unsigned char), 40, stdout);
}
At this point my expertise is exhausted, I've never had to do this before. I'd appreciate folks fixing up the little mistakes in edits and suggesting better ways to do it in the comments, like what to do with the timestamp. And maybe someone else can cover how to do 1's compliment checksums in another answer.
As a byte is composed by 8 bits (from 0 to 7) you can use bitwise operations to modify them as asked in your specifications. Take a look for general information (https://en.wikipedia.org/wiki/Bitwise_operations_in_C). As a preview, you can use >> or << operators to determine which bit to modify, and use logical operators | and & to set it's values.
I know about fscanf(), fgets() and those other functions to read the next line of a text file. However, if you are given a text file by 'cat msg1.txt | ./anonymizer' would you use the same functions?
For my program the code for the main is:
int main (void)
{
char input[1000]= {'\0'}; //the sentence the user will enter
printf("Enter a sentence:");
scanf("%[^\n]", input);
char newSentence[1000]={'\0'};
sentence=(char *) &newSentence;
line=getText(input,0);
divide(input);
printf("%s\n",sentence);
return 0;
}
In the command line I enter:
gcc -o anonymizer anonymizer.c
cat msg1.txt | ./anonymizer
My msg1 text file contains:
Hi, my email addresses are h.potter#hogwarts.edu and 1a#2b3c#lkj#
Although it's not an email addresses, I'd hate if# you saw my
secret#word. Gary.zenkel#nbcuni.comHoever, input variable only
contains the first line: 'Hi, my email addresses are
h.potter#hogwarts.edu and 1a#2b3c#lkj#'
How can I get the input variable to contain the other two lines?
Almost. While it may not actually be defined in such a way, scanf(...) is essentially equivalent to fscanf(stdin, ...). Similar for gets/fgets. You should be able to use either to read from your standard input stream.
To my limited knowledge (I could be wrong), with the standard libc, there are no efficient ways to read a line when you do not know the max line length. You may get memory overflow with scanf() and gets() because they do not check the length of your buffer. If you use fgets(), you may waste time on frequent strlen() and realloc(). If you use fgetc(), it will be slow as fgetc() has a huge overhead.
For efficient line reading, we have to keep some intermediate information. It is not that easy. I am attaching an implementation. It is quite complicated, but it is very efficient and generic. If you do not care about the details, you may just focus on the main() function about how to use the routines.
To try this program:
gcc -Wall prog.c; ./a.out < input.txt > output.txt
Program:
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#ifndef kroundup32
#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
#endif
#define kstype_t FILE* // type of file handler
#define ksread_f(fp, buf, len) fread((buf), 1, (len), (fp)) // function to read a data chunk
typedef struct {
int l, m; // l: length of string; m: allocated size
char *s; // string
} kstring_t;
typedef struct {
kstype_t f; // file handler
int begin, end, is_eof, bufsize;
unsigned char *buf; // buffer
} kstream_t;
kstream_t *ks_open(kstype_t fp, int bufsize)
{
kstream_t *ks;
ks = (kstream_t*)calloc(1, sizeof(kstream_t));
ks->bufsize = bufsize;
ks->buf = (unsigned char*)malloc(bufsize);
ks->f = fp;
return ks;
}
void ks_close(kstream_t *ks)
{
free(ks->buf); free(ks);
}
int ks_readline(kstream_t *ks, int delimiter, kstring_t *str)
{
str->l = 0;
if (ks->begin >= ks->end && ks->is_eof) return -1;
for (;;) {
int i;
if (ks->begin >= ks->end) {
if (!ks->is_eof) {
ks->begin = 0;
ks->end = ksread_f(ks->f, ks->buf, ks->bufsize);
if (ks->end < ks->bufsize) ks->is_eof = 1;
if (ks->end == 0) break;
} else break;
}
for (i = ks->begin; i < ks->end; ++i)
if (ks->buf[i] == delimiter) break;
if (str->m - str->l < i - ks->begin + 1) {
str->m = str->l + (i - ks->begin) + 1;
kroundup32(str->m);
str->s = (char*)realloc(str->s, str->m);
}
memcpy(str->s + str->l, ks->buf + ks->begin, i - ks->begin);
str->l = str->l + (i - ks->begin);
ks->begin = i + 1;
if (i < ks->end) break;
}
if (str->s == 0) {
str->m = 1;
str->s = (char*)calloc(1, 1);
}
str->s[str->l] = '\0';
return str->l;
}
int main()
{
kstream_t *ks;
kstring_t str;
str.l = str.m = 0; str.s = 0; // initialize the string struct
ks = ks_open(stdin, 4096); // initialize the file handler
while (ks_readline(ks, '\n', &str) >= 0) // read each line
puts(str.s); // print it out
ks_close(ks); free(str.s); // free
return 0;
}
#include<reg51.h>
#include<string.h>
#include"_LCD_R8C.c"
unsigned char c[12];
unsigned char chr[11];
void serial_int (void) interrupt 4
{
if (RI==1)
{
chr[11] = SBUF;
RI = 0;
TI = 0;
}
}
int main()
{
unsigned char a[2][11]={"$0016221826","$0123456789"};
int i,j;
lcd_init();
lcd_clear();
SCON = 0x50;
TMOD = 0x20;
TH1 = 0xFD;
ET0 = 0;
TR1 = 1;
RI = 1;
ES = 1;
EA = 1;
for(j=0;j<1;j++)
{
for(i=0;i<=10;i++)
{
c[i]=chr[i];
}
c[11]='\0';
}
for(i=0;i<=1;i++)
{
j=strcmp(a[i],c); /* !!! Here is the problem !!! */
if(j==0)
{
lcd_printxy(1,1,"yes");
}
else
{
lcd_printxy(1,6,"no");
}
}
}
I am getting the display as "no", please let me know what is the problem?
the problem might be
1) the received array of characters are not converted to string, or
2) the received array of characters are converted to string but not able to compare with the available string..
please go through the program
One obvious bug for starters - change:
unsigned char a[2][11]={"$0016221826","$0123456789"};
to:
unsigned char a[2][12]={"$0016221826","$0123456789"};
(You need to allow room for the terminating '\0' in each string - I'm surprised your compiler didn't complain about this ?)
Also, this line in your interrupt handler is wrong:
chr[11] = SBUF;
Several problems with this - char only has storage for 11 chars, not 12, and you probably want to be accumulating characters from index 0 and then bumping the index, otherwise you're just overwriting the same character each time.
Looking at the rest of the code there are so many other problems that I think you may need to take a step back here and start with a simpler program - get that working first and then add to it in stages.
You might also want to get a decent introductory book on C and study it as there are lots of very basic mistakes in the code, so you might benefit from a better understanding of the language itself.
You only assign a value to chr[11], the rest of the array is uninitialized and will contain random data. You then copy this array containing random data to c (you could use e.g. memcpy here instead of looping yourself), and finally you compare the complete contents of c (which is random data) with one of the entries in a. So it's kind of natural that the result of that comparison will be that the strings are not equal.
Edit: A redesign of the program in the question
Your program has too many problems to be easily fixed, so I decided to try and rewrite it:
#include <reg51.h>
#include <string.h>
#include "_LCD_R8C.c"
#define INPUT_LENGTH 11
#define ACCEPTABLE_INPUT_COUNT 2
char input[INPUT_LENGTH]; /* The input from the serial port */
int input_pos = 0; /* Current position to write in the input buffer */
int input_done = 0; /* 0 = not done yet, 1 = all input read */
void serial_int (void) interrupt 4
{
if (!input_done && RI == 1)
{
/* Put the input at next position in the input buffer */
/* Then increase the position */
input[input_pos++] = SBUF;
RI = 0;
TI = 0;
/* Check if we have received all input yet */
if (input_pos >= INPUT_LENGTH)
input_done = 1;
}
}
int main()
{
/* Array of data that this programs thinks is acceptable */
/* +1 to the input length, to fit the terminating '\0' character */
char acceptable_inputs[ACCEPTABLE_INPUT_COUNT][INPUT_LENGTH + 1] = {
"$0016221826", "$0123456789"
};
iny acceptable_found = 0; /* Acceptable input found? */
/* Initialization */
lcd_init();
lcd_clear();
SCON = 0x50;
TMOD = 0x20;
TH1 = 0xFD;
ET0 = 0;
TR1 = 1;
RI = 1;
ES = 1;
EA = 1;
/* Wait until we have received all input */
while (!input_done)
; /* Do nothing */
/* Check the received input for something we accept */
for (int i = 0; i < ACCEPTABLE_INPUT_COUNT; i++)
{
if (memcmp(acceptable_inputs[i], input, INPUT_LENGTH) == 0)
{
/* Yes, the data received is acceptable */
acceptable_found = 1;
break; /* Don't have to check any more */
}
}
if (acceptable_found)
lcd_printxy(1, 1, "Yes");
else
lcd_printxy(1, 1, "No");
return 0;
}