I want to reduce memory storage of chars using Bitwise Operations,
for Example
input: {"ACGT"}
output: {"0xE4"}
where we represtnt the number in binary then into Hexadecimal
if A=00, C=01, G=10, T=11
so ACGT = 0xE4 = 11100100b
I cant Figure the whole way so here is what I did So Far
enum NucleicAc {
A = 0,
C = 1,
G = 2,
T = 3,
} ;
struct _DNA {
char * seq ;
};
DNAString DSCreate(char * mseq) {
DNAString dna = malloc(sizeof(struct _DNA));
if (!dna){
exit(-1);
}
const int length = strlen(mseq);
// left shift will create the base , in the case of ACGT --> 0000,0000
int base = 0 << length * 2;
//loop and set bits
int i = 0 ;
int k = 0 ; // the counter where i want to modify the current bit
//simple looping gor the string
for ( ; i < length ; i++ ) {
switch (*(mseq+i)) {
case 'A': // 00
k++;
break;
case 'C': // 0 1
modifyBit(&base, k, 1);
k++;
break;
case 'G': //10
k++;
modifyBit(&base,k , 1);
break;
case 'T': // 11
modifyBit(&base, k, 1);
k++;
modifyBit(&base, k,1);
break;
default:
break;
} //end of switch
k++;
}//end of for
char * generatedSeq ;
//convert the base to hex ??
return dna;
}
void bin(unsigned n){
unsigned i;
for (i = 1 << 7; i > 0; i = i / 2){
(n & i) ? printf("1") : printf("0");
}
}
and if we print the base, the value is 11100100b as expected,
how to store the hexadecimal representation as String to the char *mseq in the struct ?
any direction or exact solutions or is there any better approach?
and also later i want to get the letter using only the index for Example
DSGet(dsStruct , '0')--> will return 'A' hence dsStruct contains the "ACGT" before Encode?
There are several ways you can approach encoding the sequence. Your enum is fine, but for your encoded sequence, a struct that captures the bytes as unsigned char, the original sequence length and the encoded size in bytes will allow an easy decoding. You will get 4-to-1 compression (plus 1 byte if you sequence isn't evenly divisible by 4). The enum and struct could be:
enum { A, C, G, T };
typedef struct {
unsigned char *seq;
size_t len, size;
} encoded;
To map characters in your string to the encoded values a simple function that returns the enum value matching the character is all you need (don't forget to handle any errors)
/* convert character to encoded value */
unsigned char getencval (const char c)
{
if (c == 'A')
return A;
else if (c == 'C')
return C;
else if (c == 'G')
return G;
else if (c == 'T')
return T;
/* exit on anything other than A, C, G, T */
fprintf (stderr, "error: invalid sequence character '%c'\n", c);
exit (EXIT_FAILURE);
}
To encode the original sequence, you will populate the encoded struct with the original length (len) and number of bytes (size) needed to hold the encoded string. Don't forget that any 1-character of the next 4-characters will require another byte of storage. You can use a simple add-and-divide to account for any partial 4-character ending portion of the sequence, e.g.
/* encode sequence of characters as 2-bit pairs (4-characters per-byte)
* returns encoded struct with allocated .seq member, on failure the .seq
* member is NULL. User is resposible for freeing .seq member when done.
*/
encoded encode_seq (const char *seq)
{
size_t len = strlen(seq),
size = (len + 3) / 4; /* integer division intentional */
encoded enc = { .seq = calloc (1, size), /* encoded sequence struct */
.len = len,
.size = size };
if (!enc.seq) { /* validate allication */
perror ("calloc-enc.seq");
return enc;
}
/* loop over each char (i) with byte index (ndx)
* shifting each 2-bit pair by (shift * 2) amount.
*/
for (int i = 0, ndx = 0, shift = 4; seq[i] && seq[i] != '\n'; i++) {
if (!shift--) /* decrement shift, reset if 0 */
shift = 3;
if (i && i % 4 == 0) /* after each 4th char, increment ndx */
ndx += 1;
/* shift each encoded value (multiply by 2 for shift of 6, 4, 2, 0) */
enc.seq[ndx] |= getencval (seq[i]) << shift * 2;
}
return enc; /* return encoded struct with allocated .seq member */
}
To get the original sequence back from your encoded struct, the use of a lookup table (shown below with the full code) makes it a breeze. You simply loop over all stored byte values appending the corresponding strings from the lookup table until the final byte. For the final byte, you need to determine if it is a partial string and, if so, how many characters remain to copy. (that's why you store the original sequence length in your struct). Then simply use strncat to append that many characters from the final byte, e.g.
/* decodes encoded sequence. Allocates storage for decoded sequence
* and loops over each encoded byte using lookup-table to obtain
* original 4-character string from byte value. User is responsible
* for freeing returned string when done. Returns NULL on allocation
* failure.
*/
char *decode_seq (encoded *eseq)
{
char *seq = malloc (eseq->len + 1); /* allocate storage for sequence */
size_t i = 0, offset = 0, remain;
if (!seq) { /* validate allocation */
perror ("malloc-seq");
return NULL;
}
/* loop appending strings from lookup table for all but last byte */
for (; i < eseq->size - 1; i++) {
memcpy (seq + offset, lookup[eseq->seq[i]], 4);
offset += 4; /* increment offset by 4 */
}
/* determine the number of characters in last byte */
remain = eseq->len - (eseq->size - 1) * 4;
memcpy (seq + offset, lookup[eseq->seq[i]], remain);
seq[offset + remain] = 0; /* nul-terminate seq */
return seq; /* return allocated sequence */
}
Adding the lookup table and putting all the pieces together, one way to approach this problem is:
(edit: lookup table reordered to match your byte-value encoding, optimized decode_seq() to not scan for end-of-string on copy)
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
enum { A, C, G, T };
typedef struct {
unsigned char *seq;
size_t len, size;
} encoded;
const char lookup[][5] = {
"AAAA","CAAA","GAAA","TAAA","ACAA","CCAA","GCAA","TCAA",
"AGAA","CGAA","GGAA","TGAA","ATAA","CTAA","GTAA","TTAA",
"AACA","CACA","GACA","TACA","ACCA","CCCA","GCCA","TCCA",
"AGCA","CGCA","GGCA","TGCA","ATCA","CTCA","GTCA","TTCA",
"AAGA","CAGA","GAGA","TAGA","ACGA","CCGA","GCGA","TCGA",
"AGGA","CGGA","GGGA","TGGA","ATGA","CTGA","GTGA","TTGA",
"AATA","CATA","GATA","TATA","ACTA","CCTA","GCTA","TCTA",
"AGTA","CGTA","GGTA","TGTA","ATTA","CTTA","GTTA","TTTA",
"AAAC","CAAC","GAAC","TAAC","ACAC","CCAC","GCAC","TCAC",
"AGAC","CGAC","GGAC","TGAC","ATAC","CTAC","GTAC","TTAC",
"AACC","CACC","GACC","TACC","ACCC","CCCC","GCCC","TCCC",
"AGCC","CGCC","GGCC","TGCC","ATCC","CTCC","GTCC","TTCC",
"AAGC","CAGC","GAGC","TAGC","ACGC","CCGC","GCGC","TCGC",
"AGGC","CGGC","GGGC","TGGC","ATGC","CTGC","GTGC","TTGC",
"AATC","CATC","GATC","TATC","ACTC","CCTC","GCTC","TCTC",
"AGTC","CGTC","GGTC","TGTC","ATTC","CTTC","GTTC","TTTC",
"AAAG","CAAG","GAAG","TAAG","ACAG","CCAG","GCAG","TCAG",
"AGAG","CGAG","GGAG","TGAG","ATAG","CTAG","GTAG","TTAG",
"AACG","CACG","GACG","TACG","ACCG","CCCG","GCCG","TCCG",
"AGCG","CGCG","GGCG","TGCG","ATCG","CTCG","GTCG","TTCG",
"AAGG","CAGG","GAGG","TAGG","ACGG","CCGG","GCGG","TCGG",
"AGGG","CGGG","GGGG","TGGG","ATGG","CTGG","GTGG","TTGG",
"AATG","CATG","GATG","TATG","ACTG","CCTG","GCTG","TCTG",
"AGTG","CGTG","GGTG","TGTG","ATTG","CTTG","GTTG","TTTG",
"AAAT","CAAT","GAAT","TAAT","ACAT","CCAT","GCAT","TCAT",
"AGAT","CGAT","GGAT","TGAT","ATAT","CTAT","GTAT","TTAT",
"AACT","CACT","GACT","TACT","ACCT","CCCT","GCCT","TCCT",
"AGCT","CGCT","GGCT","TGCT","ATCT","CTCT","GTCT","TTCT",
"AAGT","CAGT","GAGT","TAGT","ACGT","CCGT","GCGT","TCGT",
"AGGT","CGGT","GGGT","TGGT","ATGT","CTGT","GTGT","TTGT",
"AATT","CATT","GATT","TATT","ACTT","CCTT","GCTT","TCTT",
"AGTT","CGTT","GGTT","TGTT","ATTT","CTTT","GTTT","TTTT"};
/* convert character to encoded value */
unsigned char getencval (const char c)
{
if (c == 'A')
return A;
else if (c == 'C')
return C;
else if (c == 'G')
return G;
else if (c == 'T')
return T;
/* exit on anything other than A, C, G, T */
fprintf (stderr, "error: invalid sequence character '%c'\n", c);
exit (EXIT_FAILURE);
}
/* encode sequence of characters as 2-bit pairs (4-characters per-byte)
* returns encoded struct with allocated .seq member, on failure the .seq
* member is NULL. User is resposible for freeing .seq member when done.
*/
encoded encode_seq (const char *seq)
{
size_t len = strlen(seq),
size = (len + 3) / 4; /* integer division intentional */
encoded enc = { .seq = calloc (1, size), /* encoded sequence struct */
.len = len,
.size = size };
if (!enc.seq) { /* validate allication */
perror ("calloc-enc.seq");
return enc;
}
/* loop over each char (i) with byte index (ndx)
* shifting each 2-bit pair by (shift * 2) amount.
*/
for (int i = 0, ndx = 0, shift = 0; seq[i] && seq[i] != '\n'; i++, shift++) {
if (shift == 4) /* reset to 0 */
shift = 0;
if (i && i % 4 == 0) /* after each 4th char, increment ndx */
ndx += 1;
/* shift each encoded value (multiply by 2 for shift of 0, 2, 4, 6) */
enc.seq[ndx] |= getencval (seq[i]) << shift * 2;
}
return enc; /* return encoded struct with allocated .seq member */
}
/* decodes encoded sequence. Allocates storage for decoded sequence
* and loops over each encoded byte using lookup-table to obtain
* original 4-character string from byte value. User is responsible
* for freeing returned string when done. Returns NULL on allocation
* failure.
*/
char *decode_seq (encoded *eseq)
{
char *seq = malloc (eseq->len + 1); /* allocate storage for sequence */
size_t i = 0, offset = 0, remain;
if (!seq) { /* validate allocation */
perror ("malloc-seq");
return NULL;
}
/* loop appending strings from lookup table for all but last byte */
for (; i < eseq->size - 1; i++) {
memcpy (seq + offset, lookup[eseq->seq[i]], 4);
offset += 4; /* increment offset by 4 */
}
/* determine the number of characters in last byte */
remain = eseq->len - (eseq->size - 1) * 4;
memcpy (seq + offset, lookup[eseq->seq[i]], remain);
seq[offset + remain] = 0; /* nul-terminate seq */
return seq; /* return allocated sequence */
}
/* short example program that takes string to encode as 1st argument
* using "ACGT" if no argument is provided by default
*/
int main (int argc, char **argv) {
char *seq = NULL;
encoded enc = encode_seq(argc > 1 ? argv[1] : "ACGT");
if (!enc.seq) /* validate encoded allocation */
return 1;
/* output original string, length and encoded size */
printf ("encoded str : %s\nencoded len : %zu\nencoded size : %zu\n",
argc > 1 ? argv[1] : "ACGT", enc.len, enc.size);
/* loop outputting byte-values of encoded string */
fputs ("encoded seq :", stdout);
for (size_t i = 0; i < enc.size; i++)
printf (" 0x%02x", enc.seq[i]);
putchar ('\n');
seq = decode_seq (&enc); /* decode seq from byte values */
printf ("decoded seq : %s\n", seq); /* output decoded string */
free (seq); /* don't forget to free what you allocated */
free (enc.seq);
}
In most cases a lookup-table provides a great deal of efficiency advantage compared to computing and building each 4-character string during decoding. This is enhanced by the lookup table staying resident in cache for most cases.
The length of the DNA sequence you can encode and decode is limited only by the amount of virtual memory you have available.
Example Use/Output
The program takes the sequence to encode and decode as the first argument (default "ACGT"). So the default output is:
$ ./bin/dnaencodedecode
encoded str : ACGT
encoded len : 4
encoded size : 1
encoded seq : 0xe4
decoded seq : ACGT
4-byte encoded in 1-byte. Note the byte value of 0x1b and not 0xe4 due to the table ordering.
A longer example:
./bin/dnaencodedecode ACGTGGGTCAGACTTA
encoded str : ACGTGGGTCAGACTTA
encoded len : 16
encoded size : 4
encoded seq : 0xe4 0xea 0x21 0x3d
decoded seq : ACGTGGGTCAGACTTA
16-character encoded in 4-bytes.
Finally, what of a sequence that isn't divisible by 4 so you have a partial number of characters in the last encoded byte? That is handled as well, e.g.
$ ./bin/dnaencodedecode ACGTGGGTCAGACTTAG
encoded str : ACGTGGGTCAGACTTAG
encoded len : 17
encoded size : 5
encoded seq : 0xe4 0xea 0x21 0x3d 0x02
decoded seq : ACGTGGGTCAGACTTAG
17 characters encoded in 5-bytes. (not the pure 4-to-1 compression, but as the sequence size increases, the significance of any partial group of characters in the last byte becomes negligible)
As far a perfomance, for a sequence of 100,000 characters and output of the the byte values and strings replaced with a simple loop that compares the decoded seq to the original argv[1] it only takes a few thousandth of a second (on an old i7 Gen2 laptop with SSD) to encode and decode and validate, e.g.
$
time ./bin/dnaencodedecodet2big $(< dat/dnaseq100k.txt)
encoded len : 100000
encoded size : 25000
all tests passed
real 0m0.014s
user 0m0.012s
sys 0m0.003s
There are a lot of ways to do this, but given your description, this was what came to my mind that you were trying to accomplish. There is a lot here, so take your time going through it.
Look things over (the code is commented), and let me know if you have further questions. Just drop a comment below.
This should do what you want. I thought the compression was a pretty cool idea so I wrote this real quick. As mentioned by #kaylum, hex encoding is just a way to read the underlying data in memory, which is always just bits. So, you only need to worry about that on print statements.
Let me know if this works or you have any questions about what I did.
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
typedef struct {
unsigned char *bits;
unsigned long length; // use this to store the number of letters encoded, for safety
} DNA;
typedef enum {
A = 0,
C = 1,
G = 2,
T = 3
} NucleicAc;
This returns the base at a given index with some bounds checking
char base_at_index(DNA *dna, unsigned long index) {
if (index >= dna->length) {
fputs("error: base_at_index: index out of range", stderr);
exit(EXIT_FAILURE);
}
// offset is index / 4, this gives us the correct byte
// shift amount is index % 4 to give us the correct 2 bits within the byte.
// This must then be multiplied by 2 because
// each base takes 2 bits to encode
// then we have to bitwise-and this value with
// 3 (0000 0011 in binary) to retrieve the bits we want.
// so, the formula we need is
// (dna->bits[index / 4] >> (2 * (index % 4))) & 3
switch((dna->bits[index / 4] >> 2 * (index % 4)) & 3) {
case A: return 'A';
case C: return 'C';
case G: return 'G';
case T: return 'T';
default:
fputs("error: base_at_index: invalid encoding", stderr);
exit(EXIT_FAILURE);
}
}
This encodes a string of bases to bytes
/* you can fit four 2-bit DNA codes in each byte (unsigned char).
len is the maximum number of characters to read. result must be at least len bytes long
*/
void encode_dna(unsigned char *result, char *sequence, unsigned long len) {
// keep track of what byte we are on in the result
unsigned result_index = 0;
// our shift for writing to the correct position in the byte
unsigned shift = 0;
// first clear result or else bitwise operations will produce errors
// this could be removed if you were certain result parameter was zero-filled
memset(result, 0, len);
// iterate through characters of the sequence
while(*sequence) {
switch (*sequence) {
// do nothing for 'A' since it is just zero
case 'A': break;
case 'C':
// we are doing a bitwise or with the current byte
// and C (1) shifted to the appropriate position within
// the byte, and then assigning the byte with the result
result[result_index] |= C << shift;
break;
case 'G':
result[result_index] |= G << shift;
break;
case 'T':
result[result_index] |= T << shift;
break;
default:
fputs("error: encode_dna: invalid base pair", stderr);
exit(EXIT_FAILURE);
}
// increase shift amount by 2 to the next 2-bit slot in the byte
shift += 2;
// on every 4th iteration, reset our shift to zero since the byte is now full
// and move to the next byte in our result buffer
if (shift == 8) {
shift = 0;
result_index++;
}
// advance sequence to next nucleotide character
sequence++;
}
}
And here's a test
int main(int argc, char **argv) {
// allocate some storage for encoded DNA
unsigned char encoded_dna[32];
const unsigned long sample_length = 15;
// encode the given sample sequence
encode_dna(encoded_dna, "ACGTAGTCGTCATAG", sample_length);
// hh here means half of half word, which is a byte
// capital X for capitalized hex output
// here we print some bytes
printf("0x%hhX\n", encoded_dna[0]); // should output 0xE4
printf("0x%hhX\n", encoded_dna[1]); // should be 0x78
printf("0x%hhX\n", encoded_dna[2]); // should be 0x1E
printf("0x%hhX\n", encoded_dna[3]); // should be 0x23
DNA test_dna; // allocate a sample DNA structure
test_dna.bits = encoded_dna;
test_dna.length = sample_length; // length of the sample sequence above
// test some indices and see if the results are correct
printf("test_dna index 4: %c\n", base_at_index(&test_dna, 4));
printf("test_dna index 7: %c\n", base_at_index(&test_dna, 7));
printf("test_dna index 12: %c\n", base_at_index(&test_dna, 12));
return 0;
}
Output:
0xE4
0x78
0x1E
0x23
test_dna index 4: A
test_dna index 7: C
test_dna index 12: T
Assuming you really do want to encode your dna string into a hex string and that you want to read the input string from left to right, but output hex chars right to left, here's a simple, but slightly slow implementation.
First, your DNAString needs to keep track whether there are really an even or odd number of acid sequences in the list. This will make additional appendages easier.
struct DNAString
{
char* seq;
bool odd; // if odd bit is set, then the front char is already allocated and holds one acid
};
And now let's introduce a little helper function to convert ACGT into 0,1,2,3.
char acid_to_value(char c)
{
switch (c)
{
case 'A': return 0;
case 'C': return 1;
case 'G': return 2;
case 'T': return 3;
}
// ASSERT(false)
return 0;
}
Then the core implementation is to keep "prepending" new hexchars onto the string your are building. If the string is already of an odd length, the code will just "fixup" the front character by converting it from hex to integer, then shifting the new acid value into it, then converting it back to a hex char
extern char fixup(char previous, char acid);
{
char hexchars[16] = { '0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F' };
char tmp[2] = { previous, '\0' };
unsigned long asnumber = strtol(tmp, nullptr, 16);
asnumber = asnumber & 0x3; // last two bits
asnumber = asnumber | (acid_to_value(acid) << 2);
return hexchars[asnumber];
}
void prepend_nucleic_acid_to_hexstring(struct DNAString* dnastring, char acid)
{
if (dnastring->odd)
{
// find the first char in the string and fix it up hexwise
dnastring->seq[0] = fixup(dnastring->seq[0], acid);
dnastring->odd = false;
}
else
{
size_t currentlength = dnastring->seq ? strlen(dnastring->seq) : 0;
const char* currentstring = dnastring->seq ? dnastring->seq : "";
char* newseq = (char*)calloc(currentlength + 2, sizeof(char)); // +1 for new char and +1 for null char
newseq[0] = acid_to_value(acid) + '0'; // prepend the next hex char
strcpy(newseq + 1, currentstring); // copy the old string into the new string space
free(dnastring->seq);
dnastring->seq = newseq;
dnastring->odd = true;
}
}
Then your DNACreate function is real simple:
struct DNAString DSCreate(const char* mseq)
{
DNAString dnastring = { 0 };
while (*mseq)
{
prepend_nucleic_acid_to_hexstring(&dnastring, *mseq);
mseq++;
}
return dnastring;
}
I don't claim this approach to be efficient since he literally keeps reallocating memory for each char. But it does enable you to have flexability to invoke the prepend function later for additional sequencing.
And then to test it:
int main()
{
struct DNAString dnastring = DSCreate("ACGT");
printf("0x%s\n", dnastring.seq);
return 0;
}
In many board games (like checkers, go and othello/reversi) each square can be represented by three states: white, black or empty.
8x8 boards in such game engines are usually represented as two bitboards: one 64-bit integer for location of white pieces and another 64-bit integer – for black.
However, when storing local game patterns, such binary representation can require a lot of space. For example, creating a lookup table for all possible values of an 8-square row would require an array with 256*256 = 4^8 = 65536 values, compared to only 3^8 = 6561 possible positions (since a square can never be occupied by both black and white pieces).
An alternative way is to store the boards as ternary numbers – so called titboards. But I didn't find anywhere a fast algorithm to convert between two binary integers representation and ternary integer representation.
Therefore, my question is
Is there an efficient way to convert (encode) two mutually exclusive binary numbers (w & b == 0) in ternary numbers, so that each unique pair of such integers would be mapped to a resulting unique integer? (Preferably in C/C++.)
Python example
Here is my Python solution to do this:
white_black_empty = lambda w, b: int(format(w, 'b'), base=3) + \
int(format(b, 'b'), base=3)*2
Example values:
w = 10100012 = 81
b = 01001002 = 36
result = 10100013 + 01001003*2 = 10100013 + 02002003 = 12102013 = 1315
So white_black_empty(81, 36) == 1315.
But, converting integer into string representation of a binary format(x, 'b') and then from string back to integer using base 3 int(x, base=3) looks rather inefficient.
If your hardware has a fast popcount operation, then you can represent a board of n spaces as 2 n-bit values ⟨mask, colour⟩, where the second value is guaranteed to be in the range [0, 2popcount(mask)] The first value is 1 in the bit position corresponding to a square if the square is occupied; the second value is 1 in the bit position corresponding to j if the jth occupied square has a white piece. In order to access bits in colour, it's useful to have this function which returns a bit position in colour given mask and a bit position in the mask which corresponds to a 1-bit in the mask (i.e. a bit corresponding to an occupied square):
static inline int colourBitPos(unsigned mask, unsigned pos) {
return popcount(mask & ((1U << pos) - 1));
}
(In other words, it counts the number of one bits in mask following the specified position.)
You can then easily turn the ⟨mask, colour⟩ pair into a single number in the range [0, 3n-1] by way of a precomputed lookup table holding base indices. When I was originally thinking of this system, I thought in terms of n+1 concatenated tables, each corresponding to a single popcount. That's conceptually nice, since the number of possible colourings of a code with popcount i is obviously 2i while the number of masks with popcount i is C(n, i) (using C() as the binomial coefficient function since there is no MathJax here). The lovely identity:
is probably less well-known than other binomial identities.
While it is possible to take advantage of that arrangement to rather laboriously compute the index in O(n) time (bit by bit in the mask field), the easiest and fastest solution is to use a 2n-element fixed lookup table base, whose size is much less than the 3n data tables. A base value is computed for each value of mask by simply accumulating the appropriate power of two for each value:
int base[1U<<N];
for (unsigned i = 0, offset = 0; i < 1U<<N; ++i) {
base[i] = offset;
offset += 1U<<popcount(i);
}
Then the index of any pair can be computed as:
index = base[mask] + colour;
[See example below]
The two-component representation is not particularly hard to work with, although it is obviously not as easy as a two-bit three-way choice. For example, to find what's in square i:
(mask & (1U << i))
? (colour & ((1U << colouredBitPos(mask, i) - 1) ? WHITE : BLACK
: EMPTY
For another example, in order to add a piece coloured col (WHITE = 1, BLACK = 0) to currently unoccupied square i, you would do:
unsigned pos = colouredBitPos(mask, i);
colour += (colour & ~((1U << pos) - 1)) + (col << pos);
mask |= 1U << i;
effectively shifting the first part of colour left one bit in order to insert the new bit. If the square were already occupied, you would avoid the shift:
unsigned pos = colouredBitPos(mask, i);
colour &= ~(1U << pos); // Make it black
colour |= col << pos; // And give it the right colour
Other operations are similarly straight-forward.
Whether that work is justified in your case will depend on so many factors that I can't possibly render a judgement. But the space overhead is close to optimal. The only overhead aside from increased code size is a single 2n-element lookup table which can be used with all of the data tables, and which is in any case tiny relative to the size of any data table (eg., for n=8, the data tables have 6561 elements so a 256-element lookup table would add 4% overhead of a single data table whose data elements are also shorts. And there is no need to persist the lookup table if you're persisting the data tables, since it can easily be regenerated.)
Index encoding example:
Using n=4 for simplicity, the base lookup table is:
mask base mask base mask base mask base
0000 0 0100 9 1000 27 1100 45
0001 1 0101 11 1001 29 1101 49
0010 3 0110 15 1010 33 1110 57
0011 5 0111 19 1011 37 1111 65
Using U=Unoccupied, B=Black, W=White (and assuming, as above, that White is 1), some example encodings and indexes:
board mask colour compute index decimal
UUBW 0011 01 base[0011]+ 01 = 6
UUWB 0011 10 base[0010]+ 10 = 7
WUBW 1011 101 base[1011]+101 = 42
How about storing what you're trying to convert? With the scheme below, each additional 8 bits of a row, would cost 512 numbers in an array (or hash table). The tradeoff would be more additions and bit-extraction to cut storage - for example, to store 8 bits, rather than the full 8, which result in 255 numbers, we could store 2^4 and 2^4 (for the second set of 4 bits), resulting in 32 (plus 32 for the blacks) numbers stored, but necessitating extracting each set of 4 bits and another addition during the conversion.
const ones = new Array(256);
const twos = new Array(256);
for (let i=0; i<256; i++){
let one = 0;
let two = 0;
for (let j=0; j<8; j++){
if ((1 << j) & i){
one += Math.pow(3, j);
two += 2*Math.pow(3, j);
}
ones[i] = one;
twos[i] = two;
}
}
function convert(w, b){
return ones[w] + twos[b];
}
console.log(convert(81, 36));
Converting from string to integer and back will indeed be inefficient.
If you just need to encode the values, thinking of them in terms of the actual numbers they represent will be useful. For example, in considering eight rows on a board, the first position's state is effectively boardState % 3; we can use the convention that a black piece is there on a 1, a white piece on a 2, and an empty value on a 0. For the second, it becomes (boardState % 9)/3, the third (boardState % 27) / 3 and so on.
So, for encoding, we can extend this thinking: we take either a 0, 1, or 2, multiply it by 3 to the power of (whichever board position we're considering), and add it to some "result" number. Some (VERY untested) example code is below:
#include <inttypes.h>
#include <math.h>
uint64_t tritboard(uint64_t white, uint64_t black){
uint64_t onemask = 0x0000000000000001;//you could also just say "= 1"
uint64_t retval = 0;
uint64_t thisPos;
for(char i = 0; i < 8; i++){
thisPos = 0;
if(white & (oneMask << i)) thisPos += 2;
if(black & (oneMask << i)) thisPos += 1;
retval += thisPos * ( (uint64_t) pow(3, i));
}//for
return retval;
}//tritboard
Unfortunately, with computers being partial to binary, you're only going to be able to get but so clever about bitshifts. Thus, the for loop in this code(which is slightly less gross in C as it is in python, in terms of performance).
Note that you are limited in scope for this approach; as you can appreciate, you can't represent the entire board with this approach (as there aren't 3^64 possible values for a 64-bit integer).
Hopefully, that is more amenable to you than the string approach!
In practice, you'll want to store the board state in base-4 packed in unsigned longs, with each board row padded to an integral number of unsigned longs. This will give you the best memory locality, very fast access to board cells, but uses 26.2% more RAM than ternary packing.
To store the board state in a binary file, you can pack 5 ternary digits (five board cell states) into each 8-bit byte. This uses only 5.1% more memory than ternary packing, and is simple and robust to implement. In particular, this way you do not need to worry about byte order (endianness).
The problem with pure ternary packing is that each base-3 digit affects most of the binary digits representing the same numerical value. For example, 38 = 300000003 = 6561 = 11001101000012. This means that the only practical way to extract base-3 digits is via repeated division and modulus (by 3).
To describe a board of size N×M, the ternary packing and unpacking function will be essentially O(N2M2), and therefore slower and slower as the board size increases. You'll likely get better savings by using a compression library (say, liblzma) using less CPU time. For many board configurations, run-length encoding might also work well.
Here is an example implementation for boards of up to 16777215×16777215 cells (tested only up to 32768×32768 cells):
#include <stdlib.h>
#include <inttypes.h>
#include <limits.h>
#include <stdio.h>
#include <time.h>
#define ULONG_BITS (CHAR_BIT * sizeof (unsigned long))
#define ULONG_CELLS (CHAR_BIT * sizeof (unsigned long) / 2)
struct board {
int rows;
int cols;
size_t stride;
unsigned long *data;
};
enum {
EMPTY = 0, /* calloc() clears the data to zeroes */
WHITE = 1,
BLACK = 2,
ERROR = 3
};
int board_init(struct board *const b, const int rows, const int cols)
{
const size_t stride = (cols + ULONG_CELLS - 1) / ULONG_CELLS;
const size_t ulongs = stride * (size_t)rows;
if (b) {
b->rows = 0;
b->cols = 0;
b->stride = 0;
b->data = NULL;
}
if (!b || rows < 1 || cols < 1)
return -1;
if ((size_t)(ulongs / stride) != (size_t)rows)
return -1;
b->data = calloc(ulongs, sizeof b->data[0]);
if (!b->data)
return -1;
b->rows = rows;
b->cols = cols;
b->stride = stride;
return 0;
}
static inline int get_cell(const struct board *const b, const int row, const int col)
{
if (!b || row < 0 || col < 0 || row >= b->rows || col >= b->cols)
return EMPTY;
else {
const size_t i = (size_t)col / ULONG_CELLS;
const size_t c = ((size_t)col % ULONG_CELLS) * 2;
const unsigned long w = b->data[b->stride * row + i];
return (w >> c) & 3;
}
}
static inline int set_cell(struct board *const b, const int row, const int col, const int value)
{
if (!b || row < 0 || col < 0 || row >= b->rows || col >= b->cols)
return EMPTY;
else {
const size_t i = (size_t)col / ULONG_CELLS;
const size_t c = ((size_t)col % ULONG_CELLS) * 2;
unsigned long *w = b->data + b->stride * row + i;
*w = ((*w) & (3uL << c)) | ((unsigned long)(value & 3) << c);
return value & 3;
}
}
static inline int write_u24(FILE *const out, const int value)
{
unsigned int u = value;
if (!out || value < 0 || value > 16777215 || ferror(out))
return -1;
if (fputc(u & 255, out) == EOF)
return -1;
else
u >>= 8;
if (fputc(u & 255, out) == EOF)
return -1;
else
u >>= 8;
if (fputc(u & 255, out) == EOF)
return -1;
else
return 0;
}
static inline int read_u24(FILE *const in, unsigned int *const to)
{
unsigned int result;
int c;
if (!in || ferror(in))
return -1;
c = fgetc(in);
if (c == EOF)
return -1;
else
result = c & 255;
c = fgetc(in);
if (c == EOF)
return -1;
else
result |= (c & 255) << 8;
c = fgetc(in);
if (c == EOF)
return -1;
else
result |= (c & 255) << 16;
if (to)
*to = result;
return 0;
}
int board_save(const struct board *const b, FILE *const out)
{
int row, col, cache, coeff;
if (!b || !out || ferror(out) || !b->stride ||
b->rows < 1 || b->rows > 16777215 ||
b->cols < 1 || b->cols > 16777215)
return -1;
if (write_u24(out, b->rows))
return -1;
if (write_u24(out, b->cols))
return -1;
/* Clear byte cache. */
cache = 0;
coeff = 1;
for (row = 0; row < b->rows; row++) {
for (col = 0; col < b->cols; col++) {
switch (get_cell(b, row, col)) {
case EMPTY: /* Saved as 0 */
break;
case WHITE: /* Saved as 1 */
cache += coeff;
break;
case BLACK: /* Saved as 2 */
cache += coeff + coeff;
break;
default: /* Invalid cell state. */
return -1;
}
if (coeff >= 81) {
if (fputc(cache, out) == EOF)
return -1;
cache = 0;
coeff = 1;
} else
coeff *= 3;
}
}
if (coeff > 1)
if (fputc(cache, out) == EOF)
return -1;
if (fflush(out))
return -1;
return 0;
}
int board_load(struct board *const b, FILE *in)
{
unsigned int rows, cols, row, col, cache, count;
int c;
if (b) {
b->rows = 0;
b->cols = 0;
b->stride = 0;
b->data = NULL;
}
if (!b || !in || ferror(in))
return -1;
if (read_u24(in, &rows) || rows < 1 || rows > 16777215)
return -1;
if (read_u24(in, &cols) || cols < 1 || cols > 16777215)
return -1;
if (board_init(b, rows, cols))
return -1;
/* Nothing cached at this point. */
cache = 0;
count = 0;
for (row = 0; row < rows; row++) {
for (col = 0; col < cols; col++) {
if (count < 1) {
c = fgetc(in);
if (c == EOF || c < 0 || c >= 243)
return -1;
cache = c;
count = 5;
}
switch (cache % 3) {
case 0: /* Leave as background. */
break;
case 1: /* White */
if (set_cell(b, row, col, WHITE) != WHITE)
return -1;
break;
case 2: /* Black */
if (set_cell(b, row, col, BLACK) != BLACK)
return -1;
break;
}
cache /= 3;
count--;
}
}
/* No errors. */
return 0;
}
/* Xorshift 64* pseudo-random number generator. */
static uint64_t prng_state = 1;
static inline uint64_t prng_randomize(void)
{
int rounds = 1024;
uint64_t state;
state = (uint64_t)time(NULL);
while (rounds-->0) {
state ^= state >> 12;
state ^= state << 25;
state ^= state >> 27;
}
if (!state)
state = 1;
prng_state = state;
return state;
}
static inline uint64_t prng_u64(void)
{
uint64_t state = prng_state;
state ^= state >> 12;
state ^= state << 25;
state ^= state >> 27;
prng_state = state;
return state * UINT64_C(2685821657736338717);
}
/* Uniform random ternary generator. */
static uint64_t ternary_cache = 0;
static int ternary_bits = 0;
static inline int prng_ternary(void)
{
int retval;
do {
if (ternary_bits < 2) {
ternary_cache = prng_u64();
ternary_bits = 64;
}
retval = ternary_cache & 3;
ternary_cache >>= 1;
ternary_bits -= 2;
} while (retval > 2);
return retval;
}
int main(int argc, char *argv[])
{
struct board original, reloaded;
uint64_t correct, incorrect, count[3];
double percent;
FILE *file;
int rows, cols, row, col;
char dummy;
if (argc != 4) {
fprintf(stderr, "\n");
fprintf(stderr, "Usage: %s [ -h | --help ]\n", argv[0]);
fprintf(stderr, " %s FILENAME ROWS COLUMNS\n", argv[0]);
fprintf(stderr, "\n");
fprintf(stderr, "This program generates a random ternary board,\n");
fprintf(stderr, "saves it to file FILENAME, reads it back, and\n");
fprintf(stderr, "verifies that the board state is intact.\n");
fprintf(stderr, "\n");
return EXIT_SUCCESS;
}
if (!argv[1][0]) {
fprintf(stderr, "No filename specified.\n");
return EXIT_FAILURE;
}
if (sscanf(argv[2], "%d %c", &rows, &dummy) != 1 || rows < 1 || rows > 16777215) {
fprintf(stderr, "%s: Invalid number of rows.\n", argv[2]);
return EXIT_FAILURE;
}
if (sscanf(argv[3], "%d %c", &cols, &dummy) != 1 || cols < 1 || cols > 16777215) {
fprintf(stderr, "%s: Invalid number of columns.\n", argv[2]);
return EXIT_FAILURE;
}
if (board_init(&original, rows, cols)) {
fprintf(stderr, "Cannot create a board with %d rows and %d columns.\n", rows, cols);
return EXIT_FAILURE;
}
fprintf(stderr, "Filling board with a random state; random seed is %" PRIu64 ".\n", prng_randomize());
percent = 100.0 / (double)rows / (double)cols;
count[0] = count[1] = count[2] = 0;
for (row = 0; row < rows; row++)
for (col = 0; col < cols; col++) {
int t = prng_ternary();
if (t < 0 || t > 3) {
fprintf(stderr, "prng_ternary() returned %d!\n", t);
return EXIT_FAILURE;
}
count[t]++;
set_cell(&original, row, col, t);
}
fprintf(stderr, " Empty: %" PRIu64 " cells, %.3f%%.\n", count[EMPTY], (double)count[EMPTY] * percent);
fprintf(stderr, " White: %" PRIu64 " cells, %.3f%%.\n", count[WHITE], (double)count[WHITE] * percent);
fprintf(stderr, " Black: %" PRIu64 " cells, %.3f%%.\n", count[BLACK], (double)count[BLACK] * percent);
file = fopen(argv[1], "wb");
if (!file) {
fprintf(stderr, "%s: Cannot open file for writing.\n", argv[1]);
return EXIT_FAILURE;
}
fprintf(stderr, "Saving to %s.\n", argv[1]);
if (board_save(&original, file)) {
fclose(file);
fprintf(stderr, "Write error.\n");
return EXIT_FAILURE;
}
if (fclose(file)) {
fprintf(stderr, "Write error.\n");
return EXIT_FAILURE;
}
fprintf(stderr, "Reloading game board.\n");
file = fopen(argv[1], "rb");
if (!file) {
fprintf(stderr, "%s: Cannot open file for reading.\n", argv[1]);
return EXIT_FAILURE;
}
if (board_load(&reloaded, file)) {
fclose(file);
fprintf(stderr, "Read error.\n");
return EXIT_FAILURE;
}
if (fclose(file)) {
fprintf(stderr, "Read error.\n");
return EXIT_FAILURE;
}
if (original.rows != reloaded.rows) {
fprintf(stderr, "Row count mismatches.\n");
return EXIT_FAILURE;
} else
if (original.cols != reloaded.cols) {
fprintf(stderr, "Column count mismatches.\n");
return EXIT_FAILURE;
}
fprintf(stderr, "Comparing board states.\n");
correct = 0;
incorrect = 0;
for (row = 0; row < rows; row++)
for (col = 0; col < cols; col++)
if (get_cell(&original, row, col) == get_cell(&reloaded, row, col))
correct++;
else
incorrect++;
if (incorrect) {
fprintf(stderr, "Found %" PRIu64 " mismatching cells (%.3f%%).\n", incorrect, (double)incorrect * percent);
return EXIT_FAILURE;
}
if (correct != (uint64_t)((uint64_t)rows * (uint64_t)cols)) {
fprintf(stderr, "Internal bug in the board comparison double loop.\n");
return EXIT_FAILURE;
}
fprintf(stderr, "Verification successful; functions work as expected for a board with %d rows and %d columns.\n", rows, cols);
return EXIT_SUCCESS;
}
The board_init() function initializes a board, board_save() saves a board state to a stream, including the board size, in portable binary format (each file will generate the same board on both big-endian and little-endian architectures), and board_load() will load a previously saved board from a stream. They all return 0 if success, nonzero if error.
The get_cell() and set_cell() functions are static inline accessor functions to examine and set the state of individual cells in a board.
As I initially suggested, this one uses 2 bits per cell in RAM (4 cells per byte), and 5 cells per byte when stored to a file.
The example program takes three command-line parameters: a file name, the number of rows, and the number of columns. It will generate a random state of that size, save it to the named file, read it back from the named file into a separate board, and finally compare the board states, to verify if the implemented functions seem to work correctly.
Firstly I want to say I tried many times to find the answer by using google search, and I found many results but I did not understand, because I don't know the idea of reading a binary file, and convert the value that Obtained to readable value.
What I tried doing it.
unsigned char fbuff[16];
FILE *file;
file = fopen("C:\\loser.jpg", "rb");
if(file != NULL){
fseek(file, 0, SEEK_SET);
fread(fbuff, 1, 16, file);
printf("%d\n", fbuff[1]);
fclose(file);
}else{
printf("File does not exists.");
}
I want a simple explanation with example shows, how to get width/height of jpeg file from its header, and then convert that value to readable value.
Unfortunately, it doesn't seem to be simple for JPEG. You should look at the source to the jhead command line tool. It provides this information. When going through the source, you will see the function ReadJpegSections. This function scans through all the segments contained within the JPEG file to extract the desired information. The image width and height is obtained when processing the frames that have an SOFn marker.
I see the source is in the public domain, so I'll show the snippet that gets the image info:
static int Get16m(const void * Short)
{
return (((uchar *)Short)[0] << 8) | ((uchar *)Short)[1];
}
static void process_SOFn (const uchar * Data, int marker)
{
int data_precision, num_components;
data_precision = Data[2];
ImageInfo.Height = Get16m(Data+3);
ImageInfo.Width = Get16m(Data+5);
From the source code, it is clear to me there is no single "header" with this information. You have to scan through the JPEG file, parsing each segment, until you find the segment with the information in it that you want. This is described in the wikipedia article:
A JPEG image consists of a sequence of segments, each beginning with a marker, each of which begins with a 0xFF byte followed by a byte indicating what kind of marker it is. Some markers consist of just those two bytes; others are followed by two bytes indicating the length of marker-specific payload data that follows.
A JPEG file consists of a sequence of segments:
SEGMENT_0
SEGMENT_1
SEGMENT_2
...
Each segment begins with a 2-byte marker. The first byte is 0xFF, the second byte determines the type of the segment. This is followed by an encoding of the length of the segment. Within the segment is data specific to that segment type.
The image width and height is found in a segment of type SOFn, or "Start of frame [n]", where "n" is some number that means something special to a JPEG decoder. It should be good enough to look only for a SOF0, and its byte designation is 0xC0. Once you find this frame, you can decode it to find the image height and width.
So the structure of a program to do what you want would look like:
file_data = the data in the file
data = &file_data[0]
while (data not at end of file_data)
segment_type = decoded JPEG segment type at data
if (type != SOF0)
data += byte length for segment_type
continue
else
get image height and width from segment
return
This is essentially the structure found in Michael Petrov's get_jpeg_size() implementation.
then you have to find hight and width marker of jpeg that is [ffc0].
after finding ffc0 in binary formate, the the four,five bytes are hight and six and seven bytes are width.
eg: [ff c0] d8 c3 c2 [ff da] [00 ff]
| |
| |
->height ->width
int position;
unsigned char len_con[2];
/*Extract start of frame marker(FFC0) of width and hight and get the position*/
for(i=0;i<FILE_SIZE;i++)
{
if((image_buffer[i]==FF) && (image_buffer[i+1]==c0) )
{
position=i;
}
}
/*Moving to the particular byte position and assign byte value to pointer variable*/
position=position+5;
*height=buffer_src[position]<<8|buffer_src[position+1];
*width=buffer_src[position+2]<<8|buffer_src[position+3];
printf("height %d",*height);
printf("width %d",*width);
the question is old and the other answers are correct but their format is not the easiest one. I just use getc to quickly get the dimensions, while skipping irrelevant markers (it also supports Progressive JPEGs):
int height, width;
// start of image (SOI)
getc(f); // oxff
getc(f); // oxd8
// Scan miscellaneous markers until we reach SOF0 marker (0xC0)
for(;;) {
// next marker
int marker;
while((marker = getc(f)) != 0xFF);
while((marker = getc(f)) == 0xFF);
// SOF
if (marker == 0xC0 || marker == 0xC2) {
getc(f); // length (2 bytes)
getc(f); // #
getc(f); // bpp, usually 8
height = (getc(f) << 8) + getc(f); // height
width = (getc(f) << 8) + getc(f); // width
break;
}
}
Image dimensions in JPEG files can be found as follows:
1) Look for FF C0
2) At offsets +4 and +6 after this location are height and width (words), resp/ly.
In most cases, the absolute offsets of height and width are A3 and A5, resp/ly.
Here's some simple code I wrote which seems to work reliably.
#define MOTOSHORT(p) ((*(p))<<8) + *(p+1)
unsigned char cBuf[32];
int iBytes, i, j, iMarker, iFilesize;
unsigned char ucSubSample;
int iBpp, iHeight, iWidth;
Seek(iHandle, 0, 0); // read the first 32 bytes
iBytes = Read(iHandle, cBuf, 32);
i = j = 2; /* Start at offset of first marker */
iMarker = 0; /* Search for SOF (start of frame) marker */
while (i < 32 && iMarker != 0xffc0 && j < iFileSize)
{
iMarker = MOTOSHORT(&cBuf[i]) & 0xfffc;
if (iMarker < 0xff00) // invalid marker, could be generated by "Arles Image Web Page Creator" or Accusoft
{
i += 2;
continue; // skip 2 bytes and try to resync
}
if (iMarker == 0xffc0) // the one we're looking for
break;
j += 2 + MOTOSHORT(&cBuf[i+2]); /* Skip to next marker */
if (j < iFileSize) // need to read more
{
Seek(iHandle, j, 0); // read some more
iBytes = Read(iHandle, cBuf, 32);
i = 0;
}
else // error, abort
break;
} // while
if (iMarker != 0xffc0)
goto process_exit; // error - invalid file?
else
{
iBpp = cBuf[i+4]; // bits per sample
iHeight = MOTOSHORT(&cBuf[i+5]);
iWidth = MOTOSHORT(&cBuf[i+7]);
iBpp = iBpp * cBuf[i+9]; /* Bpp = number of components * bits per sample */
ucSubSample = cBuf[i+11];
}
int GetJpegDimensions(
char *pImage,
size_t nSize,
unsigned32 *u32Width,
unsigned32 *u32Height,
char *szErrMsg)
{
int nIndex;
int nStartOfFrame;
int nError = NO_ERROR;
bool markerFound = false;
unsigned char ucWord0;
unsigned char ucWord1;
// verify START OF IMAGE marker = FF D8
nIndex = 0;
ucWord0 = pImage[nIndex];
ucWord1 = pImage[nIndex+1];
// marker FF D8 starts a valid JPEG
if ((ucWord0 == 0xFF) && (ucWord1 == 0xD8))
{
// search for START OF FRAME 0 marker FF C0
for (nIndex = 2;
(nIndex < nSize-2) && (markerFound == false);
nIndex += 2)
{
ucWord0 = pImage[nIndex];
ucWord1 = pImage[nIndex+1];
if (ucWord0 == 0xFF)
{
if (ucWord1 == 0xC0)
{
markerFound = true;
nStartOfFrame = nIndex;
}
}
if (ucWord1 == 0xFF)
{
ucWord0 = pImage[nIndex+2];
if (ucWord0 == 0xC0)
{
markerFound = true;
nStartOfFrame = nIndex+1;
}
}
} // while
if (markerFound)
{
nError = NO_ERROR;
ucWord0 = pImage[nStartOfFrame+5];
ucWord1 = pImage[nStartOfFrame+6];
*u32Height = ucWord1 + (ucWord0 << 8);
ucWord0 = pImage[nStartOfFrame+7];
ucWord1 = pImage[nStartOfFrame+8];
*u32Width = ucWord1 + (ucWord0 << 8);
}
else
{
// start of frame 0 not found
nError = -2;
sprintf(szErrMsg,
"Not a valid JPEG image. START OF FRAME 0 marker FFC0 not found");
}
}
else // START OF IMAGE marker not found
{
nError = -1;
sprintf(szErrMsg,
"Not a valid JPEG image. START OF IMAGE marker FFD8 not found");
}
return nError;
}
Here's a code i wrote in Java. Works fine for jpegs taken from a camera. It scans all the code to find the biggest image size. I could not improve it to skip on the lengths of each block because it doesn't work. If anyone can improve the code to do that it would be great.
int getShort(byte[] p, int i)
{
int p0 = p[i] & 0xFF;
int p1 = p[i+1] & 0xFF;
return p1 | (p0 << 8);
}
int[] GetJpegDimensions(byte[] b)
{
int nIndex;
int height=0, width=0, size=0;
int nSize = b.length;
// marker FF D8 starts a valid JPEG
if (getShort(b,0) == 0xFFD8)
for (nIndex = 2; nIndex < nSize-1; nIndex += 4)
if (b[nIndex] == -1/*FF*/ && b[nIndex+1] == -64/*C0*/)
{
int w = getShort(b,nIndex+7);
int h = getShort(b,nIndex+5);
if (w*h > size)
{
size = w*h;
width = w;
height = h;
}
}
return new int[]{width,height};
}