I was looking around in the zephyr implementations and found this method for computing a crc16 checksum:
u16_t crc16(const u8_t *src, size_t len, u16_t polynomial,
u16_t initial_value, bool pad)
{
u16_t crc = initial_value;
size_t padding = pad ? sizeof(crc) : 0;
size_t i, b;
/* src length + padding (if required) */
for (i = 0; i < len + padding; i++) {
for (b = 0; b < 8; b++) {
u16_t divide = crc & 0x8000UL;
crc = (crc << 1U);
/* choose input bytes or implicit trailing zeros */
if (i < len) {
crc |= !!(src[i] & (0x80U >> b));
}
if (divide != 0U) {
crc = crc ^ polynomial;
}
}
}
return crc;
}
And I tripped over this line here:
crc |= !!(src[i] & (0x80U >> b));
I do not understand why they are using a boolean operator (!!) in this line. From my understanding this is what it does:
It basically does an implicit "casting" where it considers its operand on the right to be a boolean and negates it twice, which does not do anything besides making the output a 0 or a 1 depending on if the expression (src[i] & (0x80U >> b)) was bigger then 0 to start with.
Is this correct? Why are they using the operator in this way?
It is inserting bit 7-b from src[i] into the low bit of crc. If that bit is a 1, which will be somewhere in the result of the &, the !! turns it into a 1 in the low bit, which is then or'ed into crc.
This is truly painful to look at. A better and cleaner way to do it is crc |= (src[i] >> b) & 1;, where b counts down instead of up. E.g. int b = 8; do { b--; ... } while (b);. Better still would be to just exclusive-or the byte after the loop, which does the same thing:
/* src length + padding (if required) */
for (i = 0; i < len + padding; i++) {
for (b = 0; b < 8; b++)
crc = crc & 0x8000 ? (crc << 1) ^ polynomial : crc << 1;
if (i < len)
crc ^= src[i];
}
An optimizing compiler will unroll the b loop.
I have the following CRC function:
#define CRC8INIT 0x00
#define CRC8POLY 0x18 //0X18 = X^8+X^5+X^4+X^0
// ----------------------------------------------------------------------------
uint8_t crc8 (uint8_t *data, uint16_t number_of_bytes_in_data)
{
uint8_t crc;
uint16_t loop_count;
uint8_t bit_counter;
uint8_t b;
uint8_t feedback_bit;
crc = CRC8INIT;
for (loop_count = 0; loop_count != number_of_bytes_in_data; loop_count++) {
b = data[loop_count];
bit_counter = 8;
do {
feedback_bit = (crc ^ b) & 0x01;
if (feedback_bit == 0x01) {
crc = crc ^ CRC8POLY;
}
crc = (crc >> 1) & 0x7F;
if (feedback_bit == 0x01) {
crc = crc | 0x80;
}
b = b >> 1;
bit_counter--;
} while (bit_counter > 0);
}
return crc;
}
How does 0x18 relate to the polynomial X^8+X^5+X^4+X^0?
X^8+X^5+X^4+X^0 = 100110001
0x18 = 00011000
What if I define CRC8POLY as 0xEA instead (I have seen this), what polynomial would that represent?
How does 0x18 relate to the polynomial X^8+X^5+X^4+X^0?
Since the code is a right shifting CRC, the "most significant bit" of each byte is bit 0 instead of bit 7. The poly needs to be reversed from 100110001 to 100011001, which is 0x119, after the right shift, bit 0 of 0x119 is shifted off, so 0x118 can be used instead. The code uses a second if statement to or in (0x100) >> 1 == 0x80 if the feedback bit is 1. As an alternative, since feedback_bit is 0 or 1, then (0-feeback_bit) can be used as a mask (assuming two's complement math) for the poly instead of using an if statement.
typedef unsigned char uint8_t;
typedef unsigned short uint16_t;
#define CRC8INIT 0x00
#define CRC8POLY 0x8c // 0x119 >> 1
uint8_t crc8 (uint8_t *data, uint16_t number_of_bytes_in_data)
{
uint8_t crc;
uint16_t loop_count;
uint8_t bit_counter;
uint8_t b;
uint8_t feedback_bit;
crc = CRC8INIT;
for (loop_count = 0; loop_count != number_of_bytes_in_data; loop_count++) {
b = data[loop_count];
bit_counter = 8;
do {
feedback_bit = (crc ^ b) & 0x01;
crc = (crc >> 1) ^ ((0-feedback_bit) & CRC8POLY);
b = b >> 1;
bit_counter--;
} while (bit_counter > 0);
}
return crc;
}
That CRC code is written rather oddly, where the polynomial is applied in two different places, breaking up the bits. That is, at crc = crc ^ CRCPOLY and conditionally crc = crc | 0x80. It would normally be written more like this:
unsigned crc8(unsigned char const *dat, size_t len) {
unsigned crc = 0;
for (size_t i = 0; i < len; i++) {
crc ^= dat[i];
for (int k = 0; k < 8; k++)
crc = crc & 1 ? (crc >> 1) ^ 0x8c : crc >> 1;
}
return crc;
}
where 0x8c is that polynomial reversed, without the x8 term.
Referring to Wiki article on CRC , it's crc8 realization as used in Dallas 1-wire chips. Note, that polinomial can be represented in Normal, Reversed, Reciprocal and Reversed reciprocal representations (see also Polynomial representations). Seems, that it is Reversed reciprocal representation with highest bit omitted.
I need to calculate CRC of the message and put it at the beginning of this message, so that the final CRC of the message with 'prepended' patch bytes equals 0. I was able to do this very easily with the help of few articles, but not for my specific parameters. The thing is that I have to use a given CRC32 algorithm which calculates the CRC of the memory block, but I don't have that 'reverse' algorithm that calculates those 4 patch bytes/'kind of CRC'. Parameters of the given CRC32 algorithm are:
Polynomial: 0x04C11DB7
Endianess: big-endian
Initial value: 0xFFFFFFFF
Reflected: false
XOR out with: 0L
Test stream: 0x0123, 0x4567, 0x89AB, 0xCDEF results in CRC = 0x612793C3
The code to calculate the CRC (half-byte, table-driven, I hope data type definitions are self-explanatory):
uint32 crc32tab(uint16* data, uint32 len, uint32 crc)
{
uint8 nibble;
int i;
while(len--)
{
for(i = 3; i >= 0; i--)
{
nibble = (*data >> i*4) & 0x0F;
crc = ((crc << 4) | nibble) ^ tab[crc >> 28];
}
data++;
}
return crc;
}
The table needed is (I thougth the short [16] table should contain every 16th element from the large [256] table, but this table contains actually first 16 elements, but that's how it was provided to me):
static const uint32 tab[16]=
{
0x00000000, 0x04C11DB7, 0x09823B6E, 0x0D4326D9,
0x130476DC, 0x17C56B6B, 0x1A864DB2, 0x1E475005,
0x2608EDB8, 0x22C9F00F, 0x2F8AD6D6, 0x2B4BCB61,
0x350C9B64, 0x31CD86D3, 0x3C8EA00A, 0x384FBDBD
};
I modified the code so it's not so long, but the functionality stays the same. The problem is that this forward CRC calculation looks more like backward/reverse CRC calc.
I've spent almost a week trying to find out the correct polynomial/algorithm/table combination, but with no luck. If it helps, I came up with bit-wise algorithm that corresponds to table-driven code above, although that was not so hard after all:
uint32 crc32(uint16* data, uint32 len, uint32 crc)
{
uint32 i;
while(len--)
{
for(i = 0; i < 16; i++)
{
// #define POLY 0x04C11DB7
crc = (crc << 1) ^ (((crc ^ *data) & 0x80000000) ? POLY : 0);
}
crc ^= *data++;
}
return crc;
}
Here are expected results - first 2 16-bit words make the needed unknown CRC and the rest is the known data itself (by feeding these examples to provided algorithm, the result is 0).
{0x3288, 0xD244, 0xCDEF, 0x89AB, 0x4567, 0x0123}
{0xC704, 0xDD7B, 0x0000} - append as many zeros as you like, the result is the same
{0xCEBD, 0x1ADD, 0xFFFF}
{0x81AB, 0xB932, 0xFFFF, 0xFFFF}
{0x0857, 0x0465, 0x0000, 0x0123}
{0x1583, 0xD959, 0x0123}
^ ^
| |
unknown bytes that I need to calculate
I think testing this on 0xFFFF or 0x0000 words is convenient because the direction of calculation and endianess is not important (I hope :D). So be careful to use other test bytes, because the direction of calculation is quite devious :D. Also you can see that by feeding only zeros to the algorithm (both forward and backward), the result is so-called residue (0xC704DD7B), that may be helpful.
So...I wrote at least 10 different functions (bite-wise, tables, combination of polynomials etc.) trying to solve this, but with no luck. I give you here the function in which I put my hopes into. It's 'reversed' algorithm of the table-driven one above, with different table of course. The problem is that the only correct CRC I get from that is with all 0s message and that's not so unexpected. Also I have written the reversed implementation of the bit-wise algorithm (reversed shifts, etc.), but that one returns only the first byte correctly.
Here is the table-driven one, pointer to data should point to the last element of the message and crc input should be the requested crc (0s for the whole message or you can maybe take another approach - that the last 4 bytes of message are the CRC you are looking for: Calculating CRC initial value instead of appending the CRC to payload) :
uint32 crc32tabrev(uint16* data, uint32 len, uint32 crc)
{
uint8 nibble;
int i;
while(len--)
{
for(i = 0; i < 4; i++)
{
nibble = (*data >> i*4) & 0x0F;
crc = (crc >> 4) ^ revtab[((crc ^ nibble) & 0x0F)];
}
data--;
}
return reverse(crc); //reverse() flips all bits around center (MSB <-> LSB ...)
}
The table, which I hope is 'the chosen one':
static const uint32 revtab[16]=
{
0x00000000, 0x1DB71064, 0x3B6E20C8, 0x26D930AC,
0x76DC4190, 0x6B6B51F4, 0x4DB26158, 0x5005713C,
0xEDB88320, 0xF00F9344, 0xD6D6A3E8, 0xCB61B38C,
0x9B64C2B0, 0x86D3D2D4, 0xA00AE278, 0xBDBDF21C
};
As you can see, this algorithm has some perks which make me run in circles and I think I'm maybe on the right track, but I'm missing something. I hope an extra pair of eyes will see what I can not. I'm sorry for the long post (no potato :D), but I think all of that explanation was neccessary. Thank you in advance for insight or advice.
I will answer for your CRC specification, that of a CRC-32/MPEG-2. I will have to ignore your attempts at calculating that CRC, since they are incorrect.
Anyway, to answer your question, I happen to have written a program that solves this problem. It is called spoof.c. It very rapidly computes what bits to change in a message to get a desired CRC. It does this in order log(n) time, where n is the length of the message. Here is an example:
Let's take the nine-byte message 123456789 (those digits represented in ASCII). We will prepend it with four zero bytes, which we will change to get the desired CRC at the end. The message in hex is then: 00 00 00 00 31 32 33 34 35 36 37 38 39. Now we compute the CRC-32/MPEG-2 for that message. We get 373c5870.
Now we run spoof with this input, which is the CRC length in bits, the fact that it is not reflected, the polynomial, the CRC we just computed, the length of the message in bytes, and all 32 bit locations in the first four bytes (which is what we are allowing spoof to change):
32 0 04C11DB7
373c5870 13
0 0 1 2 3 4 5 6 7
1 0 1 2 3 4 5 6 7
2 0 1 2 3 4 5 6 7
3 0 1 2 3 4 5 6 7
It gives this output with what bits in those first four bytes to set:
invert these bits in the sequence:
offset bit
0 1
0 2
0 4
0 5
0 6
1 0
1 2
1 5
1 7
2 0
2 2
2 5
2 6
2 7
3 0
3 1
3 2
3 4
3 5
3 7
We then set the first four bytes to: 76 a5 e5 b7. We then test by computing the CRC-32/MPEG-2 of the message 76 a5 e5 b7 31 32 33 34 35 36 37 38 39 and we get 00000000, the desired result.
You can adapt spoof.c to your application.
Here is an example that correctly computes the CRC-32/MPEG-2 on a stream of bytes using a bit-wise algorithm:
uint32_t crc32m(uint32_t crc, const unsigned char *buf, size_t len)
{
int k;
while (len--) {
crc ^= (uint32_t)(*buf++) << 24;
for (k = 0; k < 8; k++)
crc = crc & 0x80000000 ? (crc << 1) ^ 0x04c11db7 : crc << 1;
}
return crc;
}
and with a nybble-wise algorithm using the table in the question (which is correct):
uint32_t crc_table[] = {
0x00000000, 0x04C11DB7, 0x09823B6E, 0x0D4326D9,
0x130476DC, 0x17C56B6B, 0x1A864DB2, 0x1E475005,
0x2608EDB8, 0x22C9F00F, 0x2F8AD6D6, 0x2B4BCB61,
0x350C9B64, 0x31CD86D3, 0x3C8EA00A, 0x384FBDBD
};
uint32_t crc32m_nyb(uint32_t crc, const unsigned char *buf, size_t len)
{
while (len--) {
crc ^= (uint32_t)(*buf++) << 24;
crc = (crc << 4) ^ crc_table[crc >> 28];
crc = (crc << 4) ^ crc_table[crc >> 28];
}
return crc;
}
In both cases, the initial CRC must be 0xffffffff.
Alternate approach. Assumes xorout = 0, if not, then after calculating the normal crc, then crc ^= xorout to remove it. The method here multiplies the normal crc by (1/2)%(crc polynomial) raised to (message size in bits) power % (crc polynomial) equivalent to cycling it backwards. If the message size is fixed, then the mapping is fixed and time complexity is O(1). Otherwise, it's O(log(n)).
This example code uses Visual Studio and an intrinsic for carryless multiply (PCLMULQDQ), which uses XMM (128 bit) registers. Visual Studio uses __m128i type to represent integer XMM values.
#include <stdio.h>
#include <stdlib.h>
#include <intrin.h>
typedef unsigned char uint8_t;
typedef unsigned int uint32_t;
typedef unsigned long long uint64_t;
#define POLY (0x104c11db7ull)
#define POLYM ( 0x04c11db7u)
static uint32_t crctbl[256];
static __m128i poly; /* poly */
static __m128i invpoly; /* 2^64 / POLY */
void GenMPoly(void) /* generate __m128i poly info */
{
uint64_t N = 0x100000000ull;
uint64_t Q = 0;
for(size_t i = 0; i < 33; i++){
Q <<= 1;
if(N&0x100000000ull){
Q |= 1;
N ^= POLY;
}
N <<= 1;
}
poly.m128i_u64[0] = POLY;
invpoly.m128i_u64[0] = Q;
}
void GenTbl(void) /* generate crc table */
{
uint32_t crc;
uint32_t c;
uint32_t i;
for(c = 0; c < 0x100; c++){
crc = c<<24;
for(i = 0; i < 8; i++)
/* assumes twos complement */
crc = (crc<<1)^((0-(crc>>31))&POLYM);
crctbl[c] = crc;
}
}
uint32_t GenCrc(uint8_t * bfr, size_t size) /* generate crc */
{
uint32_t crc = 0xffffffffu;
while(size--)
crc = (crc<<8)^crctbl[(crc>>24)^*bfr++];
return(crc);
}
/* carryless multiply modulo poly */
uint32_t MpyModPoly(uint32_t a, uint32_t b) /* (a*b)%poly */
{
__m128i ma, mb, mp, mt;
ma.m128i_u64[0] = a;
mb.m128i_u64[0] = b;
mp = _mm_clmulepi64_si128(ma, mb, 0x00); /* p[0] = a*b */
mt = _mm_clmulepi64_si128(mp, invpoly, 0x00); /* t[1] = (p[0]*((2^64)/POLY))>>64 */
mt = _mm_clmulepi64_si128(mt, poly, 0x01); /* t[0] = t[1]*POLY */
return mp.m128i_u32[0] ^ mt.m128i_u32[0]; /* ret = p[0] ^ t[0] */
}
/* exponentiate by repeated squaring modulo poly */
uint32_t PowModPoly(uint32_t a, uint32_t b) /* pow(a,b)%poly */
{
uint32_t prd = 0x1u; /* current product */
uint32_t sqr = a; /* current square */
while(b){
if(b&1)
prd = MpyModPoly(prd, sqr);
sqr = MpyModPoly(sqr, sqr);
b >>= 1;
}
return prd;
}
int main()
{
uint32_t inv; /* 1/2 % poly, constant */
uint32_t fix; /* fix value, constant if msg size fixed */
uint32_t crc; /* crc at end of msg */
uint32_t pre; /* prefix for msg */
uint8_t msg[13] = {0x00,0x00,0x00,0x00,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39};
GenMPoly(); /* generate __m128i polys */
GenTbl(); /* generate crc table */
inv = PowModPoly(2, 0xfffffffeu); /* inv = 2^(2^32-2) % Poly = 1/2 % poly */
fix = PowModPoly(inv, 8*sizeof(msg)); /* fix value */
crc = GenCrc(msg, sizeof(msg)); /* calculate normal crc */
pre = MpyModPoly(fix, crc); /* convert to prefix */
printf("crc = %08x pre = %08x ", crc, pre);
msg[0] = (uint8_t)(pre>>24); /* store prefix in msg */
msg[1] = (uint8_t)(pre>>16);
msg[2] = (uint8_t)(pre>> 8);
msg[3] = (uint8_t)(pre>> 0);
crc = GenCrc(msg, sizeof(msg)); /* check result */
if(crc == 0)
printf("passed\n");
else
printf("failed\n");
return 0;
}
Well, few hours after my question, someone whose name I don't remember posted an answer to my question which turned out to be correct. Somehow this answer got completely deleted, I don't know why or who did it, but I'd like to thank to this person and in the case you will see this, please post your answer again and I'll delete this one. But for other users, here's his answer that worked for me, thank you again, mysterious one (unfortunately, I can't replicate his notes and suggestions well enough, just the code itself):
Edit: The original answer came from user samgak, so this stays here until he'll post his answer.
The reverse CRC algorithm:
uint32 revcrc32(uint16* data, uint32 len, uint32 crc)
{
uint32 i;
data += len - 1;
while(len--)
{
crc ^= *data--;
for(i = 0; i < 16; i++)
{
uint32 crc1 = ((crc ^ POLY) >> 1) | 0x80000000;
uint32 crc2 = crc >> 1;
if(((crc1 << 1) ^ (((crc1 ^ *data) & 0x80000000) ? POLY : 0)) == crc)
crc = crc1;
else if(((crc2 << 1) ^ (((crc2 ^ *data) & 0x80000000) ? POLY : 0)) == crc)
crc = crc2;
}
}
return crc;
}
Find patch bytes:
#define CRC_OF_ZERO 0xb7647d
void bruteforcecrc32(uint32 targetcrc)
{
// compute prefixes:
uint16 j;
for(j = 0; j <= 0xffff; j++)
{
uint32 crc = revcrc32(&j, 1, targetcrc);
if((crc >> 16) == (CRC_OF_ZERO >> 16))
{
printf("prefixes: %04lX %04lX\n", (crc ^ CRC_OF_ZERO) & 0xffff, (uint32)j);
return;
}
}
}
Usage:
uint16 test[] = {0x0123, 0x4567, 0x89AB, 0xCDEF}; // prefix should be 0x0CD8236A
bruteforcecrc32(revcrc32(test, 4, 0L));