I need to calculate a CRC-64 using this setup into this wonderful website: http://www.sunshine2k.de/coding/javascript/crc/crc_js.html
As you can see I require "input reflected" and that means that I need to reverse the bit order of any byte (a bit annoying).
For the moment, I implemented this with a lookup table (for example 0x55 -> 0xAA), but I was wondering if there is any property of CRC which can be used to be more efficient.
This is my code (in C):
static const unsigned long long CRC64_TABLE[256] = {
0x0000000000000000, 0x42F0E1EBA9EA3693, 0x85E1C3D753D46D26, 0xC711223CFA3E5BB5, 0x493366450E42ECDF, 0x0BC387AEA7A8DA4C, 0xCCD2A5925D9681F9, 0x8E224479F47CB76A,
0x9266CC8A1C85D9BE, 0xD0962D61B56FEF2D, 0x17870F5D4F51B498, 0x5577EEB6E6BB820B, 0xDB55AACF12C73561, 0x99A54B24BB2D03F2, 0x5EB4691841135847, 0x1C4488F3E8F96ED4,
0x663D78FF90E185EF, 0x24CD9914390BB37C, 0xE3DCBB28C335E8C9, 0xA12C5AC36ADFDE5A, 0x2F0E1EBA9EA36930, 0x6DFEFF5137495FA3, 0xAAEFDD6DCD770416, 0xE81F3C86649D3285,
0xF45BB4758C645C51, 0xB6AB559E258E6AC2, 0x71BA77A2DFB03177, 0x334A9649765A07E4, 0xBD68D2308226B08E, 0xFF9833DB2BCC861D, 0x388911E7D1F2DDA8, 0x7A79F00C7818EB3B,
0xCC7AF1FF21C30BDE, 0x8E8A101488293D4D, 0x499B3228721766F8, 0x0B6BD3C3DBFD506B, 0x854997BA2F81E701, 0xC7B97651866BD192, 0x00A8546D7C558A27, 0x4258B586D5BFBCB4,
0x5E1C3D753D46D260, 0x1CECDC9E94ACE4F3, 0xDBFDFEA26E92BF46, 0x990D1F49C77889D5, 0x172F5B3033043EBF, 0x55DFBADB9AEE082C, 0x92CE98E760D05399, 0xD03E790CC93A650A,
0xAA478900B1228E31, 0xE8B768EB18C8B8A2, 0x2FA64AD7E2F6E317, 0x6D56AB3C4B1CD584, 0xE374EF45BF6062EE, 0xA1840EAE168A547D, 0x66952C92ECB40FC8, 0x2465CD79455E395B,
0x3821458AADA7578F, 0x7AD1A461044D611C, 0xBDC0865DFE733AA9, 0xFF3067B657990C3A, 0x711223CFA3E5BB50, 0x33E2C2240A0F8DC3, 0xF4F3E018F031D676, 0xB60301F359DBE0E5,
0xDA050215EA6C212F, 0x98F5E3FE438617BC, 0x5FE4C1C2B9B84C09, 0x1D14202910527A9A, 0x93366450E42ECDF0, 0xD1C685BB4DC4FB63, 0x16D7A787B7FAA0D6, 0x5427466C1E109645,
0x4863CE9FF6E9F891, 0x0A932F745F03CE02, 0xCD820D48A53D95B7, 0x8F72ECA30CD7A324, 0x0150A8DAF8AB144E, 0x43A04931514122DD, 0x84B16B0DAB7F7968, 0xC6418AE602954FFB,
0xBC387AEA7A8DA4C0, 0xFEC89B01D3679253, 0x39D9B93D2959C9E6, 0x7B2958D680B3FF75, 0xF50B1CAF74CF481F, 0xB7FBFD44DD257E8C, 0x70EADF78271B2539, 0x321A3E938EF113AA,
0x2E5EB66066087D7E, 0x6CAE578BCFE24BED, 0xABBF75B735DC1058, 0xE94F945C9C3626CB, 0x676DD025684A91A1, 0x259D31CEC1A0A732, 0xE28C13F23B9EFC87, 0xA07CF2199274CA14,
0x167FF3EACBAF2AF1, 0x548F120162451C62, 0x939E303D987B47D7, 0xD16ED1D631917144, 0x5F4C95AFC5EDC62E, 0x1DBC74446C07F0BD, 0xDAAD56789639AB08, 0x985DB7933FD39D9B,
0x84193F60D72AF34F, 0xC6E9DE8B7EC0C5DC, 0x01F8FCB784FE9E69, 0x43081D5C2D14A8FA, 0xCD2A5925D9681F90, 0x8FDAB8CE70822903, 0x48CB9AF28ABC72B6, 0x0A3B7B1923564425,
0x70428B155B4EAF1E, 0x32B26AFEF2A4998D, 0xF5A348C2089AC238, 0xB753A929A170F4AB, 0x3971ED50550C43C1, 0x7B810CBBFCE67552, 0xBC902E8706D82EE7, 0xFE60CF6CAF321874,
0xE224479F47CB76A0, 0xA0D4A674EE214033, 0x67C58448141F1B86, 0x253565A3BDF52D15, 0xAB1721DA49899A7F, 0xE9E7C031E063ACEC, 0x2EF6E20D1A5DF759, 0x6C0603E6B3B7C1CA,
0xF6FAE5C07D3274CD, 0xB40A042BD4D8425E, 0x731B26172EE619EB, 0x31EBC7FC870C2F78, 0xBFC9838573709812, 0xFD39626EDA9AAE81, 0x3A28405220A4F534, 0x78D8A1B9894EC3A7,
0x649C294A61B7AD73, 0x266CC8A1C85D9BE0, 0xE17DEA9D3263C055, 0xA38D0B769B89F6C6, 0x2DAF4F0F6FF541AC, 0x6F5FAEE4C61F773F, 0xA84E8CD83C212C8A, 0xEABE6D3395CB1A19,
0x90C79D3FEDD3F122, 0xD2377CD44439C7B1, 0x15265EE8BE079C04, 0x57D6BF0317EDAA97, 0xD9F4FB7AE3911DFD, 0x9B041A914A7B2B6E, 0x5C1538ADB04570DB, 0x1EE5D94619AF4648,
0x02A151B5F156289C, 0x4051B05E58BC1E0F, 0x87409262A28245BA, 0xC5B073890B687329, 0x4B9237F0FF14C443, 0x0962D61B56FEF2D0, 0xCE73F427ACC0A965, 0x8C8315CC052A9FF6,
0x3A80143F5CF17F13, 0x7870F5D4F51B4980, 0xBF61D7E80F251235, 0xFD913603A6CF24A6, 0x73B3727A52B393CC, 0x31439391FB59A55F, 0xF652B1AD0167FEEA, 0xB4A25046A88DC879,
0xA8E6D8B54074A6AD, 0xEA16395EE99E903E, 0x2D071B6213A0CB8B, 0x6FF7FA89BA4AFD18, 0xE1D5BEF04E364A72, 0xA3255F1BE7DC7CE1, 0x64347D271DE22754, 0x26C49CCCB40811C7,
0x5CBD6CC0CC10FAFC, 0x1E4D8D2B65FACC6F, 0xD95CAF179FC497DA, 0x9BAC4EFC362EA149, 0x158E0A85C2521623, 0x577EEB6E6BB820B0, 0x906FC95291867B05, 0xD29F28B9386C4D96,
0xCEDBA04AD0952342, 0x8C2B41A1797F15D1, 0x4B3A639D83414E64, 0x09CA82762AAB78F7, 0x87E8C60FDED7CF9D, 0xC51827E4773DF90E, 0x020905D88D03A2BB, 0x40F9E43324E99428,
0x2CFFE7D5975E55E2, 0x6E0F063E3EB46371, 0xA91E2402C48A38C4, 0xEBEEC5E96D600E57, 0x65CC8190991CB93D, 0x273C607B30F68FAE, 0xE02D4247CAC8D41B, 0xA2DDA3AC6322E288,
0xBE992B5F8BDB8C5C, 0xFC69CAB42231BACF, 0x3B78E888D80FE17A, 0x7988096371E5D7E9, 0xF7AA4D1A85996083, 0xB55AACF12C735610, 0x724B8ECDD64D0DA5, 0x30BB6F267FA73B36,
0x4AC29F2A07BFD00D, 0x08327EC1AE55E69E, 0xCF235CFD546BBD2B, 0x8DD3BD16FD818BB8, 0x03F1F96F09FD3CD2, 0x41011884A0170A41, 0x86103AB85A2951F4, 0xC4E0DB53F3C36767,
0xD8A453A01B3A09B3, 0x9A54B24BB2D03F20, 0x5D45907748EE6495, 0x1FB5719CE1045206, 0x919735E51578E56C, 0xD367D40EBC92D3FF, 0x1476F63246AC884A, 0x568617D9EF46BED9,
0xE085162AB69D5E3C, 0xA275F7C11F7768AF, 0x6564D5FDE549331A, 0x279434164CA30589, 0xA9B6706FB8DFB2E3, 0xEB46918411358470, 0x2C57B3B8EB0BDFC5, 0x6EA7525342E1E956,
0x72E3DAA0AA188782, 0x30133B4B03F2B111, 0xF7021977F9CCEAA4, 0xB5F2F89C5026DC37, 0x3BD0BCE5A45A6B5D, 0x79205D0E0DB05DCE, 0xBE317F32F78E067B, 0xFCC19ED95E6430E8,
0x86B86ED5267CDBD3, 0xC4488F3E8F96ED40, 0x0359AD0275A8B6F5, 0x41A94CE9DC428066, 0xCF8B0890283E370C, 0x8D7BE97B81D4019F, 0x4A6ACB477BEA5A2A, 0x089A2AACD2006CB9,
0x14DEA25F3AF9026D, 0x562E43B4931334FE, 0x913F6188692D6F4B, 0xD3CF8063C0C759D8, 0x5DEDC41A34BBEEB2, 0x1F1D25F19D51D821, 0xD80C07CD676F8394, 0x9AFCE626CE85B507
};
static const unsigned char REVERSE_BITS_TABLE[256] = {
// 0 1 2 3 4 5 6 7 8 9 A B C D E F
0x00, 0x80, 0x40, 0xC0, 0x20, 0xA0, 0x60, 0xE0, 0x10, 0x90, 0x50, 0xD0, 0x30, 0xB0, 0x70, 0xF0,
0x08, 0x88, 0x48, 0xC8, 0x28, 0xA8, 0x68, 0xE8, 0x18, 0x98, 0x58, 0xD8, 0x38, 0xB8, 0x78, 0xF8,
0x04, 0x84, 0x44, 0xC4, 0x24, 0xA4, 0x64, 0xE4, 0x14, 0x94, 0x54, 0xD4, 0x34, 0xB4, 0x74, 0xF4,
0x0C, 0x8C, 0x4C, 0xCC, 0x2C, 0xAC, 0x6C, 0xEC, 0x1C, 0x9C, 0x5C, 0xDC, 0x3C, 0xBC, 0x7C, 0xFC,
0x02, 0x82, 0x42, 0xC2, 0x22, 0xA2, 0x62, 0xE2, 0x12, 0x92, 0x52, 0xD2, 0x32, 0xB2, 0x72, 0xF2,
0x0A, 0x8A, 0x4A, 0xCA, 0x2A, 0xAA, 0x6A, 0xEA, 0x1A, 0x9A, 0x5A, 0xDA, 0x3A, 0xBA, 0x7A, 0xFA,
0x06, 0x86, 0x46, 0xC6, 0x26, 0xA6, 0x66, 0xE6, 0x16, 0x96, 0x56, 0xD6, 0x36, 0xB6, 0x76, 0xF6,
0x0E, 0x8E, 0x4E, 0xCE, 0x2E, 0xAE, 0x6E, 0xEE, 0x1E, 0x9E, 0x5E, 0xDE, 0x3E, 0xBE, 0x7E, 0xFE,
0x01, 0x81, 0x41, 0xC1, 0x21, 0xA1, 0x61, 0xE1, 0x11, 0x91, 0x51, 0xD1, 0x31, 0xB1, 0x71, 0xF1,
0x09, 0x89, 0x49, 0xC9, 0x29, 0xA9, 0x69, 0xE9, 0x19, 0x99, 0x59, 0xD9, 0x39, 0xB9, 0x79, 0xF9,
0x05, 0x85, 0x45, 0xC5, 0x25, 0xA5, 0x65, 0xE5, 0x15, 0x95, 0x55, 0xD5, 0x35, 0xB5, 0x75, 0xF5,
0x0D, 0x8D, 0x4D, 0xCD, 0x2D, 0xAD, 0x6D, 0xED, 0x1D, 0x9D, 0x5D, 0xDD, 0x3D, 0xBD, 0x7D, 0xFD,
0x03, 0x83, 0x43, 0xC3, 0x23, 0xA3, 0x63, 0xE3, 0x13, 0x93, 0x53, 0xD3, 0x33, 0xB3, 0x73, 0xF3,
0x0B, 0x8B, 0x4B, 0xCB, 0x2B, 0xAB, 0x6B, 0xEB, 0x1B, 0x9B, 0x5B, 0xDB, 0x3B, 0xBB, 0x7B, 0xFB,
0x07, 0x87, 0x47, 0xC7, 0x27, 0xA7, 0x67, 0xE7, 0x17, 0x97, 0x57, 0xD7, 0x37, 0xB7, 0x77, 0xF7,
0x0F, 0x8F, 0x4F, 0xCF, 0x2F, 0xAF, 0x6F, 0xEF, 0x1F, 0x9F, 0x5F, 0xDF, 0x3F, 0xBF, 0x7F, 0xFF
};
unsigned long long calculateCRC64(unsigned char* data, unsigned long size)
{
unsigned long long myCRC64 = 0;
for(unsigned long i = 0; i < size; i++)
myCRC64 = CRC64_TABLE[((myCRC64 >> 56) ^ REVERSE_BITS_TABLE[data[i]]) & 0xFF] ^ (myCRC64 << 8);
return myCRC64;
}
Thanks in advance for any help!
Update: actually I'm wondering if some properties of CRC allow me to get the same result using reflected table, opposite shifting, reciprocal / reverse polynomial... etc. That would let skip the step of reversing bits.
Solution: using reflected table, changing shift direction and reversing bits just of the very last result:
static const unsigned long long CRC64_TABLE_REF[256] = {
0x0000000000000000, 0xB32E4CBE03A75F6F, 0xF4843657A840A05B, 0x47AA7AE9ABE7FF34, 0x7BD0C384FF8F5E33, 0xC8FE8F3AFC28015C, 0x8F54F5D357CFFE68, 0x3C7AB96D5468A107,
0xF7A18709FF1EBC66, 0x448FCBB7FCB9E309, 0x0325B15E575E1C3D, 0xB00BFDE054F94352, 0x8C71448D0091E255, 0x3F5F08330336BD3A, 0x78F572DAA8D1420E, 0xCBDB3E64AB761D61,
0x7D9BA13851336649, 0xCEB5ED8652943926, 0x891F976FF973C612, 0x3A31DBD1FAD4997D, 0x064B62BCAEBC387A, 0xB5652E02AD1B6715, 0xF2CF54EB06FC9821, 0x41E11855055BC74E,
0x8A3A2631AE2DDA2F, 0x39146A8FAD8A8540, 0x7EBE1066066D7A74, 0xCD905CD805CA251B, 0xF1EAE5B551A2841C, 0x42C4A90B5205DB73, 0x056ED3E2F9E22447, 0xB6409F5CFA457B28,
0xFB374270A266CC92, 0x48190ECEA1C193FD, 0x0FB374270A266CC9, 0xBC9D3899098133A6, 0x80E781F45DE992A1, 0x33C9CD4A5E4ECDCE, 0x7463B7A3F5A932FA, 0xC74DFB1DF60E6D95,
0x0C96C5795D7870F4, 0xBFB889C75EDF2F9B, 0xF812F32EF538D0AF, 0x4B3CBF90F69F8FC0, 0x774606FDA2F72EC7, 0xC4684A43A15071A8, 0x83C230AA0AB78E9C, 0x30EC7C140910D1F3,
0x86ACE348F355AADB, 0x3582AFF6F0F2F5B4, 0x7228D51F5B150A80, 0xC10699A158B255EF, 0xFD7C20CC0CDAF4E8, 0x4E526C720F7DAB87, 0x09F8169BA49A54B3, 0xBAD65A25A73D0BDC,
0x710D64410C4B16BD, 0xC22328FF0FEC49D2, 0x85895216A40BB6E6, 0x36A71EA8A7ACE989, 0x0ADDA7C5F3C4488E, 0xB9F3EB7BF06317E1, 0xFE5991925B84E8D5, 0x4D77DD2C5823B7BA,
0x64B62BCAEBC387A1, 0xD7986774E864D8CE, 0x90321D9D438327FA, 0x231C512340247895, 0x1F66E84E144CD992, 0xAC48A4F017EB86FD, 0xEBE2DE19BC0C79C9, 0x58CC92A7BFAB26A6,
0x9317ACC314DD3BC7, 0x2039E07D177A64A8, 0x67939A94BC9D9B9C, 0xD4BDD62ABF3AC4F3, 0xE8C76F47EB5265F4, 0x5BE923F9E8F53A9B, 0x1C4359104312C5AF, 0xAF6D15AE40B59AC0,
0x192D8AF2BAF0E1E8, 0xAA03C64CB957BE87, 0xEDA9BCA512B041B3, 0x5E87F01B11171EDC, 0x62FD4976457FBFDB, 0xD1D305C846D8E0B4, 0x96797F21ED3F1F80, 0x2557339FEE9840EF,
0xEE8C0DFB45EE5D8E, 0x5DA24145464902E1, 0x1A083BACEDAEFDD5, 0xA9267712EE09A2BA, 0x955CCE7FBA6103BD, 0x267282C1B9C65CD2, 0x61D8F8281221A3E6, 0xD2F6B4961186FC89,
0x9F8169BA49A54B33, 0x2CAF25044A02145C, 0x6B055FEDE1E5EB68, 0xD82B1353E242B407, 0xE451AA3EB62A1500, 0x577FE680B58D4A6F, 0x10D59C691E6AB55B, 0xA3FBD0D71DCDEA34,
0x6820EEB3B6BBF755, 0xDB0EA20DB51CA83A, 0x9CA4D8E41EFB570E, 0x2F8A945A1D5C0861, 0x13F02D374934A966, 0xA0DE61894A93F609, 0xE7741B60E174093D, 0x545A57DEE2D35652,
0xE21AC88218962D7A, 0x5134843C1B317215, 0x169EFED5B0D68D21, 0xA5B0B26BB371D24E, 0x99CA0B06E7197349, 0x2AE447B8E4BE2C26, 0x6D4E3D514F59D312, 0xDE6071EF4CFE8C7D,
0x15BB4F8BE788911C, 0xA6950335E42FCE73, 0xE13F79DC4FC83147, 0x521135624C6F6E28, 0x6E6B8C0F1807CF2F, 0xDD45C0B11BA09040, 0x9AEFBA58B0476F74, 0x29C1F6E6B3E0301B,
0xC96C5795D7870F42, 0x7A421B2BD420502D, 0x3DE861C27FC7AF19, 0x8EC62D7C7C60F076, 0xB2BC941128085171, 0x0192D8AF2BAF0E1E, 0x4638A2468048F12A, 0xF516EEF883EFAE45,
0x3ECDD09C2899B324, 0x8DE39C222B3EEC4B, 0xCA49E6CB80D9137F, 0x7967AA75837E4C10, 0x451D1318D716ED17, 0xF6335FA6D4B1B278, 0xB199254F7F564D4C, 0x02B769F17CF11223,
0xB4F7F6AD86B4690B, 0x07D9BA1385133664, 0x4073C0FA2EF4C950, 0xF35D8C442D53963F, 0xCF273529793B3738, 0x7C0979977A9C6857, 0x3BA3037ED17B9763, 0x888D4FC0D2DCC80C,
0x435671A479AAD56D, 0xF0783D1A7A0D8A02, 0xB7D247F3D1EA7536, 0x04FC0B4DD24D2A59, 0x3886B22086258B5E, 0x8BA8FE9E8582D431, 0xCC0284772E652B05, 0x7F2CC8C92DC2746A,
0x325B15E575E1C3D0, 0x8175595B76469CBF, 0xC6DF23B2DDA1638B, 0x75F16F0CDE063CE4, 0x498BD6618A6E9DE3, 0xFAA59ADF89C9C28C, 0xBD0FE036222E3DB8, 0x0E21AC88218962D7,
0xC5FA92EC8AFF7FB6, 0x76D4DE52895820D9, 0x317EA4BB22BFDFED, 0x8250E80521188082, 0xBE2A516875702185, 0x0D041DD676D77EEA, 0x4AAE673FDD3081DE, 0xF9802B81DE97DEB1,
0x4FC0B4DD24D2A599, 0xFCEEF8632775FAF6, 0xBB44828A8C9205C2, 0x086ACE348F355AAD, 0x34107759DB5DFBAA, 0x873E3BE7D8FAA4C5, 0xC094410E731D5BF1, 0x73BA0DB070BA049E,
0xB86133D4DBCC19FF, 0x0B4F7F6AD86B4690, 0x4CE50583738CB9A4, 0xFFCB493D702BE6CB, 0xC3B1F050244347CC, 0x709FBCEE27E418A3, 0x3735C6078C03E797, 0x841B8AB98FA4B8F8,
0xADDA7C5F3C4488E3, 0x1EF430E13FE3D78C, 0x595E4A08940428B8, 0xEA7006B697A377D7, 0xD60ABFDBC3CBD6D0, 0x6524F365C06C89BF, 0x228E898C6B8B768B, 0x91A0C532682C29E4,
0x5A7BFB56C35A3485, 0xE955B7E8C0FD6BEA, 0xAEFFCD016B1A94DE, 0x1DD181BF68BDCBB1, 0x21AB38D23CD56AB6, 0x9285746C3F7235D9, 0xD52F0E859495CAED, 0x6601423B97329582,
0xD041DD676D77EEAA, 0x636F91D96ED0B1C5, 0x24C5EB30C5374EF1, 0x97EBA78EC690119E, 0xAB911EE392F8B099, 0x18BF525D915FEFF6, 0x5F1528B43AB810C2, 0xEC3B640A391F4FAD,
0x27E05A6E926952CC, 0x94CE16D091CE0DA3, 0xD3646C393A29F297, 0x604A2087398EADF8, 0x5C3099EA6DE60CFF, 0xEF1ED5546E415390, 0xA8B4AFBDC5A6ACA4, 0x1B9AE303C601F3CB,
0x56ED3E2F9E224471, 0xE5C372919D851B1E, 0xA26908783662E42A, 0x114744C635C5BB45, 0x2D3DFDAB61AD1A42, 0x9E13B115620A452D, 0xD9B9CBFCC9EDBA19, 0x6A978742CA4AE576,
0xA14CB926613CF817, 0x1262F598629BA778, 0x55C88F71C97C584C, 0xE6E6C3CFCADB0723, 0xDA9C7AA29EB3A624, 0x69B2361C9D14F94B, 0x2E184CF536F3067F, 0x9D36004B35545910,
0x2B769F17CF112238, 0x9858D3A9CCB67D57, 0xDFF2A94067518263, 0x6CDCE5FE64F6DD0C, 0x50A65C93309E7C0B, 0xE388102D33392364, 0xA4226AC498DEDC50, 0x170C267A9B79833F,
0xDCD7181E300F9E5E, 0x6FF954A033A8C131, 0x28532E49984F3E05, 0x9B7D62F79BE8616A, 0xA707DB9ACF80C06D, 0x14299724CC279F02, 0x5383EDCD67C06036, 0xE0ADA17364673F59
};
unsigned long long calculateCRC64(unsigned char* data, unsigned long size)
{
unsigned long long myCRC64 = 0;
for(unsigned long i = 0; i < size; i++)
myCRC64 = CRC64_TABLE_REF[(myCRC64 ^ data[i]) & 0xFF] ^ (myCRC64 >> 8);
unsigned long long tmp = myCRC64;
myCRC64 = 0;
for (int i = 0; i < 64; i++) {
myCRC64 = (myCRC64 << 1) | (tmp & 1);
tmp >>= 1;
}
return myCRC64;
}
Thanks to everybody for the help!
actually I'm wondering if some properties of CRC allow me to get the same result using reflected table, opposite shifting, reciprocal / reverse polynomial... etc. That would let skip the step of reversing bits.
There is such a property, and you only need one reverse, in the end. The whole logic of the CRC can be bit-reversed*, so that the bits stream the opposite way and don't need to be reversed during the calculation itself:
unsigned long long CRC64(unsigned char* data, unsigned long size)
{
unsigned long long crc = 0;
for(unsigned long i = 0; i < size; i++)
crc = CRC64_TABLE_R[(crc & 0xff) ^ data[i]] ^ (crc >> 8);
return rbit(crc);
}
CRC64_TABLE_R is like CRC64_TABLE, but with the entries and indexes both bit-reversed. So they relate to each other like this: (naturally you would hard-code this table)
for (int i = 0; i < 256; i++) {
CRC64_TABLE_R[REVERSE_BITS_TABLE[i]] = rbit(CRC64_TABLE[i]);
}
rbit is just some function that reverses bits, I used this really dumb one, use a better one if you want but it's only used once:
unsigned long long rbit(unsigned long long n)
{
unsigned long long x = 0;
for (int i = 0; i < 64; i++) {
x = (x << 1) | (n & 1);
n >>= 1;
}
return x;
}
*: really what this means is that the integer holding the bits during the calculation is viewed in reverse, with what would normally be its least significant bit instead held in its most significant bit etc. So depending on how you look at it, the calculation is not even different. The same bits are created, but we hold them in a variable differently.
The code just needs to switch from a left shifting CRC to a right shifting CRC, with the polynomial bits reversed. The "#if 0" is used to select between two common polynomials, in this case, "#if 0" will use CRC64 ecma, which is what is shown in the question's CRC table. (0x42F0E1EBA9EA3693 bit reversed == 0xC96C5795D7870F42).
uint64_t crctbl[256];
void gentbl(void)
{
uint64_t crc;
uint64_t b;
uint64_t c;
uint64_t i;
for(c = 0; c < 0x100; c++){
crc = c;
for(i = 0; i < 8; i++){
b = crc&1;
crc >>= 1;
#if 0
crc ^= (0 - b) & 0xd800000000000000ull; // crc64 iso
#else
crc ^= (0 - b) & 0xc96c5795d7870f42ull; // crc64 ecma
#endif
}
crctbl[c] = crc;
}
}
uint64_t crc64c(uint64_t crc64, uint8_t * bfr, size_t size)
{
uint64_t crc = crc64;
while(size--)
crc = (crc >> 8) ^ crctbl[(crc & 0xff)^*bfr++];
return(crc);
}
"Most efficient way" - if running on X86 X64 with carryless multiply, a really fast CRC, over 20 times as fast as the table method shown above (For a CRC of 256 MB of data, Intel Core i7-10510U .475 seconds table, .015 seconds pclmulqdq, 31.5 times as fast) , can be performed using pclmulqdq instruction and xmm registers. Link to a zip of a 520+ line assembly file, for Visual Studio's MASM (ML64.EXE), for 64 bit reflected CRC, with the same "if 0" as the gentbl() function above to choose between the two CRC polynomials.
http://rcgldr.net/misc/crc64ra.zip
Intel document explaining the algorithm for a 32 bit CRC:
https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
A carryless multiply of two 64 bit non-reflected operands produces a 127 bit product in bits 126 (msb) to bit 0 (lsb). For reflected operands, a carryless multiply effectively multiplies the product by 2, which is taken into account with the constants. For 64 bit CRC, the poly is 65 bits, so some adjustments are also needed to handle that.
I am trying to implement HMACSHA1 by Hand, to check if our serviceprovider or we calculate a wrong HMAC for a Support-Ticket (a SOAP call, hence the xml content).
The problem is, I stick to NIST Specification for HMACs and get a different result by Hand and when using openssl HMAC functions.
The following code prints:
B92674DCBA96F2DA93F7043071B931F5F2583FBD
4303E965D88D288C9AC594CE6C5E6AFF27D40B2D
while the result by openssl is the same we get in our application - so I assume, on the basis that openssl is so commonly used, that my result is wrong - but where is my error?
This is the spec I refer to:
http://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.198-1.pdf
here is my code:
#include <stdio.h>
#include <stdlib.h>
#include <openssl/sha.h>
#include <openssl/hmac.h>
#include <string.h>
#define byte unsigned char
#define IPAD 0x36
#define OPAD 0x5c
#define DIGESTSIZE 20
int main () {
byte *derivedKey = (byte[DIGESTSIZE]) {0x42,0xA9,0x78,0x90,0xFC,0xE5,0x16,0x8E,0x58,0x12,0x2F,0xF1,0xBA,0x32,0x5F,0x09,0x88,0x94,0x02,0x91};
byte *content = "<ds:SignedInfo xmlns:ds=\"http://www.w3.org/2000/09/xmldsig#\" xmlns:soap=\"http://www.w3.org/2003/05/soap-envelope\"><ds:CanonicalizationMethod Algorithm=\"http://www.w3.org/2001/10/xml-exc-c14n#\"><ec:InclusiveNamespaces xmlns:ec=\"http://www.w3.org/2001/10/xml-exc-c14n#\" PrefixList=\"soap\"></ec:InclusiveNamespaces></ds:CanonicalizationMethod><ds:SignatureMethod Algorithm=\"http://www.w3.org/2000/09/xmldsig#hmac-sha1\"></ds:SignatureMethod><ds:Reference URI=\"#TS-B183A13FEB0189143115136776276601\"><ds:Transforms><ds:Transform Algorithm=\"http://www.w3.org/2001/10/xml-exc-c14n#\"><ec:InclusiveNamespaces xmlns:ec=\"http://www.w3.org/2001/10/xml-exc-c14n#\" PrefixList=\"wsse soap\"></ec:InclusiveNamespaces></ds:Transform></ds:Transforms><ds:DigestMethod Algorithm=\"http://www.w3.org/2000/09/xmldsig#sha1\"></ds:DigestMethod><ds:DigestValue>rXJdGuDqoRrUJxuGiA1eyAozifk=</ds:DigestValue></ds:Reference></ds:SignedInfo>";
byte *oPadKey = malloc(DIGESTSIZE);
byte *iPadKey = malloc(DIGESTSIZE);
int i;
for(i=0;i<DIGESTSIZE;i++){
iPadKey[i]=derivedKey[i]^IPAD;
oPadKey[i]=derivedKey[i]^OPAD;
}
byte *rOpInput= malloc(strlen(content)+DIGESTSIZE);
//concat iPad and content
memcpy(rOpInput, iPadKey, DIGESTSIZE);
memcpy(rOpInput+DIGESTSIZE,content,strlen(content));
//SHA1 (iPad||content) gives the rightmost 20 bytes of the final SHA1 Input
byte *rOp=malloc(DIGESTSIZE); // H(iPad||content)
SHA1(rOpInput,strlen(content)+DIGESTSIZE,rOp);
free(rOpInput);
byte *finalInput = malloc(2*DIGESTSIZE); //oPad||H(iPad||content)
//concat oPad and H(ipad||content)
memcpy(finalInput, oPadKey,DIGESTSIZE);
memcpy(finalInput+DIGESTSIZE,rOp,DIGESTSIZE);
free(rOp);
free(oPadKey);
free(iPadKey);
//SHA1(oPad||H(iPad||content))
byte *hmac = malloc(DIGESTSIZE);
SHA1(finalInput,40,hmac);
free(finalInput);
//print calculated HMAC as HEX
for(i=0;i<DIGESTSIZE;i++){
printf("%02X", (hmac[i] & 0xFF ));
}
printf("\n");
//verify with openssl HMAC
byte *result = HMAC(EVP_sha1(), derivedKey, DIGESTSIZE, content, strlen(content), NULL, NULL);
for(i=0;i<DIGESTSIZE;i++){
printf("%02X", (result[i] & 0xFF ));
}
printf("\n");
return 0;
}
The bug is simple. The ipad and opad need to be (input) block size in length, not the length of the digest output. I.e. they must be 512 bits (64 bytes) for SHA-1, not 20.
I.e.
#define INPUT_BLOCK_SIZE 64
byte derivedKey[64] = {0x42,0xA9,0x78,0x90,0xFC,0xE5,0x16,0x8E,0x58,0x12,
0x2F,0xF1,0xBA,0x32,0x5F,0x09,0x88,0x94,0x02,0x91};
// null-padded on the right
and then change the DIGESTSIZE to INPUT_BLOCK_SIZE for wherever the length of ipad, opad is needed.
Result:
4303E965D88D288C9AC594CE6C5E6AFF27D40B2D
From Wikipedia
K' is another secret key, derived from the original key K (by padding K to the right with extra zeroes to the input block size of the hash function, or by hashing K if it is longer than that block size)
(emphasis mine).
P.S. It would be better to use the SHA1_Init/Update/Final, as then lots of copying would be skipped. You can also avoid allocating memory for the ipad and opad separately by first calculating ipad and after it has been used, then xorring in place by 0x6a to get the opad.
Please refer this. You can easily calculate the hash
https://csrc.nist.gov/CSRC/media/Projects/Cryptographic-Standards-and-Guidelines/documents/examples/HMAC_SHA1.pdf
#define IPAD 0x36
#define OPAD 0x5c
#define DIGESTSIZE 20
#define INPUT_BLOCK_SIZE 64
#define byte unsigned char
static byte mykey[64];
int main () {
int i;
byte mykey[INPUT_BLOCK_SIZE] = {
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E, 0x3F };
byte content[] = {0x53, 0x61, 0x6D, 0x70, 0x6C, 0x65, 0x20, 0x6D, 0x65, 0x73,
0x73, 0x61, 0x67, 0x65, 0x20, 0x66, 0x6F, 0x72, 0x20, 0x6B, 0x65, 0x79, 0x6C, 0x65, 0x6E, 0x3D, 0x62, 0x6C, 0x6F, 0x63, 0x6B, 0x6C, 0x65, 0x6E };
byte k0_ipad_text[sizeof(content) + INPUT_BLOCK_SIZE];
byte hmac[DIGESTSIZE];
byte k0_opad_hash[INPUT_BLOCK_SIZE + DIGESTSIZE];
/*
* K0 is mykey as size is 0x40 = 64 bytes
*/
byte *oPadKey = malloc(INPUT_BLOCK_SIZE);
byte *iPadKey = malloc(INPUT_BLOCK_SIZE);
memset(oPadKey, 0, INPUT_BLOCK_SIZE);
memset(iPadKey, 0, INPUT_BLOCK_SIZE);
for(i=0;i<INPUT_BLOCK_SIZE;i++){
iPadKey[i]=mykey[i]^IPAD;
oPadKey[i]=mykey[i]^OPAD;
}
printf("\n iPad key = ");
for(i=0;i<INPUT_BLOCK_SIZE;i++){
printf("%02x ", iPadKey[i]);
}
printf("\n oPad key = ");
for(i=0;i<INPUT_BLOCK_SIZE;i++){
printf("%02x ", oPadKey[i]);
}
//Key^ipad)||text)
memcpy(k0_ipad_text, iPadKey, INPUT_BLOCK_SIZE);
memcpy(k0_ipad_text + INPUT_BLOCK_SIZE, content, sizeof(content));
SHA1(k0_ipad_text, INPUT_BLOCK_SIZE + sizeof(content), hmac);
#if 0
printf("\n hmac = ");
for(i=0;i<DIGESTSIZE;i++){
printf("%02x ", hmac[i]);
}
#endif
memcpy(k0_opad_hash, oPadKey, INPUT_BLOCK_SIZE);
memcpy(k0_opad_hash + INPUT_BLOCK_SIZE, hmac, DIGESTSIZE);
SHA1(k0_opad_hash, INPUT_BLOCK_SIZE + DIGESTSIZE, hmac);
printf("\n hmac \n");
for(i=0;i<DIGESTSIZE;i++){
printf("%02x", hmac[i]);
}
printf("\n");
//verify with openssl HMAC
byte *result = HMAC(EVP_sha1(), mykey, INPUT_BLOCK_SIZE, content, sizeof(content), NULL, NULL);
for(i=0;i<DIGESTSIZE;i++){
printf("%02X", (result[i] & 0xFF ));
}
printf("\n");
return 0;
}
Suppose i have to public pem key like this
-----BEGIN PUBLIC KEY-----
MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC7vbqajDw4o6gJy8UtmIbkcpnk
O3Kwc4qsEnSZp/TR+fQi62F79RHWmwKOtFmwteURgLbj7D/WGuNLGOfa/2vse3G2
eHnHl5CB8ruRX9fBl/KgwCVr2JaEuUm66bBQeP5XeBotdR4cvX38uPYivCDdPjJ1
QWPdspTBKcxeFbccDwIDAQAB
-----END PUBLIC KEY-----
And i want to use openssl
int RSA_public_decrypt(int flen, const unsigned char *from, unsigned char *to,RSA *rsa, int padding). How can i translate the pem key to RSA *rsa structure?
FYI: I can't use BIO because i just want to transplant openssl into a bootloader which doesn't have a UNIX filesystem. The only way i can do is translate public key into a C array.
I can't use BIO because i just want to transplant openssl into a bootloader which doesn't have a UNIX filesystem
I think you can still use a BIO, it just has to be a memory BIO, and not a file BIO.
If you can't use a memory BIO either, then I'm not sure you can use OpenSSL here.
How to translate pem public key into openssl RSA* structure
Use PEM_read_PUBKEY or PEM_read_bio_PUBKEY (or one of the *_PUBKEY routines). They return an EVP_PKEY. Then, use EVP_PKEY_get1_RSA to convert it to a RSA.
Something like:
static const char key[] = "-----BEGIN PUBLIC KEY-----\n"
"MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC7vbqajDw4o6gJy8UtmIbkcpnk\n"
"O3Kwc4qsEnSZp/TR+fQi62F79RHWmwKOtFmwteURgLbj7D/WGuNLGOfa/2vse3G2\n"
"eHnHl5CB8ruRX9fBl/KgwCVr2JaEuUm66bBQeP5XeBotdR4cvX38uPYivCDdPjJ1\n"
"QWPdspTBKcxeFbccDwIDAQAB\n"
"-----END PUBLIC KEY-----\n";
BIO* bio = BIO_new_mem_buf(key, (int)sizeof(key));
ASSERT(bio != NULL);
EVP_PKEY* pkey = PEM_read_bio_PUBKEY(bio, NULL, NULL, NULL);
ASSERT(pkey != NULL);
int type = EVP_PKEY_get_type(pkey);
ASSERT(type == EVP_PKEY_RSA || type == EVP_PKEY_RSA2);
RSA* rsa = EVP_PKEY_get1_RSA(pkey);
ASSERT(rsa != NULL);
...
EVP_PKEY_free(pkey);
RSA_free(rsa);
BIO_free(bio);
And here's the helper you need. EVP_PKEY_get_type is not part of the library:
int EVP_PKEY_get_type(EVP_PKEY *pkey)
{
ASSERT(pkey);
if (!pkey)
return EVP_PKEY_NONE;
return EVP_PKEY_type(pkey->type);
}
Related, you can also use the non-PEM version, which is the ASN.1/DER encoding of the key. This will allow you to save some size in the image. Then, you would use d2i_PUBKEY_bio to read the key.
Converting from PEM to ASN.1/DER will save about 110 bytes:
$ ls -al pubkey.*
-rw-r--r-- 1 user staff 162 Jun 5 00:36 pubkey.der
-rw-r--r-- 1 user staff 272 Jun 5 00:52 pubkey.pem
To convert the PEM encoded public key to ASN.1/DER, follow these steps. First, copy the key to the clipboard. Second, pipe it into openssl pkey with a tool like pbpaste on OS X or xclip on Linux:
$ pbpaste | openssl pkey -pubin -inform PEM -out pubkey.der -outform DER
Then, inspect the public key:
$ dumpasn1 pubkey.der
0 159: SEQUENCE {
3 13: SEQUENCE {
5 9: OBJECT IDENTIFIER rsaEncryption (1 2 840 113549 1 1 1)
16 0: NULL
: }
18 141: BIT STRING, encapsulates {
22 137: SEQUENCE {
25 129: INTEGER
: 00 BB BD BA 9A 8C 3C 38 A3 A8 09 CB C5 2D 98 86
: E4 72 99 E4 3B 72 B0 73 8A AC 12 74 99 A7 F4 D1
: F9 F4 22 EB 61 7B F5 11 D6 9B 02 8E B4 59 B0 B5
: E5 11 80 B6 E3 EC 3F D6 1A E3 4B 18 E7 DA FF 6B
: EC 7B 71 B6 78 79 C7 97 90 81 F2 BB 91 5F D7 C1
: 97 F2 A0 C0 25 6B D8 96 84 B9 49 BA E9 B0 50 78
: FE 57 78 1A 2D 75 1E 1C BD 7D FC B8 F6 22 BC 20
: DD 3E 32 75 41 63 DD B2 94 C1 29 CC 5E 15 B7 1C
: 0F
157 3: INTEGER 65537
: }
: }
: }
Finally, do something like:
int main(int argc, char* argv[])
{
UNUSED(argc), UNUSED(argv);
static const unsigned char key[] = {
0x30, 0x81, 0x9f, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01,
0x05, 0x00, 0x03, 0x81, 0x8d, 0x00, 0x30, 0x81, 0x89, 0x02, 0x81, 0x81, 0x00, 0xbb, 0xbd, 0xba,
0x9a, 0x8c, 0x3c, 0x38, 0xa3, 0xa8, 0x09, 0xcb, 0xc5, 0x2d, 0x98, 0x86, 0xe4, 0x72, 0x99, 0xe4,
0x3b, 0x72, 0xb0, 0x73, 0x8a, 0xac, 0x12, 0x74, 0x99, 0xa7, 0xf4, 0xd1, 0xf9, 0xf4, 0x22, 0xeb,
0x61, 0x7b, 0xf5, 0x11, 0xd6, 0x9b, 0x02, 0x8e, 0xb4, 0x59, 0xb0, 0xb5, 0xe5, 0x11, 0x80, 0xb6,
0xe3, 0xec, 0x3f, 0xd6, 0x1a, 0xe3, 0x4b, 0x18, 0xe7, 0xda, 0xff, 0x6b, 0xec, 0x7b, 0x71, 0xb6,
0x78, 0x79, 0xc7, 0x97, 0x90, 0x81, 0xf2, 0xbb, 0x91, 0x5f, 0xd7, 0xc1, 0x97, 0xf2, 0xa0, 0xc0,
0x25, 0x6b, 0xd8, 0x96, 0x84, 0xb9, 0x49, 0xba, 0xe9, 0xb0, 0x50, 0x78, 0xfe, 0x57, 0x78, 0x1a,
0x2d, 0x75, 0x1e, 0x1c, 0xbd, 0x7d, 0xfc, 0xb8, 0xf6, 0x22, 0xbc, 0x20, 0xdd, 0x3e, 0x32, 0x75,
0x41, 0x63, 0xdd, 0xb2, 0x94, 0xc1, 0x29, 0xcc, 0x5e, 0x15, 0xb7, 0x1c, 0x0f, 0x02, 0x03, 0x01,
0x00, 0x01
};
BIO* bio = BIO_new_mem_buf(key, (int)sizeof(key));
ASSERT(bio != NULL);
EVP_PKEY* pkey = d2i_PUBKEY_bio(bio, NULL);
ASSERT(pkey != NULL);
int type = EVP_PKEY_get_type(pkey);
ASSERT(type == EVP_PKEY_RSA || type == EVP_PKEY_RSA2);
RSA* rsa = EVP_PKEY_get1_RSA(pkey);
ASSERT(rsa != NULL);
// ...
EVP_PKEY_free(pkey);
RSA_free(rsa);
BIO_free(bio);
return 0;
}
To load the RSA pub key from the pem file
int loadKey(string &fileName, RSA ** pubkey)
{
FILE *fp = fopen(fileName.data(), "r");
if (fp < 0){
printf("%s\n","Invalide certificate file name");
return -1;
}
*pubkey = PEM_read_RSA_PUBKEY(fp,pubkey, NULL, NULL);
if(*pubkey == NULL){
printf("%s\n","error reading Public key");
return -1;
}
To verify the signature
int verify_signature(vector<uint8_t> &msg, vector<uint8_t> &sig, RSA* pkey)
{
/* Returned to caller */
int result = -1;
vector <uint8_t> pDecrypted(PACKAGE_SIGNATURE_SIZE);
result = RSA_public_decrypt(PACKAGE_SIGNATURE_SIZE, sig.data(), pDecrypted.data(), pkey, RSA_NO_PADDING);
if (result == -1)
{
printf("RSA_public_decrypt failed with error %s\n", ERR_error_string(ERR_get_error(), NULL));
// goto prog_end;
}
/* verify the data */
result = RSA_verify_PKCS1_PSS(pkey,msg.data() , EVP_sha256(), pDecrypted.data(), -2 /* salt length recovered from signature*/);
if (result == 1)
{
printf("Signature verification successfull!\n");
}
else
{
printf("RSA_verify_PKCS1_PSS failed with error %s\n", ERR_error_string(ERR_get_error(), NULL));
}
return !!result;
}