Raw socket TCP handshake not working - c
I am trying to simulate a TCP handshake using raw sockets. For testing, i am using server and client on the same machine. My code that uses raw sockets is a passive server that listens for SYN and respondswith SYN/ACK. However, i am not able to get the ACK back from the client. I used tcpdump to read the traces which clearly shows my packet leaving the server. This is the trace:
13:49:35.389518 IP (tos 0x0, ttl 64, id 51534, offset 0, flags [DF], proto TCP (6), length 60)
localhost.3333 > localhost.9999: Flags [S], cksum 0xfe30 (incorrect -> 0xd31d), seq 914929523, win 43690, options [mss 65495,sackOK,TS val 588877032 ecr 0,nop,wscale 7], length 0
0x0000: 0000 0000 0000 0000 0000 0000 0800 4500 ..............E.
0x0010: 003c c94e 4000 4006 736b 7f00 0001 7f00 .<.N#.#.sk......
0x0020: 0001 0d05 270f 3688 b773 0000 0000 a002 ....'.6..s......
0x0030: aaaa fe30 0000 0204 ffd7 0402 080a 2319 ...0..........#.
0x0040: 8ce8 0000 0000 0103 0307 ..........
13:49:35.389642 IP (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60)
localhost.9999 > localhost.3333: Flags [S.], cksum 0xf9cb (incorrect -> 0x9b6d), seq 584475847, ack 914929524, win 43690, options [mss 65495,sackOK,TS val 588877032 ecr 588877032,nop,wscale 7], length 0
0x0000: 0000 0000 0000 0000 0000 0000 0800 4500 ..............E.
0x0010: 003c 0000 4000 4006 3cba 7f00 0001 7f00 .<..#.#.<.......
0x0020: 0001 270f 0d05 22d6 64c7 3688 b774 a012 ..'...".d.6..t..
0x0030: aaaa f9cb 0000 0204 ffd7 0402 080a 2319 ..............#.
0x0040: 8ce8 2319 8ce8 0103 0307 ..#.......
13:49:36.646081 IP (tos 0x0, ttl 64, id 29569, offset 0, flags [DF], proto TCP (6), length 60)
localhost.3333 > localhost.9999: Flags [S], cksum 0xfe30 (incorrect -> 0x7c87), seq 1169489315, win 43690, options [mss 65495,sackOK,TS val 588877346 ecr 0,nop,wscale 7], length 0
0x0000: 0000 0000 0000 0000 0000 0000 0800 4500 ..............E.
0x0010: 003c 7381 4000 4006 c938 7f00 0001 7f00 .<s.#.#..8......
0x0020: 0001 0d05 270f 45b4 fda3 0000 0000 a002 ....'.E.........
0x0030: aaaa fe30 0000 0204 ffd7 0402 080a 2319 ...0..........#.
0x0040: 8e22 0000 0000 0103 0307 ."........
13:49:37.645510 IP (tos 0x0, ttl 64, id 29570, offset 0, flags [DF], proto TCP (6), length 60)
localhost.3333 > localhost.9999: Flags [S], cksum 0xfe30 (incorrect -> 0x7b8d), seq 1169489315, win 43690, options [mss 65495,sackOK,TS val 588877596 ecr 0,nop,wscale 7], length 0
0x0000: 0000 0000 0000 0000 0000 0000 0800 4500 ..............E.
0x0010: 003c 7382 4000 4006 c937 7f00 0001 7f00 .<s.#.#..7......
0x0020: 0001 0d05 270f 45b4 fda3 0000 0000 a002 ....'.E.........
0x0030: aaaa fe30 0000 0204 ffd7 0402 080a 2319 ...0..........#.
0x0040: 8f1c 0000 0000 0103 0307 ..........
I understood that one cannot simply read the SYN packet and so i used libpcap libraries to read the SYN packets. I used iptables to filter the RST from being sent for a SYN since i have no one listening on the server end(its a raw socket). When i tried to simulate the same with basic socket functions for TCP and captured it's trace it looks exactly identical to mine except that i don't get my ACK from client. I am not sure what the problem is. Been stuck with this for ages. i am posting the code for raw socket creation:
/**
p_ip - IP header of SYN packet |
p_tcp - TCP header of SYN packet |
s_seq - seq no of SYN packet
*/
void send_syn_ack(int s_seq, struct ipheader *p_ip, struct tcpheader *p_tcp, __u8 *p_tcpopt)
{
printf("seq is: %lu\n", ntohl(s_seq));
char buffer[PKT_LEN];
struct ipheader *ip = (struct ipheader *) buffer;
struct tcpheader *tcp = (struct tcpheader *) (buffer + sizeof(struct ipheader));
struct tcp_options *tcpoptions = (struct tcp_options *) (buffer + sizeof(struct tcpheader) + sizeof(struct ipheader));
int one = 1;
const int *val = &one;
struct sockaddr_in sin;
memset(&buffer, 0, sizeof(buffer));
ip -> iph_ihl = 5;
ip -> iph_ver = 4;
ip -> iph_tos = 0;
ip -> iph_len = htons(sizeof(struct ipheader) + sizeof(struct tcpheader) + 20);
ip -> iph_ident = 0;
ip -> iph_offset = 0x40;
ip -> iph_ttl = 64;
ip -> iph_protocol = 6;
ip -> iph_chksum = 0;
ip -> iph_sourceip = p_ip -> iph_sourceip;
ip -> iph_destip = p_ip -> iph_destip;
//ip -> ip_sum = in_cksum((unsigned short *)&ip, sizeof(ip));
tcp -> tcph_srcport = p_tcp -> tcph_destport;
tcp -> tcph_destport = p_tcp -> tcph_srcport;
tcp -> tcph_seqnum = htonl(584475847);
tcp -> tcph_acknum = htonl(ntohl(p_tcp -> tcph_seqnum) + 1);
tcp -> tcph_hlen = 10;
tcp -> tcph_syn = 1;
tcp -> tcph_ack = 1;
tcp -> tcph_win = htons(43690);
//tcp -> tcph_chksum = (in_cksum_tcp( ntohl(ip -> iph_sourceip), ntohl(ip -> iph_destip), (buffer), 40));
tcp -> tcph_chksum = 0;
tcp -> tcph_urgptr = 0;
//tcp.th_sum = in_cksum_tcp(ip.ip_src.s_addr, ip.ip_dst.s_addr, (unsigned short *)&tcp, sizeof(tcp))
//ip -> iph_chksum = (in_cksum(buffer, sizeof(struct ipheader)));
ip -> iph_chksum = htons(0xfe30);
memset(&sin, 0, sizeof(sin));
sin.sin_family = AF_INET;
sin.sin_port = (tcp -> tcph_srcport);
sin.sin_addr.s_addr = (ip -> iph_sourceip);
printf("Packet Information:\n");
printf("Source: %s.%u\nDestination: %s.%u\nPacket Length: %d \n", printIP(ip -> iph_sourceip), ntohs(tcp -> tcph_srcport), printIP(ip -> iph_destip), ntohs(tcp -> tcph_destport), ntohs(ip -> iph_len));
printf("Seq No: %lu \nAck No: %lu \n", ntohl(tcp -> tcph_seqnum), ntohl(tcp -> tcph_acknum));
struct tcp_options *tmpopts = readOptions(p_tcpopt);
memcpy(tcpoptions, tmpopts, 20);
//tcp -> tcph_chksum = in_cksum_tcp(ip -> iph_sourceip, ip -> iph_destip, buffer + 20, 40);
tcp -> tcph_chksum = htons(0xfe30);
if ((sd = socket(PF_INET, SOCK_RAW, IPPROTO_TCP)) < 0) {
perror("raw socket");
exit(1);
}
if(setsockopt(sd, IPPROTO_IP, IP_HDRINCL, val, sizeof(one)) < 0)
{
perror("ERROR: setsockopt() can't be performed!");
exit(1);
}
else
printf(">> setsockopt() is OK\n");
int sent_length;
if ((sent_length = sendto(sd, buffer, 60, 0, (struct sockaddr *)&sin, sizeof(sin))) < 0) {
perror("sendto");
exit(1);
}
else
{
printf("Sent %d bytes\n", sent_length);
}
}
I have been stuck with this problem for ages. Can some figure out the problem i need to look at?
Update:
iptables rules:
Chain INPUT (policy DROP 27 packets, 864 bytes)
pkts bytes target prot opt in out source destination
813 48600 DROP tcp -- any any anywhere anywhere tcp dpt:9999
173K 17M fail2ban-ssh tcp -- any any anywhere anywhere multiport dports ssh
3536K 720M ufw-before-logging-input all -- any any anywhere anywhere
3536K 720M ufw-before-input all -- any any anywhere anywhere
2745K 468M ufw-after-input all -- any any anywhere anywhere
55829 2150K ufw-after-logging-input all -- any any anywhere anywhere
55829 2150K ufw-reject-input all -- any any anywhere anywhere
55829 2150K ufw-track-input all -- any any anywhere anywhere
Chain FORWARD (policy DROP 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
0 0 ufw-before-logging-forward all -- any any anywhere anywhere
0 0 ufw-before-forward all -- any any anywhere anywhere
0 0 ufw-after-forward all -- any any anywhere anywhere
0 0 ufw-after-logging-forward all -- any any anywhere anywhere
0 0 ufw-reject-forward all -- any any anywhere anywhere
Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
0 0 DROP tcp -- any any anywhere anywhere tcp spt:9999flags: RST/RST
833K 1132M ufw-before-logging-output all -- any any anywhere anywhere
833K 1132M ufw-before-output all -- any any anywhere anywhere
61973 3938K ufw-after-output all -- any any anywhere anywhere
61973 3938K ufw-after-logging-output all -- any any anywhere anywhere
61973 3938K ufw-reject-output all -- any any anywhere anywhere
61973 3938K ufw-track-output all -- any any anywhere anywhere
Normal Sockets connection established trace:
13:50:13.739038 IP (tos 0x0, ttl 64, id 53527, offset 0, flags [DF], proto TCP (6), length 60)
localhost.3333 > localhost.4040: Flags [S], cksum 0xfe30 (incorrect -> 0x23fb), seq 732324684, win 43690, options [mss 65495,sackOK,TS val 588886619 ecr 0,nop,wscale 7], length 0
0x0000: 0000 0000 0000 0000 0000 0000 0800 4500 ..............E.
0x0010: 003c d117 4000 4006 6ba2 7f00 0001 7f00 .<..#.#.k.......
0x0020: 0001 0d05 0fc8 2ba6 634c 0000 0000 a002 ......+.cL......
0x0030: aaaa fe30 0000 0204 ffd7 0402 080a 2319 ...0..........#.
0x0040: b25b 0000 0000 0103 0307 .[........
13:50:13.739050 IP (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60)
localhost.4040 > localhost.3333: Flags [S.], cksum 0xfe30 (incorrect -> 0x1502), seq 2134096447, ack 732324685, win 43690, options [mss 65495,sackOK,TS val 588886619 ecr 588886619,nop,wscale 7], length 0
0x0000: 0000 0000 0000 0000 0000 0000 0800 4500 ..............E.
0x0010: 003c 0000 4000 4006 3cba 7f00 0001 7f00 .<..#.#.<.......
0x0020: 0001 0fc8 0d05 7f33 ba3f 2ba6 634d a012 .......3.?+.cM..
0x0030: aaaa fe30 0000 0204 ffd7 0402 080a 2319 ...0..........#.
0x0040: b25b 2319 b25b 0103 0307 .[#..[....
13:50:13.739064 IP (tos 0x0, ttl 64, id 53528, offset 0, flags [DF], proto TCP (6), length 52)
localhost.3333 > localhost.4040: Flags [.], cksum 0xfe28 (incorrect -> 0xe746), seq 1, ack 1, win 342, options [nop,nop,TS val 588886619 ecr 588886619], length 0
0x0000: 0000 0000 0000 0000 0000 0000 0800 4500 ..............E.
0x0010: 0034 d118 4000 4006 6ba9 7f00 0001 7f00 .4..#.#.k.......
0x0020: 0001 0d05 0fc8 2ba6 634d 7f33 ba40 8010 ......+.cM.3.#..
0x0030: 0156 fe28 0000 0101 080a 2319 b25b 2319 .V.(......#..[#.
0x0040: b25b .[
13:50:13.739084 IP (tos 0x0, ttl 64, id 53529, offset 0, flags [DF], proto TCP (6), length 58)
localhost.3333 > localhost.4040: Flags [P.], cksum 0xfe2e (incorrect -> 0xe76c), seq 1:7, ack 1, win 342, options [nop,nop,TS val 588886619 ecr 588886619], length 6
0x0000: 0000 0000 0000 0000 0000 0000 0800 4500 ..............E.
0x0010: 003a d119 4000 4006 6ba2 7f00 0001 7f00 .:..#.#.k.......
0x0020: 0001 0d05 0fc8 2ba6 634d 7f33 ba40 8018 ......+.cM.3.#..
0x0030: 0156 fe2e 0000 0101 080a 2319 b25b 2319 .V........#..[#.
0x0040: b25b 4090 b5bc 097f .[#.....
13:50:13.739089 IP (tos 0x0, ttl 64, id 2075, offset 0, flags [DF], proto TCP (6), length 52)
localhost.4040 > localhost.3333: Flags [.], cksum 0xfe28 (incorrect -> 0xe740), seq 1, ack 7, win 342, options [nop,nop,TS val 588886619 ecr 588886619], length 0
0x0000: 0000 0000 0000 0000 0000 0000 0800 4500 ..............E.
0x0010: 0034 081b 4000 4006 34a7 7f00 0001 7f00 .4..#.#.4.......
0x0020: 0001 0fc8 0d05 7f33 ba40 2ba6 6353 8010 .......3.#+.cS..
0x0030: 0156 fe28 0000 0101 080a 2319 b25b 2319 .V.(......#..[#.
0x0040: b25b .[
Did you try to filter the packets using iptables that you are interested in? I mean, I am not sure about this problem you are having so, maybe as a hint you could use iptables to filter the packets that has the SYN flag on it and see with the counters that the packets are dropped or not.
Since you are using the Server and Client in the same computer, the packets won't go out to your NIC, but however, the packet gonna be send to the virtual loopback interface (you can find on Google "loopback iptables").
This is not how to solve the problem but, maybe it could be useful to get some hints about how to fix it. I've never used raw socket, so all I can help is with this stuff. Another point is that the port number you are trying to connect must be set to accept new connections, that means, must be "binded".
Wish you good luck.
I'm trying to guess based on the data you posted:
First, the checksum is incorrect. If you are relying on NIC calculating the checksum, it won't do it for messages going to localhost.
Second, can you show all iptables rules you are using? If you use libpcap, you should use a rule that drops TCP messages to port 9999.
Third, if the client is using AF_INET stream sockets ("normal" sockets), add the following rule:
iptables -A INPUT -p tcp --dport 9999 -j DROP
And remove the other DROP rules.
Related
after using allure #step annotation in spring boot getting error java.lang.VerifyError: Expecting a stackmap frame at branch target 15
``java.lang.VerifyError: Expecting a stackmap frame at branch target 146 Exception Details: Location: com/common/utilities/Payload(Ljava/lang/String;Ljava/lang/String;Ljava/lang/Class;)Ljava/lang/Object; #45: invokestatic Reason: Expected stackmap frame at this location. Bytecode: 0000000: 2a3a 082b 3a09 2c3a 0ab2 0155 0101 06bd 0000010: 0051 3a07 1907 0319 0853 1907 0419 0953 0000020: 1907 0519 0a53 1907 b801 5b3a 06b8 0161 0000030: 1906 b601 65b2 0002 1203 b600 04b2 0005 0000040: 2aba 0006 0000 4eb2 0007 2bb6 0008 bb00 0000050: 0959 2d03 bd00 09b8 000a b800 0bb7 000c 0000060: 3a04 1904 b800 0d3a 0519 052b b900 0e02 0000070: 00b8 000f 2cb8 0010 3a0c a700 253a 04b2 0000080: 0002 1904 b600 1219 04b6 0013 013a 0ca7 0000090: 0010 3a0b b801 6119 0bb6 0169 190b bfb8 00000a0: 0161 b601 6c19 0cb0 Exception Handler Table: bci [78, 120] => handler: 125 bci [45, 146] => handler: 146`
Tsql & operator
I just came across this command that I have never is seen being used before. What is the & operator doing in this line of code? It seams like ( #MyVar & 64 ) another way of writing #MyVar = 64 DECLARE #MyVar INT SET #MyVar = 16 -- Prints yes SET #MyVar = 64 -- Prints no IF ( ( #MyVar & 64 ) = 0 ) BEGIN SELECT 'yes' END ELSE BEGIN SELECT 'no' END
This is bitwise AND. In fact, as written, what you return in the SELECT alternates between 0 and 64 and no other numbers. This is verbatim from https://learn.microsoft.com/en-us/sql/t-sql/language-elements/bitwise-and-transact-sql : The & bitwise operator performs a bitwise logical AND between the two values, taking each corresponding bit for both expressions. The bits in the result are set to 1 if and only if both bits (for the current bit being resolved) in the input expressions have a value of 1; otherwise, the bit in the result is set to 0. Lets see what this does: DECLARE #myint int = 16 SELECT #myint & 64 [myint 0] --0 /* --This is the bitwise AND representation for 16 &64: 0000 0000 0100 0000 --&64 0000 0000 0001 0000 --#MyVar = 16 ------------------- 0000 0000 0000 0000 -- = 0 = 'Yes' */ SET #myint = 64 SELECT #myint & 64 [myint 64] --64 /* --This is the bitwise AND representation for 64 &64: 0000 0000 0100 0000 --&64 0000 0000 0100 0000 --#MyVar = 64 ------------------- 0000 0000 0100 0000 -- = 64 = 'No' */ This applies for other numbers as well, try 127 and 128: /* 0000 0000 0100 0000 --&64 0000 0000 0111 1111 --#MyVar = 127 ------------------- 0000 0000 0100 0000 --64 = 'No' 0000 0000 0100 0000 --&64 0000 0000 1000 0001 --#MyVar = 128 ------------------- 0000 0000 0000 0000 --0 = 'Yes' */ 127 &64 = 64. 128 &64 = 0.
How to find magic bitboards?
const int BitTable[64] = {
63, 30, 3, 32, 25, 41, 22, 33, 15, 50, 42, 13, 11, 53, 19, 34, 61, 29, 2,
51, 21, 43, 45, 10, 18, 47, 1, 54, 9, 57, 0, 35, 62, 31, 40, 4, 49, 5, 52,
26, 60, 6, 23, 44, 46, 27, 56, 16, 7, 39, 48, 24, 59, 14, 12, 55, 38, 28,
58, 20, 37, 17, 36, 8
};
int pop_1st_bit(uint64 *bb) {
uint64 b = *bb ^ (*bb - 1);
unsigned int fold = (unsigned) ((b & 0xffffffff) ^ (b >> 32));
*bb &= (*bb - 1);
return BitTable[(fold * 0x783a9b23) >> 26];
}
uint64 index_to_uint64(int index, int bits, uint64 m) {
int i, j;
uint64 result = 0ULL;
for(i = 0; i < bits; i++) {
j = pop_1st_bit(&m);
if(index & (1 << i)) result |= (1ULL << j);
}
return result;
}
It's from the Chess Programming Wiki:
https://www.chessprogramming.org/Looking_for_Magics
It's part of some code for finding magic numbers.
The argument uint64 m is a bitboard representing the possible blocked squares for either a rook or bishop move. Example for a rook on the e4 square:
0 0 0 0 0 0 0 0
0 0 0 0 1 0 0 0
0 0 0 0 1 0 0 0
0 0 0 0 1 0 0 0
0 1 1 1 0 1 1 0
0 0 0 0 1 0 0 0
0 0 0 0 1 0 0 0
0 0 0 0 0 0 0 0
The edge squares are zero because they always block, and reducing the number of bits needed is apparently helpful.
/* Bitboard, LSB to MSB, a1 through h8:
* 56 - - - - - - 63
* - - - - - - - -
* - - - - - - - -
* - - - - - - - -
* - - - - - - - -
* - - - - - - - -
* - - - - - - - -
* 0 - - - - - - 7
*/
So in the example above, index_to_uint64 takes an index (0 to 2^bits), and the number of bits set in the bitboard (10), and the bitboard.
It then pops_1st_bit for each number of bits, followed by another shifty bit of code. pops_1st_bit XORs the bitboard with itself minus one (why?). Then it ANDs it with a full 32-bits, and somewhere around here my brain runs out of RAM. Somehow the magical hex number 0x783a9b23 is involved (is that the number sequence from Lost?). And there is this ridiculous mystery array of randomly ordered numbers from 0-63 (BitTable[64]).
Alright, I have it figured out.
First, some terminology:
blocker mask: A bitboard containing all squares that can block a piece, for a given piece type and the square the piece is on. It excludes terminating edge squares because they always block.
blocker board: A bitboard containing occupied squares. It only has squares which are also in the blocker mask.
move board: A bitboard containing all squares a piece can move to, given a piece type, a square, and a blocker board. It includes terminating edge squares if the piece can move there.
Example for a rook on the e4 square, and there are some random pieces on e2, e5, e7, b4, and c4.
The blocker mask A blocker board The move board
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0
0 1 1 1 0 1 1 0 0 1 1 0 0 0 0 0 0 0 1 1 0 1 1 1
0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0
0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
Some things to note:
The blocker mask is always the same for a given square and piece type (either rook or bishop).
Blocker boards include friendly & enemy pieces, and it is a subset of the blocker mask.
The resulting move board may include moves that capture your own pieces, however these moves are easily removed afterward: moveboard &= ~friendly_pieces)
The goal of the magic numbers method is to very quickly look up a pre-calculated move board for a given blocker board. Otherwise you'd have to (slowly) calculate the move board every time. This only applies to sliding pieces, namely the rook and bishop. The queen is just a combination of the rook and bishop.
Magic numbers can be found for each square & piece type combo. To do this, you have to calculate every possible blocker board variation for each square/piece combo. This is what the code in question is doing. How it's doing it is still a bit of a mystery to me, but that also seems to be the case for the apparent original author, Matt Taylor. (Thanks to #Pradhan for the link)
So what I've done is re-implemented the code for generating all possible blocker board variations. It uses a different technique, and while it's a little slower, it's much easier to read and comprehend. The fact that it's slightly slower is not a problem, because this code isn't speed critical. The program only has to do it once at program startup, and it only takes microseconds on a dual-core i5.
/* Generate a unique blocker board, given an index (0..2^bits) and the blocker mask
* for the piece/square. Each index will give a unique blocker board. */
static uint64_t gen_blockerboard (int index, uint64_t blockermask)
{
/* Start with a blockerboard identical to the mask. */
uint64_t blockerboard = blockermask;
/* Loop through the blockermask to find the indices of all set bits. */
int8_t bitindex = 0;
for (int8_t i=0; i<64; i++) {
/* Check if the i'th bit is set in the mask (and thus a potential blocker). */
if ( blockermask & (1ULL<<i) ) {
/* Clear the i'th bit in the blockerboard if it's clear in the index at bitindex. */
if ( !(index & (1<<bitindex)) ) {
blockerboard &= ~(1ULL<<i); //Clear the bit.
}
/* Increment the bit index in the 0-4096 index, so each bit in index will correspond
* to each set bit in blockermask. */
bitindex++;
}
}
return blockerboard;
}
To use it, do something like this:
int bits = count_bits( RookBlockermask[square] );
/* Generate all (2^bits) blocker boards. */
for (int i=0; i < (1<<bits); i++) {
RookBlockerboard[square][i] = gen_blockerboard( i, RookBlockermask[square] );
}
How it works: There are 2^bits blocker boards, where bits is the number of 1's in the blocker mask, which are the only relevant bits. Also, each integer from 0 to 2^bits has a unique sequence of 1's and 0's of length bits. So this function just corresponds each bit in the given integer to a relevant bit in the blocker mask, and turns it off/on accordingly to generate a unique blocker board.
It's not as clever or fast, but it's readable.
Alright, I'm going to try to step through this. index_to_uint64( 7, 10, m ); 7 is just a randomly chosen number between 0 and 2^10, and 10 is the number of bits set in m. m can be represented in four ways: bitboard: 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 1 1 1 0 1 1 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 dec: 4521262379438080 hex: 0x1010106e101000 bin: 0000 0000 0001 0000 0001 0000 0001 0000 0110 1110 0001 0000 0001 0000 0000 0000 Moving on. This will be called 10 times. It has a return value and it modifies m. pop_1st_bit(&m); In pop_1st_bit, m is referred to by bb. I'll change it to m for clarity. uint64 b = m^(m-1); The m-1 part takes the least significant bit that is set and flips it and all the bits below it. After the XOR, all those changed bits are now set to 1 while all the higher bits are set to 0. m : 0000 0000 0001 0000 0001 0000 0001 0000 0110 1110 0001 0000 0001 0000 0000 0000 m-1: 0000 0000 0001 0000 0001 0000 0001 0000 0110 1110 0001 0000 0000 1111 1111 1111 b : 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0001 1111 1111 1111 Next: unsigned int fold = (unsigned) ((b & 0xffffffff) ^ (b >> 32)); The (b & 0xffffffff) part ANDs b with lower 32 set bits. So this essentially clears any bits in the upper half of b. (b & 0xffffffff) b: 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0001 1111 1111 1111 &: 0000 0000 0000 0000 0000 0000 0000 0000 1111 1111 1111 1111 1111 1111 1111 1111 =: 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0001 1111 1111 1111 The ... ^ (b >> 32) part shifts the upper half of b into the lower half, then XORs it with the result of the previous operation. So it basically XORs the top half of b with the lower half of b. This has no effect in this case because the upper half of b was empty to begin with. >> :0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 ^ :0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0001 1111 1111 1111 uint fold = 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0001 1111 1111 1111 I don't understand the point of that "folding", even if there had been bits set in the upper half of b. Anyways, moving on. This next line actually modifies m by unsetting the lowest bit. That makes some sense. m &= (m - 1); m : 0000 0000 0001 0000 0001 0000 0001 0000 0110 1110 0001 0000 0001 0000 0000 0000 m-1: 0000 0000 0001 0000 0001 0000 0001 0000 0110 1110 0001 0000 0000 1111 1111 1111 & : 0000 0000 0001 0000 0001 0000 0001 0000 0110 1110 0001 0000 0000 0000 0000 0000 This next part multiplies fold by some hex number (a prime?), right shifts the product 26, and uses that as an index into BitTable, our mysterious array of randomly ordered numbers 0-63. At this point I suspect the author might be writing a pseudo random number generator. return BitTable[(fold * 0x783a9b23) >> 26]; That concludes pop_1st_bit. That's all done 10 times (once for each bit originally set in m). Each of the 10 calls to pop_1st_bit returns a number 0-63. j = pop_1st_bit(&m); if(index & (1 << i)) result |= (1ULL << j); In the above two lines, i is the current bit we are on, 0-9. So if the index number (the 7 originally passed as an argument to index_to_uint64) has the i'th bit set, then set the j'th bit in the result, where j was the 0-63 return value from pop_1st_bit. And that's it! I'm still confused :(
When watching a video series on chess engines on youtube I had exactly the same questions as paulwal222. There seems to be some high level mathematics involved. The best links explaining the background of this difficult subject are https://chessprogramming.wikispaces.com/Matt+Taylor and https://chessprogramming.wikispaces.com/BitScan . It seems that Matt Taylor in 2003 in a google.group ( https://groups.google.com/forum/#!topic/comp.lang.asm.x86/3pVGzQGb1ys ) (also found by pradhan) came up with something that is now called Matt Taylor's folding trick, a 32-bit friendly implementation to find the bit-index of LS1B ( https://en.wikipedia.org/wiki/Find_first_set ). Taylor's folding trick apparently is an adaptation of the De Bruijn ( https://en.wikipedia.org/wiki/Nicolaas_Govert_de_Bruijn ) bitscan, devised in 1997, according to Donald Knuth by Martin Läuter to determine the LS1B index by minimal perfect hashing ( https://en.wikipedia.org/wiki/Perfect_hash_function ). The numbers of the BitTable (63, 30, ..) and the fold in PopBit (0x783a9b23) are probably the so called magic numbers (uniquely?) related to Matt Taylor's 32-bit folding trick. This folding trick seems to be very fast, because lots of engines have copied this approach (f.i Stockfish).
The goal is to find and return the position of the least significant bit (LSB) in a given integer and then set the LSB to zero, therefore if m = 1010 0100, we want the function to return the index 2 and to modify m as: m = 1010 0000. In order to see what is going on in this process, it is easiest to look at the 8-bit case and what happens in the first 2 lines of pop_1st_bit:
m m^(m-1) "fold"
XXXX XXX1 0000 0001 0001
XXXX XX10 0000 0011 0011
XXXX X100 0000 0111 0111
XXXX 1000 0000 1111 1111
XXX1 0000 0001 1111 1110
XX10 0000 0011 1111 1100
X100 0000 0111 1111 1000
1000 0000 1111 1111 0000
In the above, X means we don't care about the value of the bits in those positions. So, m^(m-1) maps every 8-bit number to one of eight 8-bit keys based on the position of its LSB. The fold operation then converts each of these 8-bit keys to a unique 4-bit key. The reason for doing this in the original (64-bit) case is to avoid a 64-bit multiplication in fold*magic by replacing it with a 32-bit multiplication.
The table for the 8-bit case contains eight values (one for each of the possible LSB positions in an 8-bit integer). Thus, we need 3 bits to index the table. The right shift is going to give us those 3 bits, the formula for calculating how many bits to shift by:
[Number of bits in key] - log2([size of table])
So, in the original case, we get: 32 - log2(64) = 26, whereas in the 8-bit case we get 4 - log2(8) = 1.
Therefore: table_index = fold*magic >> 1, will give us numbers between 0-7 which we need to index the table. All we need to do now is find magic that gives us a different value for each of our eight 4-bit keys.
In this case, the 4-bit magic number that we need is 5 (0b0101). You can find the number by brute force, by looking for magic such that fold*magic >> 1 yields a unique value for each of the eight "fold" keys. (It is worth noting here that I am assuming that any overflowing bits from the 4-bit multiplication of fold and magic disappear):
m m^(m-1) "fold" fold*5 fold*5 >> 1
XXXX XXX1 0000 0001 0001 0101 010 (dec: 2)
XXXX XX10 0000 0011 0011 1111 111 (dec: 7)
XXXX X100 0000 0111 0111 0011 001 (dec: 1)
XXXX 1000 0000 1111 1111 1011 101 (dec: 5)
XXX1 0000 0001 1111 1110 0110 011 (dec: 3)
XX10 0000 0011 1111 1100 1100 110 (dec: 6)
X100 0000 0111 1111 1000 1000 100 (dec: 4)
1000 0000 1111 1111 0000 0000 000 (dec: 0)
So when the LSB is the 0th bit, fold*5 >> 1 will be 2, so we put zero at index 2 of the array. When the LSB is the 1st bit, fold*5 >> 1 will be 7, so we put 1 at index 7 of the array. Etc, etc.
From the results above we can build the array as {7, 2, 0, 4, 6, 3, 5, 1}. The ordering of the array looks meaningless, but really it is just related to the order in which fold*5 >> 1 spits out the index values.
As to whether or not you should actually do this, I imagine that it is hardware dependent, however, my processor can do the same thing about five times faster with:
unsigned long idx;
_BitScanForward64(&idx, bb);
bb &= bb - 1;
MIL-STD-1750A to Decimal Conversion Examples
I am looking at some examples in the 1750A format webpage and some of the examples do not really make sense. I have included the 1750A format specification at the bottom of this post in case anyone isn't familiar with it. Take this example from Table 3 of the 1750A format webpage: .625x2^4 = 5000 00 04 In binary 5000 00 04 is 0101 0000 0000 0000 0000 0000 0000 0100 If you convert this to decimal, it does not equal 10, which is .625x2^4. Maybe I am converting it incorrectly. Take the mantissa, 101 0000 0000 0000 0000 0000 and subtract 1 giving 100 1111 1111 1111 1111 1111. Then flip the bits, giving 011 0000 0000 0000 0000 0000. Move the decimal 4 places (since our exponent, 0100 is 4), giving us 0110.0000 0000 0000 0000 000. This equals 6.0, which is not .625x2^4. I believe the actual value, should be 0011 0000 0000 0000 0000 0000 0000 01000 or 30000004 in hex. Can anyone else confirm my suspicions that this value is labeled incorrectly in Table 3 of the 1750A format page above? Thank you
As explained previously, the sign+mantissa is interpreted as a 2's-complement value between -1 and +1. In your case, it's 0.101000000... (base-2). Which is 1/2 + 1/8 = 0.625 (base-10).
It all makes perfect sense. Here: 0101 0000 0000 0000 0000 0000 0000 0100 you've got: (0*20 + 1*2-1 + 0*2-2 + 1*2-3 + 0*2-4 + ... + 0*2-23) * 24 = (0.5 + 0.125) * 16 = 0.625 * 16 = 10 Just do the math.
C - 2D Array - Magic Square order 4
114 void fillDoubly(int square[20][20], int n){
115
116 int i, j, k=0, l=0, counter=0, test[400]={0}, diff=n/4-1;
117
118 for(i=0;i<n;i++) //first nested for loops for part 1)
119 for(j=0;j<n;j++){
120 counter++;
121 if( i=j || j=(n-1-i) ){
122 {
123 square[i][j] = counter;
124 test[counter-1] = 1;
125 }
126 }
127 }
128
129 for(i=n-1;i>=0;i--) // for part 2)
130 for(j=n-1;j>=0;j--){
131 if(square[i][j]==0){
132 while(test[k]!=0){
133 k++;
134 }
135 test[k]=1;
136 square[i][j]=k+1;
137 }
138 }
139 }
So basically, I have to generate magic square's of order 4
i.e. the rows and columns are divisible by 4.
I was provided the algorithm which is
to traverse the array and fill in the diagonal subsets
to traverse the array backwards and fill in the rest
I've done the 4x4 array with the above code and this extends to 8x8,12x12 etc.
but I'm stuck at part 1) which is to fill in the diagonal subsets(e.g. split 8x8 into 4x4 and take that diagonal instead)...I'm not sure how to do that, only managed to fill in the diagonal itself
if( i=j || j=(n-1-i) ){
tldr, The above is the condition I use to know if it's diagonal, any suggestions how I can change the condition to know if it's the diagonal subset not diagonal?
Thanks
From what I understand from the tutorial you linked you wish to split your matrix into 16 equal submatrices, and fill take the diagonals accross these submatrices. Hence for a 8x8 matrix you want to achive: | 0 | 1 | 2 | 3 | _ 0001 0002 0000 0000 0000 0000 0007 0008 0 0009 0010 0000 0000 0000 0000 0015 0016 _ 0000 0000 0019 0020 0021 0022 0000 0000 1 0000 0000 0027 0028 0029 0030 0000 0000 _ 0000 0000 0035 0036 0037 0038 0000 0000 2 0000 0000 0043 0044 0045 0046 0000 0000 _ 0049 0050 0000 0000 0000 0000 0055 0056 3 0057 0058 0000 0000 0000 0000 0063 0064 _ Here the submatrices are 2x2, if the matrix were 12x12, it would be subdivided into 16 submatrixes of 3x3. If you use these submatrices as indexes with which to find the diagonal (i.e. i==j) you can use the expression: if( (i/w)==(j/w) || (j/w)==(3-(i/w))) Where w = n/4, which is the order of your square submatrix (for 8x8, this is 2). So i/w will state in which submatrix (0 to 3) the current matrix index i resides.