What can cause `recvfrom()` not to set - c

Question

So I'm using Unix Domain Socket (datagram) to communicate between two processes on my machine. The client sends queries to the server which is making some calculations and returns the result to the client.
My server receives queries from the client using recvfrom(), and my problem is that on the first time recvfrom() is being called, it does not set struct sockaddr *restrict address which is the address of the sender.
This happens only on the first time recvfrom() is being called, the following queries are working perfectly fine.
What am I doing wrong here?
client.c:
/* relevant code from client.c */
sockfd = socket(AF_UNIX, SOCK_DGRAM, 0);
memset(&client, 0, sizeof(client));
client.sun_family = AF_UNIX;
strcpy(client.sun_path, CLIENT_PATH);
bind(sockfd, (struct sockaddr *)&client, sizeof(struct sockaddr_un);
memset(&server, 0, sizeof(server));
server.sun_family = AF_UNIX;
strcpy(server.sun_path, SERVER_PATH);
while(1) {
FD_ZERO(&readfds);
FD_SET(sockfd, &readfds);
FD_SET(STDIN, &readfds);
select(sockfd+1, &readfds, NULL, NULL, NULL);
if (FD_ISSET(STDIN, &readfds)) {
nbytes = read(STDIN, buf, sizeof buf);
buf[nbytes] = '\0';
if (sendto(sockfd, buf, strlen(buf), 0,
(struct sockaddr *)&server,
sizeof(struct sockaddr_un)) < strlen(buf)) {
perror("send");
}
}
if (FD_ISSET(sockfd, &readfds)) {
if ((nbytes = recvfrom(sockfd, buf, MAXDATASIZE, 0, NULL, NULL)) < 0) {
perror("recv");
}
buf[nbytes] = '\0';
printf(">> %s", buf);
}
}
server.c:
/* relevant code from server.c */
FD_ZERO(&master);
FD_ZERO(&read_fds);
if ((client = socket(AF_UNIX, SOCK_DGRAM, 0)) < 0) {
perror("client socket");
exit(3);
}
memset(&server, 0, sizeof(struct sockaddr_un));
server.sun_family = AF_UNIX;
strcpy(server.sun_path, SOCK_PATH);
unlink(server.sun_path);
if (bind(client, (struct sockaddr *) &server, sizeof(struct sockaddr_un)) < 0) {
perror("uds bind");
exit(3);
}
FD_SET(client, &master);
// keep track of the biggest file descriptor
fdmax = client; //so far, it's this one
// main loop
for(;;) {
read_fds = master; // copy it
if (select(fdmax+1, &read_fds, NULL, NULL, NULL) == -1) {
perror("select");
exit(4);
}
// run through the existing connections looking for data to read
for(i = 0; i <= fdmax; i++) {
if (FD_ISSET(i, &read_fds)) { // we got one!!
if (i == client) {
// data is from client console
if ((nbytes = recvfrom(i, &buf, sizeof buf, 0, (struct sockaddr *)&client_addr, &remotelen)) <= 0) {
if (nbytes == 0) {
FD_CLR(i, &master);
close(i);
} else {
perror("client: recv");
}
}
buf[nbytes-1] = '\0';
if (strcmp(buf, "command-1") == 0) {
if ((nbytes = sendto(i, cmd1_str, strlen(cmd1_str), 0, (struct sockaddr *)&client_addr, remotelen)) < 0) {
perror("sendto");
}
buf[0] = '\0';
} else {
sendto(i, unknown_str, strlen(unknown_str), 0, (struct sockaddr *)&client_addr, remotelen);
buf[0] = '\0';
}
}
}
} // END got new incoming connection
} // END looping through file descriptors
} // END for(;;)--and you thought it would never end!
Code was trimmed a to include only what's supposed to be important and relevant. Please, if anything important is missing, let me know and I'll add it.