Is there a way to convert from UTF8 to ISO-8859-1? - c

Question

My software is getting some strings in UTF8 than I need to convert to ISO 8859 1. I know that UTF8 domain is bigger than ISO 8859. But the data in UTF8 has been previously upconverted from ISO, so I should not miss anything.
I would like to know if there is an easy / direct way to convert from UTF8 to iso-8859-1.

Answer 1

Here is a function you might find useful: utf8_to_latin9(). It converts to ISO-8859-15 (including EURO, which ISO-8859-1 does not have), but also works correctly for the UTF-8->ISO-8859-1 conversion part of a ISO-8859-1->UTF-8->ISO-8859-1 round-trip.
The function ignores invalid code points similar to //IGNORE flag for iconv, but does not recompose decomposed UTF-8 sequences; that is, it won't turn U+006E U+0303 into U+00F1. I don't bother recomposing because iconv does not either.
The function is very careful about the string access. It will never scan beyond the buffer. The output buffer must be one byte longer than length, because it always appends the end-of-string NUL byte. The function returns the number of characters (bytes) in output, not including the end-of-string NUL byte.
/* UTF-8 to ISO-8859-1/ISO-8859-15 mapper.
* Return 0..255 for valid ISO-8859-15 code points, 256 otherwise.
*/
static inline unsigned int to_latin9(const unsigned int code)
{
/* Code points 0 to U+00FF are the same in both. */
if (code < 256U)
return code;
switch (code) {
case 0x0152U: return 188U; /* U+0152 = 0xBC: OE ligature */
case 0x0153U: return 189U; /* U+0153 = 0xBD: oe ligature */
case 0x0160U: return 166U; /* U+0160 = 0xA6: S with caron */
case 0x0161U: return 168U; /* U+0161 = 0xA8: s with caron */
case 0x0178U: return 190U; /* U+0178 = 0xBE: Y with diaresis */
case 0x017DU: return 180U; /* U+017D = 0xB4: Z with caron */
case 0x017EU: return 184U; /* U+017E = 0xB8: z with caron */
case 0x20ACU: return 164U; /* U+20AC = 0xA4: Euro */
default: return 256U;
}
}
/* Convert an UTF-8 string to ISO-8859-15.
* All invalid sequences are ignored.
* Note: output == input is allowed,
* but input < output < input + length
* is not.
* Output has to have room for (length+1) chars, including the trailing NUL byte.
*/
size_t utf8_to_latin9(char *const output, const char *const input, const size_t length)
{
unsigned char *out = (unsigned char *)output;
const unsigned char *in = (const unsigned char *)input;
const unsigned char *const end = (const unsigned char *)input + length;
unsigned int c;
while (in < end)
if (*in < 128)
*(out++) = *(in++); /* Valid codepoint */
else
if (*in < 192)
in++; /* 10000000 .. 10111111 are invalid */
else
if (*in < 224) { /* 110xxxxx 10xxxxxx */
if (in + 1 >= end)
break;
if ((in[1] & 192U) == 128U) {
c = to_latin9( (((unsigned int)(in[0] & 0x1FU)) << 6U)
| ((unsigned int)(in[1] & 0x3FU)) );
if (c < 256)
*(out++) = c;
}
in += 2;
} else
if (*in < 240) { /* 1110xxxx 10xxxxxx 10xxxxxx */
if (in + 2 >= end)
break;
if ((in[1] & 192U) == 128U &&
(in[2] & 192U) == 128U) {
c = to_latin9( (((unsigned int)(in[0] & 0x0FU)) << 12U)
| (((unsigned int)(in[1] & 0x3FU)) << 6U)
| ((unsigned int)(in[2] & 0x3FU)) );
if (c < 256)
*(out++) = c;
}
in += 3;
} else
if (*in < 248) { /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
if (in + 3 >= end)
break;
if ((in[1] & 192U) == 128U &&
(in[2] & 192U) == 128U &&
(in[3] & 192U) == 128U) {
c = to_latin9( (((unsigned int)(in[0] & 0x07U)) << 18U)
| (((unsigned int)(in[1] & 0x3FU)) << 12U)
| (((unsigned int)(in[2] & 0x3FU)) << 6U)
| ((unsigned int)(in[3] & 0x3FU)) );
if (c < 256)
*(out++) = c;
}
in += 4;
} else
if (*in < 252) { /* 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx */
if (in + 4 >= end)
break;
if ((in[1] & 192U) == 128U &&
(in[2] & 192U) == 128U &&
(in[3] & 192U) == 128U &&
(in[4] & 192U) == 128U) {
c = to_latin9( (((unsigned int)(in[0] & 0x03U)) << 24U)
| (((unsigned int)(in[1] & 0x3FU)) << 18U)
| (((unsigned int)(in[2] & 0x3FU)) << 12U)
| (((unsigned int)(in[3] & 0x3FU)) << 6U)
| ((unsigned int)(in[4] & 0x3FU)) );
if (c < 256)
*(out++) = c;
}
in += 5;
} else
if (*in < 254) { /* 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx */
if (in + 5 >= end)
break;
if ((in[1] & 192U) == 128U &&
(in[2] & 192U) == 128U &&
(in[3] & 192U) == 128U &&
(in[4] & 192U) == 128U &&
(in[5] & 192U) == 128U) {
c = to_latin9( (((unsigned int)(in[0] & 0x01U)) << 30U)
| (((unsigned int)(in[1] & 0x3FU)) << 24U)
| (((unsigned int)(in[2] & 0x3FU)) << 18U)
| (((unsigned int)(in[3] & 0x3FU)) << 12U)
| (((unsigned int)(in[4] & 0x3FU)) << 6U)
| ((unsigned int)(in[5] & 0x3FU)) );
if (c < 256)
*(out++) = c;
}
in += 6;
} else
in++; /* 11111110 and 11111111 are invalid */
/* Terminate the output string. */
*out = '\0';
return (size_t)(out - (unsigned char *)output);
}
Note that you can add custom transliteration for specific code points in the to_latin9() function, but you are limited to one-character replacements.
As it is currently written, the function can do in-place conversion safely: input and output pointers can be the same. The output string will never be longer than the input string. If your input string has room for an extra byte (for example, it has the NUL terminating the string), you can safely use the above function to convert it from UTF-8 to ISO-8859-1/15. I deliberately wrote it this way, because it should save you some effort in an embedded environment, although this approach is a bit limited wrt. customization and extension.
Edit:
I included a pair of conversion functions in an edit to this answer for both Latin-1/9 to/from UTF-8 conversion (ISO-8859-1 or -15 to/from UTF-8); the main difference is that those functions return a dynamically allocated copy, and keep the original string intact.

Answer 2

iconv - perform character set conversion
size_t iconv(iconv_t cd,
char **inbuf, size_t *inbytesleft,
char **outbuf, size_t *outbytesleft);
iconv_t iconv_open(const char *tocode, const char *fromcode);
tocode is "ISO_8859-1" and fromcode is "UTF-8".
Working example:
#include <iconv.h>
#include <stdio.h>
int main (void) {
iconv_t cd = iconv_open("ISO_8859-1", "UTF-8");
if (cd == (iconv_t) -1) {
perror("iconv_open failed!");
return 1;
}
char input[] = "Test äöü";
char *in_buf = &input[0];
size_t in_left = sizeof(input) - 1;
char output[32];
char *out_buf = &output[0];
size_t out_left = sizeof(output) - 1;
do {
if (iconv(cd, &in_buf, &in_left, &out_buf, &out_left) == (size_t) -1) {
perror("iconv failed!");
return 1;
}
} while (in_left > 0 && out_left > 0);
*out_buf = 0;
iconv_close(cd);
printf("%s -> %s\n", input, output);
return 0;
}

Answer 3

The following example uses iconv library also.
It works even when you have a file (or input stream) that contains mixed UTF-8 and ISO-8859-1 characters (this could happen, for example, if you have an UTF-8 file and edit it in environement that uses ISO-8859-1).
int Utf8ToLatin1(char* input, char* output, size_t size)
{
size_t in_left = size;
size_t out_left = size;
char *in_buf = input;
char *out_buf = output;
iconv_t cd = iconv_open("ISO_8859-1", "UTF-8");
if (cd == (iconv_t)-1) {
(void) fprintf(stderr, "iconv_open() failed, msg encoding will be kept!");
strncpy(output, input, size);
return -1;
}
do {
if (iconv(cd, &in_buf, &in_left, &out_buf, &out_left) == (size_t) -1) {
if (errno == EILSEQ) {
/* Input conversion stopped due to an input byte that
* does not belong to the input codeset.
*/
printf("Input conversion stopped due to an input byte that does not belong to the input codeset.\n");
*out_buf= *in_buf;
out_buf++ ;out_left--;
in_buf++ ;in_left--;
} else if (errno == E2BIG) {
/* Input conversion stopped due to lack of space in
* the output buffer.
*/
printf("Input conversion stopped due to lack of space in the output buffer.\n");
perror("iconv failed!, propably the encoding is already Latin, msg encoding will be kept!\n");
strncpy(output, input, size);
return -1;
} else if (errno == EINVAL) {
/* Input conversion stopped due to an incomplete
* character or shift sequence at the end of the
* input buffer.
*/
printf("Input conversion stopped due to an incomplete character or shift sequence at the end of the input buffer.\n");
*out_buf= *in_buf;
out_buf++ ;out_left--;
in_buf++ ;in_left--;
}
}
} while (in_left > 0 && out_left > 0);
*out_buf = 0;
iconv_close(cd);
printf("*********************************************************\n");
printf("ISO-8859-1:\n %s\n", input, output);
return 0;
}