ANSI to UTF8

[teo]

On Mon, 31 Jan 2011 17:08:33 +0000, Janusch wrote:

> Hello!
> 
> I'm trying to convert ANSI characters to UTF8 that but it doesn't work
> correctly.
> 
> I used the following:
> 
> void main() {
> 	writeln(convertToUTF8("ä"));
> }
> 
> string convertToUTF8(string text) {
> 
> 	string result;
> 
> 	for (uint i=0; i<text.length; i++) {
> 		char ch = text[i];
> 		if (ch < 0x80) {
> 			result ~= ch;
> 		} else {
> 			result ~= 0xC0 | (ch >> 6);
> 			result ~= 0x80 | (ch & 0x3F);
> 		}
> 	}
> 	return result;
> 
> }
> 
> But writeln doesn't print anything (only a blank line), but not my
> character. The same problem exists for similar characters like ü or ö.
> 
> Is there anything I'm doing wrong?


You may give a try to the following code. It is based on PHP 5.2.9
---
module ISO88591;

/+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
++++++
decode latin-1 (ISO-8859-1) string to UTF-8
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+++++/
string decode(byte[] content)
{
	byte[] result = new byte[content.length * 4];
	uint n = 0;
	uint i = content.length;
	ubyte* p = cast(ubyte*)content.ptr;
	while (i > 0)
	{
		uint c = cast(uint)*p;
		if (c < 0x80)
		{
			result[n++] = cast(ubyte)c;
		}
		else if (c < 0x800)
		{
			result[n++] = cast(ubyte)(0xC0 | (c >> 6));
			result[n++] = cast(ubyte)(0x80 | (c & 0x3F));
		}
		else if (c < 0x10000)
		{
			result[n++] = cast(ubyte)(0xE0 | (c >> 12));
			result[n++] = cast(ubyte)(0xC0 | ((c >> 6) & 
0x3F));
			result[n++] = cast(ubyte)(0x80 | (c & 0x3F));
		}
		else if (c < 0x200000)
		{
			result[n++] = cast(ubyte)(0xF0 | (c >> 18));
			result[n++] = cast(ubyte)(0xE0 | ((c >> 12) & 
0x3F));
			result[n++] = cast(ubyte)(0xC0 | ((c >> 6) & 
0x3F));
			result[n++] = cast(ubyte)(0x80 | (c & 0x3F));
		}
		p++;
		i--;
	}
	result.length = n;
	return cast(string)result;
}

/+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
++++++
encode UTF-8 string to latin-1 (ISO-8859-1)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+++++/
byte[] encode(string content)
{
	byte[] buf = cast(byte[])content;
	byte[] result = new byte[buf.length];
	uint n = 0;
	uint i = buf.length;
	ubyte* p = cast(ubyte*)buf.ptr;
	while (i > 0)
	{
		uint c = *p;
		if (c >= 0xF0)
		{
			// four bytes encoded, 21 bits
			if (i >= 4)
			{
				c = ((p[0] & 0x07) << 18) | ((p[1] & 
0x3F) << 12) | ((p[2] & 0x3F) << 6) | (p[3] & 0x3F);
			}
			else
			{
				c = 0x3F;
			}
			p += 4;
			i -= 4;
		}
		else if (c >= 0xE0)
		{
			// three bytes encoded, 16 bits
			if (i >= 3)
			{
				c = ((p[0] & 0x3F) << 12) | ((p[1] & 
0x3F) << 6) | (p[2] & 0x3F);
			}
			else
			{
				c = 0x3F;
			}
			p += 3;
			i -= 3;
		}
		else if (c >= 0xC0)
		{
			// two bytes encoded, 11 bits
			if (i >= 2)
			{
				c = ((p[0] & 0x3F) << 6) | (p[1] & 0x3F);
			}
			else
			{
				c = 0x3F;
			}
			p += 2;
			i -= 2;
		}
		else
		{
			p++;
			i--;
		}
		// use '?' (0x3F) if no mapping is possible
		result[n++] = cast(ubyte)((c > 0xFF) ? 0x3F : c);
	}
	result.length = n;
	return result;
}
---

I wrote it for D1 and did now quick tests with D2. It should be working.
Please give feedback.

And here is my test program:

import std.stdio;
import ISO88591;

void main()
{
	string str = "äöüß";
	auto tmp = encode(str);
	writefln("latin-1:%x", cast(ubyte[])tmp);
	auto res = decode(tmp);
	writefln("utf-8:%x:%s", cast(ubyte[])res, res);

	return;
}