TextEncoder.cpp #1

#include "StdAfx.h"
#include "TextEncoder.h"
#include <String.h>

TextEncoder::TextEncoder(void)
{
}

TextEncoder::~TextEncoder(void)
{
}

void TextEncoder::SmokeTest(void)
{
    // test mode fo Unicode transcoding
    //wchar_t * wTestStr = L"The € quick ¢ brown fox jumped the fence <АБВГ>";
    wchar_t * wTestStr = L"\xD852\xDF62 The € quick ¢ brown fox jumped the fence <АБВГ>";
    //wchar_t * wTestStr = L"\xD852\xDF62 T €<АБВГ>";

    unsigned long uniCodeLong = 0x00024B62; // a Unicode point that requires a surrogate pair to encode in UTF-16
        
    SurrogatePair sp =  TextEncoder::UnicodePointToSurrogatePair(uniCodeLong);
    unsigned long uniCodeTest = TextEncoder::SurrogatePairToUnicodePoint(sp);

    wprintf(L"UTF-16 String: %s\r\n", wTestStr);

    char * aTestStr = TextEncoder::Utf16ToUtf8(wTestStr);
    printf("UTF-8 Translation: %s\r\n", aTestStr);

    wchar_t * wResultStr = TextEncoder::Utf8ToUtf16(aTestStr);

    wprintf(L"UTF-16 Retranslation: %s\r\n", wResultStr);
}

char * TextEncoder::CopyStr(const char *s)
{
    int cnt = strlen(s);
    char * newStr = new char[cnt+1];
    for (int idx = 0; idx < cnt; idx++)
        newStr[idx] = s[idx];
    newStr[cnt] = '\0';
    return newStr;
}

wchar_t * TextEncoder::CopyWStr(const wchar_t *s)
{
    int cnt = wcslen(s);
    wchar_t * newStr = new wchar_t[cnt+1];
    for (int idx = 0; idx < cnt; idx++)
        newStr[idx] = s[idx];
    newStr[cnt] = '\0';
    return newStr;
}

SurrogatePair TextEncoder::UnicodePointToSurrogatePair(unsigned long codePoint)
{
    SurrogatePair val;
    val.first = 0;
    val.second = 0;

    // need to encode an up to 21 bit code point into a two wchar surrogate pair.
        
    codePoint -= 0x10000; //this gives us a 20 bit number, yyyyyyyyyyxxxxxxxxxx

    // yyyyyyyyyyxxxxxxxxxx -> the two surrogate pair using this transfermation
    //              110110yyyyyyyyyy (0xD800 + yyyyyyyyyyy) 110111xxxxxxxxxx (0xDC00 +yyyyyyyyyy)

    val.first = 0xD800 | ((codePoint >> 10) & 0x03FF); //yyyyyyyyyy goes in the first of the pair

    val.second = 0xDC00 | (codePoint & 0x03FF); //xxxxxxxxxx goes in the second of the pair

    return val;
}

unsigned long TextEncoder::SurrogatePairToUnicodePoint(wchar_t first, wchar_t second)
{
    unsigned long val = 0;

    // get the 20 bit value from the pair
    //  110110yyyyyyyyyy 110111xxxxxxxxxx --> 000000000000YYYY yyyyyyxxxxxxxxxx (32 bit number, need only 20 bits)
    val = ((unsigned long)(0x03FF & first)) << 10; // get yyyyyyyyyy in the right place
    val |= ((unsigned long)(0x03FF & second)); //get xxxxxxxxxx

    val += 0x10000;

    return val;
}

unsigned long TextEncoder::SurrogatePairToUnicodePoint(SurrogatePair surrogates)
{
    return SurrogatePairToUnicodePoint(surrogates.first, surrogates.second);
}

void TextEncoder::RecodeUtf16CharInUtf8(const wchar_t * pSrc, char * pDest)
{
    // check two see if this is a sing UTF-16 wchar or a surrogate pair
    if (( *pSrc < 0xD800 ) || ( *pSrc > 0xDFFF ))
    {
        //single wchar 
        if (*pSrc < 128)
        {
            // single byte ASCII UTF16 character
            *pDest = (char) *pSrc;
            return;
        }
        if (*pSrc < 0x0800)
        {
            // two byte UTF8 character
            // 00000yyy XXxxxxxx --> 110yyyXX 10xxxxxx
            
            // first char
            char c = 0xC0 | (char)(((wchar_t)(*pSrc >> 6)) & 0x1F); 
            *pDest = c;

            //second char
            c = 0x80 | (char)(*pSrc & 0x3F);
            *(pDest + 1) = c;
            return;
        }
        else
        {
            // three byte UTF8 character
            // YYYYyyyy XXxxxxxx --> 1110YYYY 10yyyyXX 10xxxxxx
            
            // first char
            char c = 0xE0 | (((wchar_t)(*pSrc >> 12)) & 0x0F); 
            *pDest = c;

            //second char
            c = 0x80 | (char)(((wchar_t)(*pSrc >> 6)) & 0x3F);
            *(pDest + 1) = c;

            //third char
            c = 0x80 | (char)(*pSrc & 0x3F);
            *(pDest + 2) = c;
            return;
        }
    }

    // this is a two wchar surrogate pair and it's going to take 4 bytes

    // get the 20 bit value from the pair
    //  110110YYYYyyyyyy 110111xxxxxxxxxx --> 000000000000YYYY yyyyyyxxxxxxxxxx +0x10000 (32 bit number, need only 21 bits)
    unsigned long uCodePoint = SurrogatePairToUnicodePoint(*pSrc, *(pSrc + 1));

    // encode the now bits into 4 UTF-8 bytes
    // 000ZZZzz YYYYyyyy XXxxxxxx --> 11110ZZZ 10zzYYYY 10yyyyXX 10xxxxxx

    // first char
    char c = 0xF0 | (char)((unsigned long)(((uCodePoint >> 18)) & 0x07)); 
    *pDest = c;

    //second char
    c = 0x80 | (char)((unsigned long)(((uCodePoint >> 12)) & 0x3F));
    *(pDest + 1) = c;

    //third char
    c = 0x80 | (char)((unsigned long)(((uCodePoint >> 6)) & 0x3F));
    *(pDest + 2) = c;

    //fourth char
    c = 0x80 | (char)((unsigned long)(uCodePoint & 0x3F));
    *(pDest + 3) = c;

    return;
}

void TextEncoder::RecodeUtf8CharInUtf16(const char * pSrc, wchar_t * pDest)
{

    if ( !(*pSrc & 0x80) ) // ASCII character < 128
    {
        *pDest = (wchar_t) *pSrc;
        return;
    }
    unsigned int w1 = 0;
    unsigned long w2 = 0;

    if ( (*pSrc & 0xE0) == 0xC0) // 110xxxxx 2 bytes in UTF-8
    {
        // 110yyyXX 10xxxxxx UTF-8 -->  00000yyy XXxxxxxx UTF-16

        w1 = ((wchar_t)(*pSrc & 0x1C)) << 6; // puts the yyy in the lower bits of the upper byte where they belong
        w1 &= 0xFF00; // mask off the lower byte

        w1 |= ((*pSrc & 0x03)) << 6; // get the two XX bits from the first utf-8 character
                                   // into the right bits of the lower byte
        w1 |= (*(pSrc+1) & 0x3F);  // copy the remaining xxxxxx bits from the 2nd UTF-8 byte

        *pDest = (wchar_t) w1;

        return;
    }
    if ( (*pSrc & 0xF0) == 0xE0) // 1110xxxx 3 bytes in UTF-8
    {
        // 1110YYYY 10yyyyXX 10xxxxxx UTF-8 -->  YYYYyyyy XXxxxxxx UTF-16

        w1 = ((wchar_t)(*pSrc & 0x0F)) << 12; // puts the YYYY in the upper bits of the upper byte

        w1 |= ((wchar_t)(*(pSrc + 1) & 0x3C)) << 6; //puts yyyy in the lower bits of the upper byte

        w1 |= ((wchar_t)(*(pSrc + 1) & 0x03)) << 6; // get the two XX bits from the second utf-8 character
                                       // into the upper bits of the lower byte

        w1 |= (*(pSrc+2) & 0x2F);  // copy the remaining xxxxxx bits from the 3rd UTF-8 byte

        *pDest = (wchar_t) w1;

        return;
    }
    if ( (*pSrc & 0xF1) == 0xF0) // 11110xxx
    {
        // 11110ZZZ 10zzYYYY 10yyyyXX 10xxxxxx UTF-8 --> 000000 000ZZZzz YYYYyyyy XXxxxxxx UTF-16 (extended code plane)

        w2 = ((unsigned long)(*pSrc & 0x07)) << 2; // puts the ZZZ in the middle bits of the lower byte
        w2 |= ((unsigned long)(*(pSrc + 1) & 0x30)) >> 4; // puts the zz in the lower bits of the lower byte
        w2 <<= 16; //Shift the lower byte to the third byte

        w1 = ((unsigned long)(*(pSrc + 1) & 0x0F)) << 4; // puts the YYYY in the upper bits of the lower byte
        w1 |= ((unsigned long)(*(pSrc + 2) & 0x3C)) >> 2; //puts yyyy in the lower bits of the lower byte
        w1 <<= 8; // shift into the upper byte where they belong

        w1 |= ((unsigned long)(*(pSrc+2) & 0x03)) << 6; // get the two XX bits from the 3rd utf-8 character
                                       // into the right bits of the lower byte

        w1 |= ((unsigned long)(*(pSrc+3)) & 0x2F);  // copy the remaining xxxxxx bits from the 4th UTF-8 byte

        w2 |= w1; // put both bytes in w1 we now have up to a 21 bit number


        // need to encode the 21 bit Unicode point into a two wchar surrogate pair.
        SurrogatePair sp = UnicodePointToSurrogatePair(w2);

        *pDest = sp.first;

        *(pDest + 1) = sp.second;

        return;
    }
    // Not a valid Utf-8 leading byte

    *pDest = (wchar_t) 0xFFFD;

    return;

}
wchar_t * TextEncoder::Utf8ToUtf16(const char * pStr)
{
    int cnt = Utf8StrLen( pStr );

    // initially allow space for each character to take 4 bytes
    wchar_t * pCpy = new wchar_t[(2 * cnt) + 1];

    // copy each 1,2,3, or 4 byte utf-8 encoded character into the proper
    // sized (1 or 2 wchar_t) utf-16 encoded character
    int idx1 = 0;
    int idx2 = 0;
    for (int idx = 0; idx < cnt; idx++)
    {
        RecodeUtf8CharInUtf16(&pStr[idx2], &pCpy[idx1]);

        idx1 += Utf16CharSize(pCpy[idx1]);
        idx2 += Utf8CharSize(pStr[idx2]);
    }
    pCpy[idx1] = '\0';

    // copy into the 'right' sized array
    wchar_t * pRet = CopyWStr(pCpy);
    delete pCpy;

    return pRet;
}

wchar_t * TextEncoder::AsciiToUtf16(const char * pStr)
{
    int cnt = strlen( pStr );

    wchar_t * pCpy = new wchar_t[cnt + 1]; 

    for (int idx = 0; idx < cnt; idx++)
    {
        pCpy[idx] = (wchar_t) pStr[idx];
    }
    pCpy[cnt] = '\0';
    return pCpy;

}

int TextEncoder::Utf16StrLen(const wchar_t * pStr)  // length in characters, not bytes
{
    int cnt = 0;
    int idx = 0;
    while ( pStr[idx] != '\0')
     {
         idx += Utf16CharSize(pStr[idx]);
         cnt++;
    }
    return cnt;
}

int TextEncoder::Utf16StrBytes(const wchar_t * pStr)  // length in bytes, not characters
{
    int cnt = 0;
    while ( pStr[cnt] != '\0')
        cnt += 2;
    return cnt;
}

int TextEncoder::Utf16CharSize(const wchar_t c)  // How many wchar_t's in this character, 1 or 2?
{
    if (( c >= 0xD800 ) && ( c <= 0xDFFF ))
        return 2;
    return 1;
}

char * TextEncoder::Utf16ToUtf8(const wchar_t * pStr)
{
    int cnt = Utf16StrLen(pStr);

    // allow 4 bytes per character (max length possible)
    char * pCpy = new char[(4 * cnt) + 1];

    // copy each 1 or 2 wchar character into a 1,2,3, or 4 byte utf-8 encoded character
    int idx1 = 0;
    int idx2 = 0;
    for (int idx = 0; idx < cnt; idx++)
    {
        RecodeUtf16CharInUtf8(&pStr[idx2], &pCpy[idx1]);

        idx1 += Utf8CharSize(pCpy[idx1]);
        idx2 += Utf16CharSize(pStr[idx2]);
    }
    pCpy[idx1] = '\0';

    // copy into the 'right' sized array
    char * pRet = CopyStr(pCpy);
    delete pCpy;

    return pRet;
}
char * TextEncoder::AsciiToUtf8(const char * pStr)
{
    // An Ascii string is already utf-8, so just copy it
    return CopyStr(pStr);
}

// length in characters, not bytes
int TextEncoder::Utf8StrLen(const char* pStr)
{
    int cnt = 0;
    int idx = 0;
    while ( pStr[idx] != '\0')
     {
         idx += Utf8CharSize(pStr[idx]);
         cnt++;
    }
    return cnt;
}
 
// length in bytes, not characters
int TextEncoder::Utf8StrBytes(const char * pStr)
{
    int cnt = 0;
    while ( pStr[cnt] != '\0')
        cnt++;
    return cnt;
}

// How many bytes in this character, 1, 2, 3, or 4?
int TextEncoder::Utf8CharSize(char c)
{
    if ( !(c & 0x80) ) // ASCII character < 128
        return 1;
    if ( (c & 0xE0) == 0xC0) // 110xxxxx
        return 2;
    if ( (c & 0xF0) == 0xE0) // 1110xxxx
        return 3;
    if ( (c & 0xF1) == 0xF0) // 11110xxx
        return 4;

    // Not a valid Utf-8 leading byte
    return 1;
}

char * TextEncoder::Utf16ToAscii(const wchar_t * pStr)
{
    int cnt = Utf16StrLen(pStr);

    char * pCpy = new char[cnt + 1];

    int idx2 = 0;
    for (int idx = 0; idx < cnt; idx++)
    {
        if (pStr[idx2] < 128)
            pCpy[idx] = (char) pStr[idx2];
        else
            pCpy[idx] = '?'; // not mapable

        idx2 += Utf16CharSize(pStr[idx2]);
    }
    pCpy[cnt] = '\0';
    return pCpy;
}

char * TextEncoder::Utf8ToAscii(const char * pStr)
{
    int cnt = Utf8StrLen(pStr);

    char * pCpy = new char[cnt + 1];

    int idx2 = 0;
    for (int idx = 0; idx < cnt; idx++)
    {
        if (((unsigned char) pStr[idx2]) < 128)
            pCpy[idx] = (char) pStr[idx2];
        else
            pCpy[idx] = '?'; // not mapable

        idx2 += Utf8CharSize(pStr[idx2]);
    }
    pCpy[cnt] = '\0';
    return pCpy;
}