Changeset 349

Timestamp:

Nov 26, 2009, 12:34:37 AM (15 years ago)

Author:

Dmitry A. Kuminov

Message:

corelib/codec: Implemented the "System" text codec which is always used as the default codec for the locale and uses the OS/2 ULS API to perform conversions from the process codepage to Unicode and back. The encoding part of the LANG variable is no longer recognized (#46).

File:

• trunk/src/corelib/codecs/qtextcodec.cpp (modified) (5 diffs)

trunk/src/corelib/codecs/qtextcodec.cpp ¶

@@ -59 +59 @@
 #endif
 
+#if defined(Q_OS_OS2)
+#  include <unidef.h>
+#  include <uconv.h>
+#  include "qvector.h"
+#endif
+
 #include "qutfcodec_p.h"
 #include "qsimplecodec_p.h"
@@ -392 +398 @@
 }
 
+#elif defined(Q_OS_OS2)
+
+class QOs2LocalCodec: public QTextCodec
+{
+public:
+    QOs2LocalCodec();
+    ~QOs2LocalCodec();
+
+    QString convertToUnicode(const char *, int, ConverterState *) const;
+    QByteArray convertFromUnicode(const QChar *, int, ConverterState *) const;
+
+    QByteArray name() const;
+    int mibEnum() const;
+
+private:
+    UconvObject uoSubYes;
+    UconvObject uoSubNo;
+};
+
+QOs2LocalCodec::QOs2LocalCodec() : uoSubYes(0), uoSubNo(0)
+{
+    // create the conversion object for the process code page that performs
+    // substitution of invalid characters with '?'
+    UniCreateUconvObject((UniChar *)L"@sub=yes", &uoSubYes);
+    // same as above but doesn't perform substitution
+    UniCreateUconvObject((UniChar *)L"@sub=no", &uoSubNo);
+}
+
+QOs2LocalCodec::~QOs2LocalCodec()
+{
+    UniFreeUconvObject(uoSubNo);
+    UniFreeUconvObject(uoSubYes);
+}
+
+static void qOs2LocalCodecStateFree(QTextCodec::ConverterState *state)
+{
+    delete reinterpret_cast<char *>(state->d);
+}
+
+QString QOs2LocalCodec::convertToUnicode(const char *chars, int length,
+                                         ConverterState *state) const
+{
+    QString res;
+
+    if (!chars)
+        return res;
+    if (!length)
+        return QLatin1String("");
+
+    UconvObject uo = uoSubYes;
+    if (state && (state->flags & ConvertInvalidToNull))
+        uo = uoSubNo;
+    Q_ASSERT(uo);
+    if (!uo)
+        return res;
+
+    int remainingChars = 0;
+    char *remainingBuffer = 0;
+
+    if (state) {
+        // stateful conversion
+        remainingBuffer = reinterpret_cast<char *>(&state->d);
+        if (remainingBuffer) {
+            // restore state
+            remainingChars = state->remainingChars;
+        } else {
+            // first time, add the destructor for state->d
+            state->flags |= FreeFunction;
+            QTextCodecUnalignedPointer::encode(state->state_data,
+                                               qOs2LocalCodecStateFree);
+        }
+    }
+
+    const char *mbPtr = chars;
+    size_t mbLeft = length;
+
+    QByteArray mbExtra;
+    if (remainingChars) {
+        // we have to prepend the remaining bytes from the previous conversion
+        mbLeft += remainingChars;
+        mbExtra.resize(mbLeft);
+        mbPtr = mbExtra.data();
+
+        memcpy(mbExtra.data(), remainingBuffer, remainingChars);
+        memcpy(mbExtra.data() + remainingChars, chars, length);
+
+        remainingBuffer = 0;
+        remainingChars = 0;
+    }
+
+    size_t ucLen = mbLeft;
+    QString ucBuf(ucLen, '\0');
+    UniChar *ucPtr = reinterpret_cast<UniChar *>(ucBuf.data());
+    size_t ucLeft = ucLen;
+
+    size_t nonIdent = 0;
+    int rc;
+
+    while (mbLeft) {
+        rc = UniUconvToUcs(uo, (void**)&mbPtr, &mbLeft, &ucPtr, &ucLeft,
+                           &nonIdent);
+        if (rc == ULS_BUFFERFULL) {
+            size_t ucDone = ucLen - ucLeft;
+            size_t mbDone = length - mbLeft;
+            // assume that mbLeft/ucLeft is an approximation of mbDone/ucDone
+            ucLen = ucDone + (mbLeft * ucDone) / mbDone;
+            ucBuf.resize(ucLen);
+            ucPtr = reinterpret_cast<UniChar *>(ucBuf.data() + ucDone);
+        } else if (rc == ULS_ILLEGALSEQUENCE && state) {
+            // conversion stopped because the remaining inBytesLeft make up
+            // an incomplete multi-byte sequence; save them for later
+            remainingBuffer = new char[mbLeft];
+            memcpy(remainingBuffer, mbPtr, mbLeft);
+            remainingChars = mbLeft;
+            break;
+        } else if (rc != ULS_SUCCESS) {
+            // just fail on an unexpected error (will return what we've got)
+            qWarning("QOs2LocalCodec::convertToUnicode: UniUconvToUcs failed "
+                     "with %d", rc);
+            break;
+        }
+    }
+
+    ucBuf.resize(ucLen - ucLeft);
+    res = ucBuf;
+
+    if (state) {
+        // update the state
+        state->invalidChars = nonIdent;
+        state->remainingChars = remainingChars;
+        state->d = remainingBuffer;
+    }
+
+    return res;
+}
+
+QByteArray QOs2LocalCodec::convertFromUnicode(const QChar *uchars, int length,
+                                              ConverterState *state) const
+{
+    QByteArray res;
+
+    if (!uchars)
+        return res;
+    if (!length)
+        return QByteArray("");
+
+    UconvObject uo = uoSubYes;
+    if (state && (state->flags & ConvertInvalidToNull))
+        uo = uoSubNo;
+    Q_ASSERT(uo);
+    if (!uo)
+        return res;
+
+    const UniChar *ucPtr = reinterpret_cast<const UniChar *>(uchars);
+    size_t ucLeft = length;
+
+    QVector<QChar> ucExtra;
+    if (state && state->remainingChars) {
+        // we have one surrogate char to be prepended
+        Q_ASSERT(state->remainingChars == 1);
+        ucLeft += 1;
+        ucExtra.resize(ucLeft);
+        ucPtr = reinterpret_cast<const UniChar *>(ucExtra.data());
+
+        ucExtra[0] = state->state_data[0];
+        memcpy(ucExtra.data() + 1, uchars, length * sizeof(QChar));
+
+        state->remainingChars = 0;
+    }
+
+    // be optimistic (imply that one byte is necessary per every Unicode char)
+    size_t mbLen = length;
+    QByteArray mbBuf(mbLen, '\0');
+    char *mbPtr = mbBuf.data();
+    size_t mbLeft = mbLen;
+
+    size_t nonIdent = 0;
+    int rc;
+
+    while (ucLeft) {
+        rc = UniUconvFromUcs(uo, const_cast<UniChar **>(&ucPtr), &ucLeft,
+                             (void**)&mbPtr, &mbLeft, &nonIdent);
+        if (rc == ULS_BUFFERFULL) {
+            size_t mbDone = mbLen - mbLeft;
+            size_t ucDone = length - ucLeft;
+            size_t newLen = mbLen;
+            if (ucDone) {
+                // assume that ucLeft/mbLeft is an approximation of ucDone/mbDone
+                newLen = mbDone + (ucLeft * mbDone) / ucDone;
+            }
+            if (newLen == mbLen) {
+                // could not process a single Unicode char, double the size
+                mbLen *= 2;
+            } else {
+                mbLen = newLen;
+            }
+            mbBuf.resize(mbLen);
+            mbPtr = mbBuf.data() + mbDone;
+            mbLeft = mbLen - mbDone;
+        } else if (rc == ULS_ILLEGALSEQUENCE && state) {
+            // buffer ends in a surrogate
+            Q_ASSERT(ucLeft == 2);
+            state->state_data[0] = *ucPtr;
+            state->remainingChars = 1;
+            break;
+        } else if (rc != ULS_SUCCESS) {
+            // just fail on an unexpected error (will return what we've got)
+            qWarning("QOs2LocalCodec::convertFromUnicode: UniUconvFromUcs failed "
+                     "with %d", rc);
+            break;
+        }
+    }
+
+    mbBuf.resize(mbLen - mbLeft);
+    res = mbBuf;
+
+    if (state) {
+        // update the state
+        state->invalidChars = nonIdent;
+    }
+
+    return res;
+}
+
+QByteArray QOs2LocalCodec::name() const
+{
+    return "System";
+}
+
+int QOs2LocalCodec::mibEnum() const
+{
+    return 0;
+}
+
 #else
 
@@ -509 +749 @@
 #endif
 
-#if !defined(Q_OS_WIN32) && !defined(Q_OS_WINCE)
+#if !defined(Q_OS_WIN32) && !defined(Q_OS_WINCE) && !defined(Q_OS_OS2)
 static QTextCodec *checkForCodec(const char *name) {
     QTextCodec *c = QTextCodec::codecForName(name);
@@ -529 +769 @@
 {
 #if defined(Q_OS_WIN32) || defined(Q_OS_WINCE)
+    localeMapper = QTextCodec::codecForName("System");
+#elif defined(Q_OS_OS2)
     localeMapper = QTextCodec::codecForName("System");
 #else
@@ -712 +954 @@
     (void) new QWindowsLocalCodec;
 #endif // Q_OS_WIN32
+
+#if defined(Q_OS_OS2)
+    (void) new QOs2LocalCodec;
+#endif // Q_OS_OS2
 
     (void)new QUtf16Codec;