eci: Add support for all ECIs (Big5, Korean, UCS-2BE)

backend/common.c

@@ -2,7 +2,7 @@
 
 /*
     libzint - the open source barcode library
-    Copyright (C) 2008 - 2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2008 - 2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -85,6 +85,18 @@ INTERNAL void to_upper(unsigned char source[]) {
     }
 }
 
+/* Returns the number of times a character occurs in a string */
+INTERNAL int chr_cnt(const unsigned char string[], const int length, const unsigned char c) {
+    int count = 0;
+    int i;
+    for (i = 0; i < length; i++) {
+        if (string[i] == c) {
+            count++;
+        }
+    }
+    return count;
+}
+
 /* Verifies that a string only uses valid characters */
 INTERNAL int is_sane(const char test_string[], const unsigned char source[], const int length) {
     int i, j, lt = (int) strlen(test_string);

backend/common.h

@@ -2,7 +2,7 @@
 
 /*
     libzint - the open source barcode library
-    Copyright (C) 2009 - 2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2009 - 2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -93,6 +93,7 @@ extern "C" {
     INTERNAL char itoc(const int source);
     INTERNAL int to_int(const unsigned char source[], const int length);
     INTERNAL void to_upper(unsigned char source[]);
+    INTERNAL int chr_cnt(const unsigned char string[], const int length, const unsigned char c);
     INTERNAL int is_sane(const char test_string[], const unsigned char source[], const int length);
     INTERNAL void lookup(const char set_string[], const char *table[], const char data, char dest[]);
     INTERNAL void bin_append(const int arg, const int length, char *binary);

backend/eci.c

@@ -1,7 +1,7 @@
 /*  eci.c - Extended Channel Interpretations
 
     libzint - the open source barcode library
-    Copyright (C) 2009 - 2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2009 - 2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -30,240 +30,241 @@
  */
 /* vim: set ts=4 sw=4 et : */
 
-#include <string.h>
-#include <stdio.h>
-#include "eci.h"
-#include "common.h"
 #ifdef _MSC_VER
 #include <malloc.h>
 #endif
+#include "common.h"
+#include "eci.h"
+#include "eci_sb.h"
+#include "sjis.h"
+#include "big5.h"
+#include "gb2312.h"
+#include "ksx1001.h"
 
-/* Convert Unicode to other character encodings */
+/* ECI 20 Shift JIS */
-INTERNAL int utf_to_eci(const int eci, const unsigned char source[], unsigned char dest[], int *length) {
+static int sjis_wctomb(unsigned char *r, const unsigned int wc) {
-    int in_posn;
+    int ret;
-    int out_posn;
+    unsigned int c;
-    int ext;
-    int done;
 
-    if (eci == 26 || eci == 899) {
+    ret = sjis_wctomb_zint(&c, wc);
-        /* Unicode or 8-bit binary data, do not process - just copy data across */
+    if (ret == 0) {
-        memcpy(dest, source, *length);
-        dest[*length] = '\0';
         return 0;
     }
+    if (ret == 2) {
+        r[0] = (unsigned char) (c >> 8);
+        r[1] = (unsigned char) (c & 0xff);
+    } else {
+        *r = (unsigned char) c;
+    }
+    return ret;
+}
+
+/* ECI 27 ASCII (ISO/IEC 646:1991 IRV (US)) */
+static int ascii_wctosb(unsigned char *r, const unsigned int wc) {
+    if (wc < 0x80) {
+        *r = (unsigned char) wc;
+        return 1;
+    }
+    return 0;
+}
+
+/* ECI 170 ASCII subset (ISO/IEC 646:1991 Invariant, excludes chars that historically had national variants) */
+static int ascii_invariant_wctosb(unsigned char *r, const unsigned int wc) {
+    if (wc == 0x7f || (wc <= 'z' && wc != '#' && wc != '$' && wc != '@' && (wc <= 'Z' || wc == '_' || wc >= 'a'))) {
+        *r = (unsigned char) wc;
+        return 1;
+    }
+    return 0;
+}
+
+/* ECI 28 Big5 Chinese (Taiwan) */
+static int big5_wctomb(unsigned char *r, const unsigned int wc) {
+    unsigned int c;
+
+    if (wc < 0x80) {
+        *r = (unsigned char) wc;
+        return 1;
+    }
+    if (big5_wctomb_zint(&c, wc)) {
+        r[0] = (unsigned char) (c >> 8);
+        r[1] = (unsigned char) (c & 0xff);
+        return 2;
+    }
+    return 0;
+}
+
+/* ECI 29 GB 2312 Chinese (PRC) */
+static int gb2312_wctomb(unsigned char *r, const unsigned int wc) {
+    unsigned int c;
+
+    if (wc < 0x80) {
+        *r = (unsigned char) wc;
+        return 1;
+    }
+    if (gb2312_wctomb_zint(&c, wc)) {
+        r[0] = (unsigned char) (c >> 8);
+        r[1] = (unsigned char) (c & 0xff);
+        return 2;
+    }
+    return 0;
+}
+
+/* ECI 30 KS X 1001 (KS C 5601) Korean */
+static int ksx1001_wctomb(unsigned char *r, const unsigned int wc) {
+    unsigned int c;
+
+    if (wc < 0x80) {
+        *r = (unsigned char) wc;
+        return 1;
+    }
+    if (ksx1001_wctomb_zint(&c, wc)) {
+        r[0] = (unsigned char) (c >> 8);
+        r[1] = (unsigned char) (c & 0xff);
+        return 2;
+    }
+    return 0;
+}
+
+/* Helper to count the number of chars in a string within a range */
+static int chr_range_cnt(const unsigned char string[], const int length, const unsigned char c1,
+            const unsigned char c2) {
+    int count = 0;
+    int i;
+    if (c1) {
+        for (i = 0; i < length; i++) {
+            if (string[i] >= c1 && string[i] <= c2) {
+                count++;
+            }
+        }
+    } else {
+        for (i = 0; i < length; i++) {
+            if (string[i] <= c2) {
+                count++;
+            }
+        }
+    }
+    return count;
+}
+
+/* Is ECI convertible from UTF-8? */
+INTERNAL int is_eci_convertible(const int eci) {
+    if (eci == 26 || (eci > 30 && eci != 170)) { /* Exclude ECI 170 - ASCII Invariant */
+        /* UTF-8 (26) or 8-bit binary data (899) or undefined (> 30 and < 899) or not character set (> 899) */
+        return 0;
+    }
+    return 1;
+}
+
+/* Calculate length required to convert UTF-8 to (double-byte) encoding */
+INTERNAL int get_eci_length(const int eci, const unsigned char source[], int length) {
+    if (eci == 20) { /* Shift JIS */
+        /* Only ASCII backslash (reverse solidus) exceeds UTF-8 length */
+        length += chr_cnt(source, length, '\\');
+
+    } else if (eci == 25) { /* UCS-2BE */
+        /* All ASCII chars take 2 bytes */
+        length += chr_range_cnt(source, length, 0, 0x7F);
+
+    } else if (eci == 29) { /* GB 2312 (and GB 18030 if Han Xin) */
+        /* Not needed for GB 2312 but allow for GB 18030 4 byters */
+        length *= 2;
+    }
+
+    /* Big5 and KS X 1001 fit in UTF-8 length */
+
+    return length;
+}
+
+/* Convert UTF-8 Unicode to other character encodings */
+INTERNAL int utf8_to_eci(const int eci, const unsigned char source[], unsigned char dest[], int *p_length) {
+
+    typedef int (*eci_func_t)(unsigned char *r, const unsigned int wc);
+    static const eci_func_t eci_funcs[31] = {
+                     NULL,              NULL,              NULL,              NULL,  iso8859_2_wctosb,
+         iso8859_3_wctosb,  iso8859_4_wctosb,  iso8859_5_wctosb,  iso8859_6_wctosb,  iso8859_7_wctosb,
+         iso8859_8_wctosb,  iso8859_9_wctosb, iso8859_10_wctosb, iso8859_11_wctosb,              NULL,
+        iso8859_13_wctosb, iso8859_14_wctosb, iso8859_15_wctosb, iso8859_16_wctosb,              NULL,
+              sjis_wctomb,     cp1250_wctosb,     cp1251_wctosb,     cp1252_wctosb,     cp1256_wctosb,
+            ucs2be_wctomb,              NULL,      ascii_wctosb,       big5_wctomb,     gb2312_wctomb,
+           ksx1001_wctomb,
+    };
+    eci_func_t eci_func;
+    unsigned int codepoint, state;
+    int in_posn;
+    int out_posn;
+    int length = *p_length;
 
     in_posn = 0;
     out_posn = 0;
+
+    /* Special case ISO/IEC 8859-1 */
+    if (eci == 0 || eci == 3) { /* Default ECI 0 to ISO/IEC 8859-1 */
+        state = 0;
+        while (in_posn < length) {
             do {
-        /* Single byte (ASCII) character */
+                decode_utf8(&state, &codepoint, source[in_posn++]);
-        int bytelen = 1;
+            } while (in_posn < length && state != 0 && state != 12);
-        int glyph = (int) source[in_posn];
+            if (state != 0) {
-
-        if ((source[in_posn] >= 0x80) && (source[in_posn] < 0xc0)) {
-            /* Something has gone wrong, abort */
                 return ZINT_ERROR_INVALID_DATA;
             }
-
+            if (codepoint >= 0x80 && (codepoint < 0x00a0 || codepoint >= 0x0100)) {
-        if ((source[in_posn] >= 0xc0) && (source[in_posn] < 0xe0)) {
-            /* Two-byte character */
-            bytelen = 2;
-            glyph = (source[in_posn] & 0x1f) << 6;
-
-            if (*length < (in_posn + 2)) {
                 return ZINT_ERROR_INVALID_DATA;
             }
-
+            dest[out_posn++] = (unsigned char) codepoint;
-            if (source[in_posn + 1] > 0xc0) {
-                return ZINT_ERROR_INVALID_DATA;
         }
-
-            glyph += (source[in_posn + 1] & 0x3f);
-        }
-
-        if ((source[in_posn] >= 0xe0) && (source[in_posn] < 0xf0)) {
-            /* Three-byte character */
-            bytelen = 3;
-            glyph = (source[in_posn] & 0x0f) << 12;
-
-            if (*length < (in_posn + 2)) {
-                return ZINT_ERROR_INVALID_DATA;
-            }
-
-            if (*length < (in_posn + 3)) {
-                return ZINT_ERROR_INVALID_DATA;
-            }
-
-            if (source[in_posn + 1] > 0xc0) {
-                return ZINT_ERROR_INVALID_DATA;
-            }
-
-            if (source[in_posn + 2] > 0xc0) {
-                return ZINT_ERROR_INVALID_DATA;
-            }
-
-            glyph += (source[in_posn + 1] & 0x3f) << 6;
-            glyph += (source[in_posn + 2] & 0x3f);
-        }
-
-        if (source[in_posn] >= 0xf0 || glyph > 0x2122) {
-            /* Not in any ISO 8859 or Windows page */
-            return ZINT_ERROR_INVALID_DATA;
-        }
-
-        if (glyph < 128) {
-            dest[out_posn] = glyph;
-        } else {
-            done = 0;
-            for (ext = 0; ext < 128; ext++) {
-                switch (eci) {
-                    case 3: // Latin-1
-                        if (glyph == iso_8859_1[ext]) {
-                            dest[out_posn] = ext + 128;
-                            done = 1;
-                        }
-                        break;
-                    case 4: // Latin-2
-                        if (glyph == iso_8859_2[ext]) {
-                            dest[out_posn] = ext + 128;
-                            done = 1;
-                        }
-                        break;
-                    case 5: // Latin-3
-                        if (glyph == iso_8859_3[ext]) {
-                            dest[out_posn] = ext + 128;
-                            done = 1;
-                        }
-                        break;
-                    case 6: // Latin-4
-                        if (glyph == iso_8859_4[ext]) {
-                            dest[out_posn] = ext + 128;
-                            done = 1;
-                        }
-                        break;
-                    case 7: // Latin/Cyrillic
-                        if (glyph == iso_8859_5[ext]) {
-                            dest[out_posn] = ext + 128;
-                            done = 1;
-                        }
-                        break;
-                    case 8: // Latin/Arabic
-                        if (glyph == iso_8859_6[ext]) {
-                            dest[out_posn] = ext + 128;
-                            done = 1;
-                        }
-                        break;
-                    case 9: // Latin/Greek
-                        if (glyph == iso_8859_7[ext]) {
-                            dest[out_posn] = ext + 128;
-                            done = 1;
-                        }
-                        break;
-                    case 10: // Latin/Hebrew
-                        if (glyph == iso_8859_8[ext]) {
-                            dest[out_posn] = ext + 128;
-                            done = 1;
-                        }
-                        break;
-                    case 11: // Latin-5
-                        if (glyph == iso_8859_9[ext]) {
-                            dest[out_posn] = ext + 128;
-                            done = 1;
-                        }
-                        break;
-                    case 12: // Latin-6
-                        if (glyph == iso_8859_10[ext]) {
-                            dest[out_posn] = ext + 128;
-                            done = 1;
-                        }
-                        break;
-                    case 13: // Latin/Thai
-                        if (glyph == iso_8859_11[ext]) {
-                            dest[out_posn] = ext + 128;
-                            done = 1;
-                        }
-                        break;
-                    case 15: // Latin-7
-                        if (glyph == iso_8859_13[ext]) {
-                            dest[out_posn] = ext + 128;
-                            done = 1;
-                        }
-                        break;
-                    case 16: // Latin-8
-                        if (glyph == iso_8859_14[ext]) {
-                            dest[out_posn] = ext + 128;
-                            done = 1;
-                        }
-                        break;
-                    case 17: // Latin-9
-                        if (glyph == iso_8859_15[ext]) {
-                            dest[out_posn] = ext + 128;
-                            done = 1;
-                        }
-                        break;
-                    case 18: // Latin-10
-                        if (glyph == iso_8859_16[ext]) {
-                            dest[out_posn] = ext + 128;
-                            done = 1;
-                        }
-                        break;
-                    case 21: // Windows-1250
-                        if (glyph == windows_1250[ext]) {
-                            dest[out_posn] = ext + 128;
-                            done = 1;
-                        }
-                        break;
-                    case 22: // Windows-1251
-                        if (glyph == windows_1251[ext]) {
-                            dest[out_posn] = ext + 128;
-                            done = 1;
-                        }
-                        break;
-                    case 23: // Windows-1252
-                        if (glyph == windows_1252[ext]) {
-                            dest[out_posn] = ext + 128;
-                            done = 1;
-                        }
-                        break;
-                    case 24: // Windows-1256
-                        if (glyph == windows_1256[ext]) {
-                            dest[out_posn] = ext + 128;
-                            done = 1;
-                        }
-                        break;
-                    default:
-                        break;
-                }
-                if (done) {
-                    break;
-                }
-            }
-
-            if (!(done)) {
-                return ZINT_ERROR_INVALID_DATA;
-            }
-        }
-
-        in_posn += bytelen;
-        out_posn++;
-    } while (in_posn < *length);
         dest[out_posn] = '\0';
-    *length = out_posn;
+        *p_length = out_posn;
+        return 0;
+    }
+
+    if (eci == 170) { /* ASCII Invariant (archaic subset) */
+        eci_func = ascii_invariant_wctosb;
+    } else {
+        eci_func = eci_funcs[eci];
+        if (eci_func == NULL) {
+            return ZINT_ERROR_INVALID_DATA;
+        }
+    }
+
+    state = 0;
+    while (in_posn < length) {
+        int incr;
+        do {
+            decode_utf8(&state, &codepoint, source[in_posn++]);
+        } while (in_posn < length && state != 0 && state != 12);
+        if (state != 0) {
+            return ZINT_ERROR_INVALID_DATA;
+        }
+        incr = (*eci_func)(dest + out_posn, codepoint);
+        if (incr == 0) {
+            return ZINT_ERROR_INVALID_DATA;
+        }
+        out_posn += incr;
+    }
+    dest[out_posn] = '\0';
+    *p_length = out_posn;
 
     return 0;
 }
 
-/* Find the lowest ECI mode which will encode a given set of Unicode text */
+/* Find the lowest single-byte ECI mode which will encode a given set of Unicode text */
-INTERNAL int get_best_eci(unsigned char source[], int length) {
+INTERNAL int get_best_eci(const unsigned char source[], int length) {
     int eci = 3;
 
+    /* Note: attempting single-byte conversions only, so get_eci_length() unnecessary */
 #ifndef _MSC_VER
     unsigned char local_source[length + 1];
 #else
-    unsigned char *local_source = (unsigned char*) _alloca(length + 1);
+    unsigned char *local_source = (unsigned char *) _alloca(length + 1);
 #endif
 
     do {
-        if (utf_to_eci(eci, source, local_source, &length) == 0) {
+        if (eci == 14) { /* Reserved */
+            eci = 15;
+        } else if (eci == 19) { /* Reserved */
+            eci = 21; /* Skip 20 Shift JIS */
+        }
+        if (utf8_to_eci(eci, source, local_source, &length) == 0) {
             return eci;
         }
         eci++;

backend/eci.h

@@ -1,7 +1,7 @@
 /*  eci.c - Extended Channel Interpretations to Unicode tables
 
     libzint - the open source barcode library
-    Copyright (C) 2009-2017 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2009-2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -28,6 +28,7 @@
     OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     SUCH DAMAGE.
  */
+/* vim: set ts=4 sw=4 et : */
 
 #ifndef ECI_H
 #define ECI_H
@@ -36,214 +37,10 @@
 extern "C" {
 #endif
 
-static const unsigned short int iso_8859_1[] = {// Latin alphabet No. 1
+INTERNAL int is_eci_convertible(const int eci);
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+INTERNAL int get_eci_length(const int eci, const unsigned char source[], int length);
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+INTERNAL int utf8_to_eci(const int eci, const unsigned char source[], unsigned char dest[], int *p_length);
-    0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
+INTERNAL int get_best_eci(const unsigned char source[], int length);
-    0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf,
-    0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
-    0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df,
-    0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
-    0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff
-};
-
-static const unsigned short int iso_8859_2[] = {// Latin alphabet No. 2
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x00a0, 0x0104, 0x02d8, 0x0141, 0x00a4, 0x013d, 0x015a, 0x00a7, 0x00a8, 0x0160, 0x015e, 0x0164, 0x0179, 0x00ad, 0x017d, 0x017b,
-    0x00b0, 0x0105, 0x02db, 0x0142, 0x00b4, 0x013e, 0x015b, 0x02c7, 0x00b8, 0x0161, 0x015f, 0x0165, 0x017a, 0x02dd, 0x017e, 0x017c,
-    0x0154, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0139, 0x0106, 0x00c7, 0x010c, 0x00c9, 0x0118, 0x00cb, 0x011a, 0x00cd, 0x00ce, 0x010e,
-    0x0110, 0x0143, 0x0147, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x00d7, 0x0158, 0x016e, 0x00da, 0x0170, 0x00dc, 0x00dd, 0x0162, 0x00df,
-    0x0155, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x013a, 0x0107, 0x00e7, 0x010d, 0x00e9, 0x0119, 0x00eb, 0x011b, 0x00ed, 0x00ee, 0x010f,
-    0x0111, 0x0144, 0x0148, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x00f7, 0x0159, 0x016f, 0x00fa, 0x0171, 0x00fc, 0x00fd, 0x0163, 0x02d9
-};
-
-static const unsigned short int iso_8859_3[] = {// Latin alphabet No. 3
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x00a0, 0x0126, 0x02d8, 0x00a3, 0x00a4, 0x0000, 0x0124, 0x00a7, 0x00a8, 0x0130, 0x015e, 0x011e, 0x0134, 0x00ad, 0x0000, 0x017b,
-    0x00b0, 0x0127, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x0125, 0x00b7, 0x00b8, 0x0131, 0x015f, 0x011f, 0x0135, 0x00bd, 0x0000, 0x017c,
-    0x00c0, 0x00c1, 0x00c2, 0x0000, 0x00c4, 0x010a, 0x0108, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
-    0x0000, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x0120, 0x00d6, 0x00d7, 0x011c, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x016c, 0x015c, 0x00df,
-    0x00e0, 0x00e1, 0x00e2, 0x0000, 0x00e4, 0x010b, 0x0109, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
-    0x0000, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x0121, 0x00f6, 0x00f7, 0x011d, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x016d, 0x015d, 0x02d9
-};
-
-static const unsigned short int iso_8859_4[] = {// Latin alphabet No. 4
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x00a0, 0x0104, 0x0138, 0x0156, 0x00a4, 0x012b, 0x013b, 0x00a7, 0x00a8, 0x0160, 0x0112, 0x0122, 0x0166, 0x00ad, 0x017d, 0x00af,
-    0x00b0, 0x0105, 0x02db, 0x0157, 0x00b4, 0x0129, 0x013c, 0x02c7, 0x00b8, 0x0161, 0x0113, 0x0123, 0x0167, 0x014a, 0x017e, 0x014b,
-    0x0100, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x012e, 0x010c, 0x00c9, 0x0118, 0x00cb, 0x0116, 0x00cd, 0x00ce, 0x012a,
-    0x0110, 0x0145, 0x014c, 0x0136, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x0172, 0x00da, 0x00db, 0x00dc, 0x0168, 0x016a, 0x00df,
-    0x0101, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x012f, 0x010d, 0x00e9, 0x0119, 0x00eb, 0x0117, 0x00ed, 0x00ee, 0x012b,
-    0x0111, 0x0146, 0x014d, 0x0137, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x0173, 0x00fa, 0x00fb, 0x00fc, 0x0169, 0x016b, 0x02d9
-};
-
-static const unsigned short int iso_8859_5[] = {// Latin/Cyrillic alphabet
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x00a0, 0x0401, 0x0402, 0x0403, 0x0404, 0x0405, 0x0406, 0x0407, 0x0408, 0x0409, 0x040a, 0x040b, 0x040c, 0x00ad, 0x040e, 0x040f,
-    0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f,
-    0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f,
-    0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f,
-    0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f,
-    0x2116, 0x0451, 0x0452, 0x0453, 0x0454, 0x0455, 0x0456, 0x0457, 0x0458, 0x0459, 0x045a, 0x045b, 0x045c, 0x00a7, 0x045e, 0x045f
-};
-
-static const unsigned short int iso_8859_6[] = {// Latin/Arabic alphabet
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x00a0, 0x0000, 0x0000, 0x0000, 0x00a4, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x060c, 0x00ad, 0x0000, 0x0000,
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x061b, 0x0000, 0x0000, 0x0000, 0x061f,
-    0x0000, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, 0x0628, 0x0629, 0x062a, 0x062b, 0x062c, 0x062d, 0x062e, 0x062f,
-    0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x0637, 0x0638, 0x0639, 0x063a, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x0640, 0x0641, 0x0642, 0x0643, 0x0644, 0x0645, 0x0646, 0x0647, 0x0648, 0x0649, 0x064a, 0x064b, 0x064c, 0x064d, 0x064e, 0x064f,
-    0x0650, 0x0651, 0x0652, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000
-};
-
-static const unsigned short int iso_8859_7[] = {// Latin/Greek alphabet
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x00a0, 0x2018, 0x2019, 0x00a3, 0x20ac, 0x20af, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x037a, 0x00ab, 0x00ac, 0x00ad, 0x0000, 0x2015,
-    0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x0384, 0x0385, 0x0386, 0x00b7, 0x0388, 0x0389, 0x038a, 0x00bb, 0x038c, 0x00bd, 0x038e, 0x038f,
-    0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, 0x0398, 0x0399, 0x039a, 0x039b, 0x039c, 0x039d, 0x039e, 0x039f,
-    0x03a0, 0x03a1, 0x0000, 0x03a3, 0x03a4, 0x03a5, 0x03a6, 0x03a7, 0x03a8, 0x03a9, 0x03aa, 0x03ab, 0x03ac, 0x03ad, 0x03ae, 0x03af,
-    0x03b0, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf,
-    0x03c0, 0x03c1, 0x03c2, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, 0x03c8, 0x03c9, 0x03ca, 0x03cb, 0x03cc, 0x03cd, 0x03ce, 0x0000
-};
-
-static const unsigned short int iso_8859_8[] = {// Latin/Hebrew alphabet
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x00a0, 0x0000, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00d7, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
-    0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00f7, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x0000,
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2017,
-    0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x05d5, 0x05d6, 0x05d7, 0x05d8, 0x05d9, 0x05da, 0x05db, 0x05dc, 0x05dd, 0x05de, 0x05df,
-    0x05e0, 0x05e1, 0x05e2, 0x05e3, 0x05e4, 0x05e5, 0x05e6, 0x05e7, 0x05e8, 0x05e9, 0x05ea, 0x0000, 0x0000, 0x200e, 0x200f, 0x0000
-};
-
-static const unsigned short int iso_8859_9[] = {// Latin alphabet No. 5
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
-    0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf,
-    0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
-    0x011e, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x0130, 0x015e, 0x00df,
-    0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
-    0x011f, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x0131, 0x015f, 0x00ff
-};
-
-static const unsigned short int iso_8859_10[] = {// Latin alphabet No. 6
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x00a0, 0x0104, 0x0112, 0x0122, 0x012a, 0x012b, 0x0136, 0x00a7, 0x013b, 0x0110, 0x0160, 0x0166, 0x017d, 0x00ad, 0x016a, 0x014a,
-    0x00b0, 0x0105, 0x0113, 0x0123, 0x012b, 0x0129, 0x0137, 0x00b7, 0x013c, 0x0111, 0x0161, 0x0167, 0x017e, 0x2015, 0x016b, 0x014b,
-    0x0100, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x012e, 0x010c, 0x00c9, 0x0118, 0x00cb, 0x0116, 0x00cd, 0x00ce, 0x00cf,
-    0x00d0, 0x0145, 0x014c, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x0168, 0x00d8, 0x0172, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df,
-    0x0101, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x012f, 0x010d, 0x00e9, 0x0119, 0x00eb, 0x0117, 0x00ed, 0x00ee, 0x00ef,
-    0x00f0, 0x0146, 0x014d, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x0169, 0x00f8, 0x0173, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x0138
-};
-
-static const unsigned short int iso_8859_11[] = {// Latin/Thai alphabet
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x00a0, 0x0e01, 0x0e02, 0x0e03, 0x0e04, 0x0e05, 0x0e06, 0x0e07, 0x0e08, 0x0e09, 0x0e0a, 0x0e0b, 0x0e0c, 0x0e0d, 0x0e0e, 0x0e0f,
-    0x0e10, 0x0e11, 0x0e12, 0x0e13, 0x0e14, 0x0e15, 0x0e16, 0x0e17, 0x0e18, 0x0e19, 0x0e1a, 0x0e1b, 0x0e1c, 0x0e1d, 0x0e1e, 0x0e1f,
-    0x0e20, 0x0e21, 0x0e22, 0x0e23, 0x0e24, 0x0e25, 0x0e26, 0x0e27, 0x0e28, 0x0e29, 0x0e2a, 0x0e2b, 0x0e2c, 0x0e2d, 0x0e2e, 0x0e2f,
-    0x0e30, 0x0e31, 0x0e32, 0x0e33, 0x0e34, 0x0e36, 0x0e36, 0x0e37, 0x0e38, 0x0e39, 0x0e3a, 0x0000, 0x0000, 0x0000, 0x0000, 0x0e3f,
-    0x0e40, 0x0e41, 0x0e42, 0x0e43, 0x0e44, 0x0e45, 0x0e46, 0x0e47, 0x0e48, 0x0e49, 0x0e4a, 0x0e4b, 0x0e4c, 0x0e4d, 0x0e4e, 0x0e4f,
-    0x0e50, 0x0e51, 0x0e52, 0x0e53, 0x0e54, 0x0e55, 0x0e56, 0x0e57, 0x0e58, 0x0e59, 0x0e5a, 0x0e5b, 0x0000, 0x0000, 0x0000, 0x0000
-};
-
-static const unsigned short int iso_8859_13[] = {// Latin alphabet No. 7
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x00a0, 0x201d, 0x00a2, 0x00a3, 0x00a4, 0x201e, 0x00a6, 0x00a7, 0x00d8, 0x00a9, 0x0156, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00c6,
-    0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x201c, 0x00b5, 0x00b6, 0x00b7, 0x00f8, 0x00b9, 0x0157, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00e6,
-    0x0104, 0x012e, 0x0100, 0x0106, 0x00c4, 0x00c5, 0x0118, 0x0112, 0x010c, 0x00c9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012a, 0x013b,
-    0x0160, 0x0143, 0x0145, 0x00d3, 0x014c, 0x00d5, 0x00d6, 0x00d7, 0x0172, 0x0141, 0x015a, 0x016a, 0x00dc, 0x017b, 0x017d, 0x00df,
-    0x0105, 0x012f, 0x0101, 0x0107, 0x00e4, 0x00e5, 0x0119, 0x0113, 0x010d, 0x00e9, 0x017a, 0x0117, 0x0123, 0x0137, 0x012b, 0x013c,
-    0x0161, 0x0144, 0x0146, 0x00f3, 0x014d, 0x00f5, 0x00f6, 0x00f7, 0x0173, 0x0142, 0x015b, 0x016b, 0x00fc, 0x017c, 0x017e, 0x2019
-};
-
-static const unsigned short int iso_8859_14[] = {// Latin alphabet No. 8 (Celtic)
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x00a0, 0x1e02, 0x1e03, 0x00a3, 0x010a, 0x010b, 0x1e0a, 0x00a7, 0x1e80, 0x00a9, 0x1e82, 0x1e0b, 0x1ef2, 0x00ad, 0x00ae, 0x0178,
-    0x1e1e, 0x1e1f, 0x0120, 0x0121, 0x1e40, 0x1e41, 0x00b6, 0x1e56, 0x1e81, 0x1e57, 0x1e83, 0x1e60, 0x1ef3, 0x1e84, 0x1e85, 0x1e61,
-    0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
-    0x0174, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x1e6a, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x0176, 0x00df,
-    0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
-    0x0175, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x1e6b, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x0177, 0x00ff
-};
-
-static const unsigned short int iso_8859_15[] = {// Latin alphabet No. 9
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x20ac, 0x00a5, 0x0160, 0x00a7, 0x0161, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
-    0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x017d, 0x00b5, 0x00b6, 0x00b7, 0x017e, 0x00b9, 0x00ba, 0x00bb, 0x0152, 0x0153, 0x0178, 0x00bf,
-    0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
-    0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df,
-    0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
-    0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff
-};
-
-static const unsigned short int iso_8859_16[] = {// Latin alphabet No. 10
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
-    0x00a0, 0x0104, 0x0105, 0x0141, 0x20ac, 0x201e, 0x0160, 0x00a7, 0x0161, 0x00a9, 0x0218, 0x00ab, 0x0179, 0x00ad, 0x017a, 0x017b,
-    0x00b0, 0x00b1, 0x010c, 0x0142, 0x017d, 0x201d, 0x00b6, 0x00b7, 0x017e, 0x010d, 0x0219, 0x00bb, 0x0152, 0x0153, 0x0178, 0x017c,
-    0x00c0, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0106, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
-    0x0110, 0x0143, 0x00d2, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x015a, 0x0170, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x0118, 0x021a, 0x00df,
-    0x00e0, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x0107, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
-    0x0111, 0x0144, 0x00f2, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x015b, 0x0171, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x0119, 0x021b, 0x00ff
-};
-
-static const unsigned short int windows_1250[] = {
-    0x20ac, 0x0000, 0x201a, 0x0000, 0x201e, 0x2026, 0x2020, 0x2021, 0x0000, 0x2030, 0x0160, 0x2039, 0x015a, 0x0164, 0x017d, 0x0179,
-    0x0000, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x0000, 0x2122, 0x0161, 0x203a, 0x015b, 0x0165, 0x017e, 0x017a,
-    0x00a0, 0x02c7, 0x02db, 0x0141, 0x00a4, 0x0104, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x015e, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x017b,
-    0x00b0, 0x00b1, 0x02db, 0x0142, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x0105, 0x015f, 0x00bb, 0x013d, 0x02dd, 0x013e, 0x017c,
-    0x0154, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0139, 0x0106, 0x00c7, 0x010c, 0x00c9, 0x0118, 0x00cb, 0x011a, 0x00cd, 0x00ce, 0x010e,
-    0x0110, 0x0143, 0x0147, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x00d7, 0x0158, 0x016e, 0x00da, 0x0170, 0x00dc, 0x00dd, 0x0162, 0x00df,
-    0x0155, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x013a, 0x0107, 0x00e7, 0x010d, 0x00e9, 0x0119, 0x00eb, 0x011b, 0x00ed, 0x00ee, 0x010f,
-    0x0111, 0x0144, 0x0148, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x00f7, 0x0159, 0x016f, 0x00fa, 0x0171, 0x00fc, 0x00fd, 0x0163, 0x02d9
-};
-
-static const unsigned short int windows_1251[] = {
-    0x0402, 0x0403, 0x201a, 0x0453, 0x201e, 0x2026, 0x2020, 0x2021, 0x20ac, 0x2030, 0x0409, 0x2039, 0x040a, 0x040c, 0x040b, 0x040f,
-    0x0452, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x0000, 0x2122, 0x0459, 0x203a, 0x045a, 0x045c, 0x045b, 0x045f,
-    0x00a0, 0x040e, 0x045e, 0x0408, 0x00a4, 0x0490, 0x00a6, 0x00a7, 0x0401, 0x00a9, 0x0404, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x0407,
-    0x00b0, 0x00b1, 0x0406, 0x0456, 0x0491, 0x00b5, 0x00b6, 0x00b7, 0x0451, 0x2116, 0x0454, 0x00bb, 0x0458, 0x0405, 0x0455, 0x0457,
-    0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f,
-    0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f,
-    0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f,
-    0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f
-};
-
-static const unsigned short int windows_1252[] = {
-    0x20ac, 0x0000, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0x02c6, 0x2030, 0x0160, 0x2039, 0x0152, 0x0000, 0x017d, 0x0000,
-    0x0000, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, 0x0000, 0x017e, 0x0178,
-    0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
-    0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf,
-    0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
-    0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df,
-    0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
-    0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff
-};
-
-static const unsigned short int windows_1256[] = {
-    0x20ac, 0x067e, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0x02c6, 0x2030, 0x0679, 0x2039, 0x0152, 0x0686, 0x0698, 0x0688,
-    0x06af, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x06a9, 0x2122, 0x0691, 0x203a, 0x0153, 0x200c, 0x200d, 0x06ba,
-    0x00a0, 0x060c, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x06be, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
-    0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x061b, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x061f,
-    0x06c1, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, 0x0628, 0x0629, 0x062a, 0x062b, 0x062c, 0x062d, 0x062e, 0x062f,
-    0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x00d7, 0x0637, 0x0638, 0x0639, 0x063a, 0x0640, 0x0641, 0x0642, 0x0643,
-    0x00e0, 0x0644, 0x00e2, 0x0645, 0x0646, 0x0647, 0x0648, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x0649, 0x064a, 0x00ee, 0x00ef,
-    0x064b, 0x064c, 0x064d, 0x064e, 0x00f4, 0x064f, 0x0650, 0x00f7, 0x0651, 0x00f9, 0x0652, 0x00fb, 0x00fc, 0x200e, 0x200f, 0x06d2
-};
 
 #ifdef __cplusplus
 }

backend/gb18030.c

@@ -1,6 +1,6 @@
 /*
     libzint - the open source barcode library
-    Copyright (C) 2008-2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2019-2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -58,9 +58,7 @@
 #include "common.h"
 #include "gb2312.h"
 #include "gb18030.h"
-
+#include "eci.h"
-/* Convert Unicode to other encodings */
-INTERNAL int utf_to_eci(const int eci, const unsigned char source[], unsigned char dest[], int *length);
 
 /*
  * CP936 extensions (libiconv-1.16/lib/cp936ext.h)
@@ -2870,7 +2868,7 @@ INTERNAL int gb18030_wctomb_zint(unsigned int *r1, unsigned int *r2, const unsig
 }
 
 /* Convert UTF-8 string to GB 18030 and place in array of ints */
-INTERNAL int gb18030_utf8tomb(struct zint_symbol *symbol, const unsigned char source[], int *p_length,
+INTERNAL int gb18030_utf8(struct zint_symbol *symbol, const unsigned char source[], int *p_length,
                 unsigned int *gbdata) {
     int error_number, ret;
     unsigned int i, j, length;
@@ -2905,23 +2903,29 @@ INTERNAL int gb18030_utf8tomb(struct zint_symbol *symbol, const unsigned char so
     return 0;
 }
 
-/* Convert UTF-8 string to single byte ECI and place in array of ints */
+/* Convert UTF-8 string to ECI and place in array of ints */
-INTERNAL int gb18030_utf8tosb(const int eci, const unsigned char source[], int *p_length, unsigned int *gbdata,
+INTERNAL int gb18030_utf8_to_eci(const int eci, const unsigned char source[], int *p_length, unsigned int *gbdata,
                 const int full_multibyte) {
+
+    if (is_eci_convertible(eci)) {
         int error_number;
+        int eci_length = get_eci_length(eci, source, *p_length);
 #ifndef _MSC_VER
-    unsigned char single_byte[*p_length + 1];
+        unsigned char converted[eci_length + 1];
 #else
-    unsigned char *single_byte = (unsigned char *) _alloca(*p_length + 1);
+        unsigned char *converted = (unsigned char *) _alloca(eci_length + 1);
 #endif
 
-    error_number = utf_to_eci(eci, source, single_byte, p_length);
+        error_number = utf8_to_eci(eci, source, converted, p_length);
         if (error_number != 0) {
             /* Note not setting `symbol->errtxt`, up to caller */
             return error_number;
         }
 
-    gb18030_cpy(single_byte, p_length, gbdata, full_multibyte);
+        gb18030_cpy(converted, p_length, gbdata, full_multibyte);
+    } else {
+        gb18030_cpy(source, p_length, gbdata, full_multibyte);
+    }
 
     return 0;
 }

backend/gb18030.h

@@ -1,7 +1,7 @@
 /*  gb18030.h - Unicode to GB 18030
 
     libzint - the open source barcode library
-    Copyright (C) 2009-2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2009-2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -38,9 +38,9 @@ extern "C" {
 #endif /* __cplusplus */
 
 INTERNAL int gb18030_wctomb_zint(unsigned int *r1, unsigned int *r2, const unsigned int wc);
-INTERNAL int gb18030_utf8tomb(struct zint_symbol *symbol, const unsigned char source[], int *p_length,
+INTERNAL int gb18030_utf8(struct zint_symbol *symbol, const unsigned char source[], int *p_length,
                 unsigned int *gbdata);
-INTERNAL int gb18030_utf8tosb(const int eci, const unsigned char source[], int *p_length, unsigned int *gbdata,
+INTERNAL int gb18030_utf8_to_eci(const int eci, const unsigned char source[], int *p_length, unsigned int *gbdata,
                 const int full_multibyte);
 INTERNAL void gb18030_cpy(const unsigned char source[], int *p_length, unsigned int *gbdata,
                 const int full_multibyte);

backend/gb2312.c

@@ -1,6 +1,6 @@
 /*
     libzint - the open source barcode library
-    Copyright (C) 2008-2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2019-2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -57,9 +57,7 @@
 #endif
 #include "common.h"
 #include "gb2312.h"
-
+#include "eci.h"
-/* Convert Unicode to other encodings */
-INTERNAL int utf_to_eci(const int eci, const unsigned char source[], unsigned char dest[], int *length);
 
 /*
  * GB2312.1980-0 (libiconv-1.16/lib/gb2312.h)
@@ -1544,7 +1542,7 @@ INTERNAL int gb2312_wctomb_zint(unsigned int *r, const unsigned int wc) {
 }
 
 /* Convert UTF-8 string to GB 2312 (EUC-CN) and place in array of ints */
-INTERNAL int gb2312_utf8tomb(struct zint_symbol *symbol, const unsigned char source[], int *p_length,
+INTERNAL int gb2312_utf8(struct zint_symbol *symbol, const unsigned char source[], int *p_length,
                 unsigned int *gbdata) {
     int error_number;
     unsigned int i, length;
@@ -1573,23 +1571,29 @@ INTERNAL int gb2312_utf8tomb(struct zint_symbol *symbol, const unsigned char sou
     return 0;
 }
 
-/* Convert UTF-8 string to single byte ECI and place in array of ints */
+/* Convert UTF-8 string to ECI and place in array of ints */
-INTERNAL int gb2312_utf8tosb(const int eci, const unsigned char source[], int *p_length, unsigned int *gbdata,
+INTERNAL int gb2312_utf8_to_eci(const int eci, const unsigned char source[], int *p_length, unsigned int *gbdata,
                 const int full_multibyte) {
+
+    if (is_eci_convertible(eci)) {
         int error_number;
+        int eci_length = get_eci_length(eci, source, *p_length);
 #ifndef _MSC_VER
-    unsigned char single_byte[*p_length + 1];
+        unsigned char converted[eci_length + 1];
 #else
-    unsigned char *single_byte = (unsigned char *) _alloca(*p_length + 1);
+        unsigned char *converted = (unsigned char *) _alloca(eci_length + 1);
 #endif
 
-    error_number = utf_to_eci(eci, source, single_byte, p_length);
+        error_number = utf8_to_eci(eci, source, converted, p_length);
         if (error_number != 0) {
             /* Note not setting `symbol->errtxt`, up to caller */
             return error_number;
         }
 
-    gb2312_cpy(single_byte, p_length, gbdata, full_multibyte);
+        gb2312_cpy(converted, p_length, gbdata, full_multibyte);
+    } else {
+        gb2312_cpy(source, p_length, gbdata, full_multibyte);
+    }
 
     return 0;
 }

backend/gb2312.h

@@ -1,7 +1,7 @@
 /*  gb2312.h - Unicode to GB 2312-1980 (EUC-CN)
 
     libzint - the open source barcode library
-    Copyright (C) 2009-2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2009-2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -38,9 +38,9 @@ extern "C" {
 #endif /* __cplusplus */
 
 INTERNAL int gb2312_wctomb_zint(unsigned int *r, const unsigned int wc);
-INTERNAL int gb2312_utf8tomb(struct zint_symbol *symbol, const unsigned char source[], int *p_length,
+INTERNAL int gb2312_utf8(struct zint_symbol *symbol, const unsigned char source[], int *p_length,
                 unsigned int *gbdata);
-INTERNAL int gb2312_utf8tosb(const int eci, const unsigned char source[], int *p_length, unsigned int *gbdata,
+INTERNAL int gb2312_utf8_to_eci(const int eci, const unsigned char source[], int *p_length, unsigned int *gbdata,
                 const int full_multibyte);
 INTERNAL void gb2312_cpy(const unsigned char source[], int *p_length, unsigned int *gbdata,
                 const int full_multibyte);

backend/gridmtx.c

@@ -1,7 +1,7 @@
 /*  gridmtx.c - Grid Matrix
 
     libzint - the open source barcode library
-    Copyright (C) 2009-2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2009-2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -41,6 +41,7 @@
 #include "reedsol.h"
 #include "gridmtx.h"
 #include "gb2312.h"
+#include "eci.h"
 
 /* define_mode() stuff */
 
@@ -123,7 +124,7 @@ static int in_numeral(const unsigned int gbdata[], const int length, const int i
 
 /* Calculate optimized encoding modes. Adapted from Project Nayuki */
 /* Copyright (c) Project Nayuki. (MIT License) See qr.c for detailed notice */
-static void define_mode(char* mode, const unsigned int gbdata[], const int length, const int debug) {
+static void define_mode(char *mode, const unsigned int gbdata[], const int length, const int debug) {
     /* Must be in same order as GM_H etc */
     static const char mode_types[] = { GM_CHINESE, GM_NUMBER, GM_LOWER, GM_UPPER, GM_MIXED, GM_BYTE, '\0' };
 
@@ -340,7 +341,7 @@ static int gm_encode(unsigned int gbdata[], const int length, char binary[], con
 #ifndef _MSC_VER
     char mode[length];
 #else
-    char* mode = (char*) _alloca(length);
+    char *mode = (char *) _alloca(length);
 #endif
 
     *binary = '\0';
@@ -1007,15 +1008,16 @@ INTERNAL int grid_matrix(struct zint_symbol *symbol, unsigned char source[], int
     int data_max, reader = 0;
     int size_squared;
     int bin_len;
+    int eci_length = get_eci_length(symbol->eci, source, length);
 
 #ifndef _MSC_VER
-    unsigned int gbdata[length + 1];
+    unsigned int gbdata[eci_length + 1];
 #else
-    char* grid;
+    char *grid;
-    unsigned int* gbdata = (unsigned int *) _alloca((length + 1) * sizeof (unsigned int));
+    unsigned int *gbdata = (unsigned int *) _alloca((eci_length + 1) * sizeof(unsigned int));
 #endif
 
-    /* If ZINT_FULL_MULTIBYTE set use Hanzi mode in DATA_MODE or for single-byte Latin */
+    /* If ZINT_FULL_MULTIBYTE set use Hanzi mode in DATA_MODE or for non-GB 2312 in UNICODE_MODE */
     full_multibyte = (symbol->option_3 & 0xFF) == ZINT_FULL_MULTIBYTE;
 
     if ((symbol->input_mode & 0x07) == DATA_MODE) {
@@ -1023,19 +1025,18 @@ INTERNAL int grid_matrix(struct zint_symbol *symbol, unsigned char source[], int
     } else {
         int done = 0;
         if (symbol->eci != 29) { /* Unless ECI 29 (GB) */
-            /* Try single byte (Latin) conversion first */
+            /* Try other conversions (ECI 0 defaults to ISO/IEC 8859-1) */
-            error_number = gb2312_utf8tosb(symbol->eci && symbol->eci <= 899 ? symbol->eci : 3, source, &length,
+            error_number = gb2312_utf8_to_eci(symbol->eci, source, &length, gbdata, full_multibyte);
-                                gbdata, full_multibyte);
             if (error_number == 0) {
                 done = 1;
-            } else if (symbol->eci && symbol->eci <= 899) {
+            } else if (symbol->eci) {
                 strcpy(symbol->errtxt, "575: Invalid characters in input data");
                 return error_number;
             }
         }
         if (!done) {
             /* Try GB 2312 (EUC-CN) */
-            error_number = gb2312_utf8tomb(symbol, source, &length, gbdata);
+            error_number = gb2312_utf8(symbol, source, &length, gbdata);
             if (error_number != 0) {
                 return error_number;
             }
@@ -1148,7 +1149,7 @@ INTERNAL int grid_matrix(struct zint_symbol *symbol, unsigned char source[], int
 #ifndef _MSC_VER
     char grid[size_squared];
 #else
-    grid = (char *) _alloca(size_squared * sizeof(char));
+    grid = (char *) _alloca(size_squared);
 #endif
 
     memset(grid, '0', size_squared);

backend/gs1.c

@@ -2,7 +2,7 @@
 
 /*
     libzint - the open source barcode library
-    Copyright (C) 2009 - 2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2009 - 2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -68,18 +68,6 @@ static void itostr(char ai_string[], int ai_value) {
     strcat(ai_string, ")");
 }
 
-/* Returns the number of times a character occurs in a string */
-static int ustrchr_cnt(const unsigned char string[], const int length, const unsigned char c) {
-    int count = 0;
-    int i;
-    for (i = 0; i < length; i++) {
-        if (string[i] == c) {
-            count++;
-        }
-    }
-    return count;
-}
-
 INTERNAL int gs1_verify(struct zint_symbol *symbol, const unsigned char source[], const int src_len,
                 unsigned char reduced[]) {
     int i, j, last_ai, ai_latch;
@@ -94,7 +82,7 @@ INTERNAL int gs1_verify(struct zint_symbol *symbol, const unsigned char source[]
     int *data_location;
     int *data_length;
 #endif
-    int ai_max = ustrchr_cnt(source, src_len, '[') + 1; /* Plus 1 so non-zero */
+    int ai_max = chr_cnt(source, src_len, '[') + 1; /* Plus 1 so non-zero */
 #ifndef _MSC_VER
     int ai_value[ai_max], ai_location[ai_max], data_location[ai_max], data_length[ai_max];
 #else

backend/hanxin.c

@@ -1,7 +1,7 @@
 /*  hanxin.c - Han Xin Code
 
     libzint - the open source barcode library
-    Copyright (C) 2009-2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2009-2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -42,7 +42,8 @@
 #include "hanxin.h"
 #include "gb2312.h"
 #include "gb18030.h"
-#include "assert.h"
+#include "eci.h"
+#include <assert.h>
 
 /* Find which submode to use for a text character */
 static int getsubmode(const unsigned int input) {
@@ -1450,21 +1451,22 @@ INTERNAL int han_xin(struct zint_symbol *symbol, unsigned char source[], int len
     int size_squared;
     int codewords;
     int bin_len;
+    int eci_length = get_eci_length(symbol->eci, source, length);
 
 #ifndef _MSC_VER
-    unsigned int gbdata[(length + 1) * 2];
+    unsigned int gbdata[eci_length + 1];
-    char mode[length];
+    char mode[eci_length];
 #else
-    unsigned int* gbdata = (unsigned int *) _alloca(((length + 1) * 2) * sizeof (unsigned int));
+    unsigned int *gbdata = (unsigned int *) _alloca((eci_length + 1) * sizeof(unsigned int));
-    char *mode = (char *) _alloca(length);
+    char *mode = (char *) _alloca(eci_length);
-    char* binary;
+    char *binary;
     unsigned char *datastream;
     unsigned char *fullstream;
     unsigned char *picket_fence;
     unsigned char *grid;
 #endif
 
-    /* If ZINT_FULL_MULTIBYTE set use Hanzi mode in DATA_MODE or for single-byte Latin */
+    /* If ZINT_FULL_MULTIBYTE set use Hanzi mode in DATA_MODE or for non-GB 18030 in UNICODE_MODE */
     full_multibyte = (symbol->option_3 & 0xFF) == ZINT_FULL_MULTIBYTE;
     user_mask = (symbol->option_3 >> 8) & 0x0F; /* User mask is pattern + 1, so >= 1 and <= 4 */
     if (user_mask > 4) {
@@ -1476,19 +1478,18 @@ INTERNAL int han_xin(struct zint_symbol *symbol, unsigned char source[], int len
     } else {
         int done = 0;
         if (symbol->eci != 29) { /* Unless ECI 29 (GB) */
-            /* Try single byte (Latin) conversion first */
+            /* Try other conversions (ECI 0 defaults to ISO/IEC 8859-1) */
-            int error_number = gb18030_utf8tosb(symbol->eci && symbol->eci <= 899 ? symbol->eci : 3, source, &length,
+            int error_number = gb18030_utf8_to_eci(symbol->eci, source, &length, gbdata, full_multibyte);
-                                    gbdata, full_multibyte);
             if (error_number == 0) {
                 done = 1;
-            } else if (symbol->eci && symbol->eci <= 899) {
+            } else if (symbol->eci) {
                 strcpy(symbol->errtxt, "575: Invalid characters in input data");
                 return error_number;
             }
         }
         if (!done) {
             /* Try GB 18030 */
-            int error_number = gb18030_utf8tomb(symbol, source, &length, gbdata);
+            int error_number = gb18030_utf8(symbol, source, &length, gbdata);
             if (error_number != 0) {
                 return error_number;
             }
@@ -1502,7 +1503,7 @@ INTERNAL int han_xin(struct zint_symbol *symbol, unsigned char source[], int len
 #ifndef _MSC_VER
     char binary[est_binlen + 1];
 #else
-    binary = (char *) _alloca((est_binlen + 1) * sizeof (char));
+    binary = (char *) _alloca((est_binlen + 1));
 #endif
 
     if ((ecc_level <= 0) || (ecc_level >= 5)) {

backend/library.c

@@ -1,7 +1,7 @@
 /*  library.c - external functions of libzint
 
     libzint - the open source barcode library
-    Copyright (C) 2009 - 2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2009 - 2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -37,6 +37,7 @@
 #include <malloc.h>
 #endif
 #include "common.h"
+#include "eci.h"
 #include "gs1.h"
 
 #define TECHNETIUM  "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ-. $/+%"
@@ -123,9 +124,6 @@ void ZBarcode_Delete(struct zint_symbol *symbol) {
     free(symbol);
 }
 
-INTERNAL int get_best_eci(unsigned char source[], int length); /* Calculate suitable ECI mode */
-INTERNAL int utf_to_eci(const int eci, const unsigned char source[], unsigned char dest[], int *length); /* Convert Unicode to other encodings */
-
 INTERNAL int eanx(struct zint_symbol *symbol, unsigned char source[], int length); /* EAN system barcodes */
 INTERNAL int c39(struct zint_symbol *symbol, unsigned char source[], int length); /* Code 3 from 9 (or Code 39) */
 INTERNAL int pharmazentral(struct zint_symbol *symbol, unsigned char source[], int length); /* Pharmazentral Nummer (PZN) */
@@ -748,17 +746,17 @@ static int reduced_charset(struct zint_symbol *symbol, unsigned char *source, in
     int error_number = 0;
     unsigned char *preprocessed = source;
 
+    int eci_length = get_eci_length(symbol->eci, source, in_length);
 #ifndef _MSC_VER
-    unsigned char preprocessed_buf[in_length + 1];
+    unsigned char preprocessed_buf[eci_length + 1];
 #else
-    unsigned char *preprocessed_buf = (unsigned char *) _alloca(in_length + 1);
+    unsigned char *preprocessed_buf = (unsigned char *) _alloca(eci_length + 1);
 #endif
 
-    if ((symbol->input_mode & 0x07) == UNICODE_MODE) {
+    if ((symbol->input_mode & 0x07) == UNICODE_MODE && is_eci_convertible(symbol->eci)) {
         /* Prior check ensures ECI only set for those that support it */
         preprocessed = preprocessed_buf;
-        error_number = utf_to_eci(symbol->eci && symbol->eci <= 899 ? symbol->eci : 3, source, preprocessed,
+        error_number = utf8_to_eci(symbol->eci, source, preprocessed, &in_length);
-                        &in_length);
         if (error_number != 0) {
             strcpy(symbol->errtxt, "204: Invalid characters in input data");
             return error_number;
@@ -1078,10 +1076,24 @@ static int escape_char_process(struct zint_symbol *symbol, unsigned char *input_
     return error_number;
 }
 
+/* Is string valid UTF-8? */
+static int is_valid_utf8(const unsigned char source[], const int length) {
+    int i;
+    unsigned int codepoint, state = 0;
+
+    for (i = 0; i < length; i++) {
+        if (decode_utf8(&state, &codepoint, source[i]) == 12) {
+            return 0;
+        }
+    }
+
+    return state == 0;
+}
+
 int ZBarcode_Encode(struct zint_symbol *symbol, const unsigned char *source, int in_length) {
     int error_number, error_buffer;
 #ifdef _MSC_VER
-    unsigned char* local_source;
+    unsigned char *local_source;
 #endif
 
     if (!symbol) return ZINT_ERROR_INVALID_DATA;
@@ -1112,7 +1124,7 @@ int ZBarcode_Encode(struct zint_symbol *symbol, const unsigned char *source, int
     }
     if (in_length > ZINT_MAX_DATA_LEN) {
         strcpy(symbol->errtxt, "243: Input data too long");
-        error_tag(symbol->errtxt, ZINT_ERROR_INVALID_DATA);
+        error_tag(symbol->errtxt, ZINT_ERROR_TOO_LONG);
         return ZINT_ERROR_TOO_LONG;
     }
 
@@ -1123,11 +1135,6 @@ int ZBarcode_Encode(struct zint_symbol *symbol, const unsigned char *source, int
         strcpy(symbol->outfile, "out.png");
 #endif
     }
-#ifndef _MSC_VER
-    unsigned char local_source[in_length + 1];
-#else
-    local_source = (unsigned char*) _alloca(in_length + 1);
-#endif
 
     /* First check the symbology field */
     if (!ZBarcode_ValidID(symbol->symbology)) {
@@ -1272,6 +1279,11 @@ int ZBarcode_Encode(struct zint_symbol *symbol, const unsigned char *source, int
         error_number = ZINT_ERROR_INVALID_OPTION;
     }
 
+    if ((symbol->input_mode & 0x07) == UNICODE_MODE && !is_valid_utf8(source, in_length)) {
+        strcpy(symbol->errtxt, "245: Invalid UTF-8");
+        error_number = ZINT_ERROR_INVALID_DATA;
+    }
+
     if ((symbol->input_mode & 0x07) > 2) {
         symbol->input_mode = DATA_MODE; /* Reset completely */
     }
@@ -1282,6 +1294,12 @@ int ZBarcode_Encode(struct zint_symbol *symbol, const unsigned char *source, int
     }
     error_buffer = error_number;
 
+#ifndef _MSC_VER
+    unsigned char local_source[in_length + 1];
+#else
+    local_source = (unsigned char *) _alloca(in_length + 1);
+#endif
+
     memcpy(local_source, source, in_length);
     local_source[in_length] = '\0';
 
@@ -1601,14 +1619,14 @@ int ZBarcode_Encode_File(struct zint_symbol *symbol, char *filename) {
         }
         if (fileLen > ZINT_MAX_DATA_LEN) {
             strcpy(symbol->errtxt, "230: Input file too long");
-            error_tag(symbol->errtxt, ZINT_ERROR_INVALID_DATA);
+            error_tag(symbol->errtxt, ZINT_ERROR_TOO_LONG);
             fclose(file);
-            return ZINT_ERROR_INVALID_DATA;
+            return ZINT_ERROR_TOO_LONG;
         }
     }
 
     /* Allocate memory */
-    buffer = (unsigned char *) malloc(fileLen * sizeof (unsigned char));
+    buffer = (unsigned char *) malloc(fileLen);
     if (!buffer) {
         strcpy(symbol->errtxt, "231: Internal memory error");
         error_tag(symbol->errtxt, ZINT_ERROR_MEMORY);

backend/qr.c

@@ -1,7 +1,7 @@
 /* qr.c Handles QR Code, Micro QR Code, UPNQR and rMQR
 
     libzint - the open source barcode library
-    Copyright (C) 2009 - 2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2009 - 2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -36,14 +36,12 @@
 #endif
 #include "common.h"
 #include <stdio.h>
+#include "eci.h"
 #include "sjis.h"
 #include "qr.h"
 #include "reedsol.h"
 #include <assert.h>
 
-/* Convert Unicode to other encodings */
-INTERNAL int utf_to_eci(const int eci, const unsigned char source[], unsigned char dest[], int *length);
-
 /* Returns true if input glyph is in the Alphanumeric set */
 static int is_alpha(const unsigned int glyph, const int gs1) {
     int retval = 0;
@@ -1515,22 +1513,23 @@ INTERNAL int qr_code(struct zint_symbol *symbol, unsigned char source[], int len
     int canShrink;
     int size_squared;
     int debug_print = symbol->debug & ZINT_DEBUG_PRINT;
+    int eci_length = get_eci_length(symbol->eci, source, length);
 
 #ifndef _MSC_VER
-    unsigned int jisdata[length + 1];
+    unsigned int jisdata[eci_length + 1];
-    char mode[length];
+    char mode[eci_length];
-    char prev_mode[length];
+    char prev_mode[eci_length];
 #else
-    unsigned char* datastream;
+    unsigned char *datastream;
-    unsigned char* fullstream;
+    unsigned char *fullstream;
-    unsigned char* grid;
+    unsigned char *grid;
-    unsigned int* jisdata = (unsigned int *) _alloca((length + 1) * sizeof (unsigned int));
+    unsigned int *jisdata = (unsigned int *) _alloca((eci_length + 1) * sizeof(unsigned int));
-    char *mode = (char *) _alloca(length);
+    char *mode = (char *) _alloca(eci_length);
-    char *prev_mode = (char *) _alloca(length);
+    char *prev_mode = (char *) _alloca(eci_length);
 #endif
 
     gs1 = ((symbol->input_mode & 0x07) == GS1_MODE);
-    /* If ZINT_FULL_MULTIBYTE use Kanji mode in DATA_MODE or for single-byte Latin */
+    /* If ZINT_FULL_MULTIBYTE use Kanji mode in DATA_MODE or for non-Shift JIS in UNICODE_MODE */
     full_multibyte = (symbol->option_3 & 0xFF) == ZINT_FULL_MULTIBYTE;
     user_mask = (symbol->option_3 >> 8) & 0x0F; /* User mask is pattern + 1, so >= 1 and <= 8 */
     if (user_mask > 8) {
@@ -1542,19 +1541,18 @@ INTERNAL int qr_code(struct zint_symbol *symbol, unsigned char source[], int len
     } else {
         int done = 0;
         if (symbol->eci != 20) { /* Unless ECI 20 (Shift JIS) */
-            /* Try single byte (Latin) conversion first */
+            /* Try other encodings (ECI 0 defaults to ISO/IEC 8859-1) */
-            int error_number = sjis_utf8tosb(symbol->eci && symbol->eci <= 899 ? symbol->eci : 3, source, &length,
+            int error_number = sjis_utf8_to_eci(symbol->eci, source, &length, jisdata, full_multibyte);
-                                    jisdata, full_multibyte);
             if (error_number == 0) {
                 done = 1;
-            } else if (symbol->eci && symbol->eci <= 899) {
+            } else if (symbol->eci) {
                 strcpy(symbol->errtxt, "575: Invalid characters in input data");
                 return error_number;
             }
         }
         if (!done) {
             /* Try Shift-JIS */
-            int error_number = sjis_utf8tomb(symbol, source, &length, jisdata);
+            int error_number = sjis_utf8(symbol, source, &length, jisdata);
             if (error_number != 0) {
                 return error_number;
             }
@@ -2388,7 +2386,7 @@ INTERNAL int microqr(struct zint_symbol *symbol, unsigned char source[], int len
         ecc_level = symbol->option_1;
     }
 
-    /* If ZINT_FULL_MULTIBYTE use Kanji mode in DATA_MODE or for single-byte Latin */
+    /* If ZINT_FULL_MULTIBYTE use Kanji mode in DATA_MODE or for non-Shift JIS in UNICODE_MODE */
     full_multibyte = (symbol->option_3 & 0xFF) == ZINT_FULL_MULTIBYTE;
     user_mask = (symbol->option_3 >> 8) & 0x0F; /* User mask is pattern + 1, so >= 1 and <= 4 */
     if (user_mask > 4) {
@@ -2399,10 +2397,10 @@ INTERNAL int microqr(struct zint_symbol *symbol, unsigned char source[], int len
         sjis_cpy(source, &length, jisdata, full_multibyte);
     } else {
         /* Try ISO 8859-1 conversion first */
-        int error_number = sjis_utf8tosb(3, source, &length, jisdata, full_multibyte);
+        int error_number = sjis_utf8_to_eci(3, source, &length, jisdata, full_multibyte);
         if (error_number != 0) {
             /* Try Shift-JIS */
-            error_number = sjis_utf8tomb(symbol, source, &length, jisdata);
+            error_number = sjis_utf8(symbol, source, &length, jisdata);
             if (error_number != 0) {
                 return error_number;
             }
@@ -2687,7 +2685,7 @@ INTERNAL int upnqr(struct zint_symbol *symbol, unsigned char source[], int lengt
             return ZINT_ERROR_INVALID_OPTION;
             break;
         case UNICODE_MODE:
-            error_number = utf_to_eci(4, source, preprocessed, &length);
+            error_number = utf8_to_eci(4, source, preprocessed, &length);
             if (error_number != 0) {
                 strcpy(symbol->errtxt, "572: Invalid characters in input data");
                 return error_number;
@@ -2897,17 +2895,17 @@ INTERNAL int rmqr(struct zint_symbol *symbol, unsigned char source[], int length
 #endif
 
     gs1 = ((symbol->input_mode & 0x07) == GS1_MODE);
-    /* If ZINT_FULL_MULTIBYTE use Kanji mode in DATA_MODE or for single-byte Latin */
+    /* If ZINT_FULL_MULTIBYTE use Kanji mode in DATA_MODE or for non-Shift JIS in UNICODE_MODE */
     full_multibyte = (symbol->option_3 & 0xFF) == ZINT_FULL_MULTIBYTE;
 
     if ((symbol->input_mode & 0x07) == DATA_MODE) {
         sjis_cpy(source, &length, jisdata, full_multibyte);
     } else {
         /* Try ISO 8859-1 conversion first */
-        int error_number = sjis_utf8tosb(3, source, &length, jisdata, full_multibyte);
+        int error_number = sjis_utf8_to_eci(3, source, &length, jisdata, full_multibyte);
         if (error_number != 0) {
             /* Try Shift-JIS */
-            error_number = sjis_utf8tomb(symbol, source, &length, jisdata);
+            error_number = sjis_utf8(symbol, source, &length, jisdata);
             if (error_number != 0) {
                 return error_number;
             }

backend/reedsol.c

@@ -59,7 +59,7 @@
 
 // rs_init_gf(&rs, prime_poly) initialises the parameters for the Galois Field.
 // The symbol size is determined from the highest bit set in poly
-// This implementation will support sizes up to 8 bits (see rs_unit_init_gf()
+// This implementation will support sizes up to 8 bits (see rs_uint_init_gf()
 // for sizes > 8 bits and <= 30 bits) - bit sizes of 8 or 4 are typical
 //
 // The poly is the bit pattern representing the GF characteristic
@@ -104,7 +104,6 @@ INTERNAL void rs_init_gf(rs_t *rs, const unsigned int prime_poly) {
 // (x + 2**i)*(x + 2**(i+1))*...   [nsym terms]
 // For ECC200, index is 1.
 
-#include <stdio.h>
 INTERNAL void rs_init_code(rs_t *rs, const int nsym, int index) {
     int i, k;
     const unsigned char *logt = rs->logt;

backend/sjis.c

@@ -1,6 +1,6 @@
 /*
     libzint - the open source barcode library
-    Copyright (C) 2008-2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2019-2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -57,9 +57,7 @@
 #endif
 #include "common.h"
 #include "sjis.h"
-
+#include "eci.h"
-/* Convert Unicode to other encodings */
-INTERNAL int utf_to_eci(const int eci, const unsigned char source[], unsigned char dest[], int *length);
 
 /*
  * JISX0201.1976-0 (libiconv-1.16/lib/jisx0201.h)
@@ -1516,7 +1514,7 @@ INTERNAL int sjis_wctomb_zint(unsigned int *r, const unsigned int wc) {
 }
 
 /* Convert UTF-8 string to Shift JIS and place in array of ints */
-INTERNAL int sjis_utf8tomb(struct zint_symbol *symbol, const unsigned char source[], int *p_length,
+INTERNAL int sjis_utf8(struct zint_symbol *symbol, const unsigned char source[], int *p_length,
                 unsigned int *jisdata) {
     int error_number;
     unsigned int i, length;
@@ -1541,23 +1539,29 @@ INTERNAL int sjis_utf8tomb(struct zint_symbol *symbol, const unsigned char sourc
     return 0;
 }
 
-/* Convert UTF-8 string to single byte ECI and place in array of ints */
+/* Convert UTF-8 string to ECI and place in array of ints */
-INTERNAL int sjis_utf8tosb(const int eci, const unsigned char source[], int *p_length, unsigned int *jisdata,
+INTERNAL int sjis_utf8_to_eci(const int eci, const unsigned char source[], int *p_length, unsigned int *jisdata,
                 const int full_multibyte) {
+
+    if (is_eci_convertible(eci)) {
         int error_number;
+        int eci_length = get_eci_length(eci, source, *p_length);
 #ifndef _MSC_VER
-    unsigned char single_byte[*p_length + 1];
+        unsigned char converted[eci_length + 1];
 #else
-    unsigned char *single_byte = (unsigned char *) _alloca(*p_length + 1);
+        unsigned char *converted = (unsigned char *) _alloca(eci_length + 1);
 #endif
 
-    error_number = utf_to_eci(eci, source, single_byte, p_length);
+        error_number = utf8_to_eci(eci, source, converted, p_length);
         if (error_number != 0) {
             // Note not setting `symbol->errtxt`, up to caller
             return error_number;
         }
 
-    sjis_cpy(single_byte, p_length, jisdata, full_multibyte);
+        sjis_cpy(converted, p_length, jisdata, full_multibyte);
+    } else {
+        sjis_cpy(source, p_length, jisdata, full_multibyte);
+    }
 
     return 0;
 }

backend/sjis.h

@@ -1,7 +1,7 @@
 /*  sjis.h - Unicode to Shift JIS
 
     libzint - the open source barcode library
-    Copyright (C) 2009-2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2009-2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -38,9 +38,9 @@ extern "C" {
 #endif /* __cplusplus */
 
 INTERNAL int sjis_wctomb_zint(unsigned int *r, const unsigned int wc);
-INTERNAL int sjis_utf8tomb(struct zint_symbol *symbol, const unsigned char source[], int *p_length,
+INTERNAL int sjis_utf8(struct zint_symbol *symbol, const unsigned char source[], int *p_length,
                 unsigned int *jisdata);
-INTERNAL int sjis_utf8tosb(const int eci, const unsigned char source[], int *p_length, unsigned int *jisdata,
+INTERNAL int sjis_utf8_to_eci(const int eci, const unsigned char source[], int *p_length, unsigned int *jisdata,
                 const int full_multibyte);
 INTERNAL void sjis_cpy(const unsigned char source[], int *p_length, unsigned int *jisdata, const int full_multibyte);
 

backend/tests/CMakeLists.txt

@@ -80,6 +80,7 @@ endmacro()
 zint_add_test(2of5, test_2of5)
 zint_add_test(auspost, test_auspost)
 zint_add_test(aztec, test_aztec)
+zint_add_test(big5, test_big5)
 zint_add_test(bmp, test_bmp)
 zint_add_test(channel, test_channel)
 zint_add_test(codablock, test_codablock)
@@ -101,6 +102,7 @@ zint_add_test(gridmtx, test_gridmtx)
 zint_add_test(gs1, test_gs1)
 zint_add_test(hanxin, test_hanxin)
 zint_add_test(imail, test_imail)
+zint_add_test(ksx1001, test_ksx1001)
 zint_add_test(large, test_large)
 zint_add_test(library, test_library)
 zint_add_test(mailmark, test_mailmark)

backend/tests/test_aztec.c

@@ -1,6 +1,6 @@
 /*
     libzint - the open source barcode library
-    Copyright (C) 2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2020 - 2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -1612,7 +1612,7 @@ static void test_fuzz(int index, int debug) {
                     "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377"
                     "\377\377\377\377\377\377\261\261\261\261\261\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\135\135\135\135\135\135"
                     "\135\335\135\060\060\010\010\010\010\010\060",
-                    2251, UNICODE_MODE, -1, ZINT_ERROR_TOO_LONG }, // Original OSS-Fuzz triggering data for malloc leak
+                    2251, DATA_MODE, -1, ZINT_ERROR_TOO_LONG }, // Original OSS-Fuzz triggering data for malloc leak
         /* 1*/ { BARCODE_AZTEC,
                     "\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060"
                     "\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\060\000\060\060\060\060\000\060\060\000\060\060\060\060"
@@ -1702,7 +1702,7 @@ static void test_fuzz(int index, int debug) {
                     "\060\060\060\363\060\060\060\060\060\060\060\060\060\060\060\060\362\060\060\060\060\060\000\060\060\377\060\060\060\175\175\175"
                     "\175\060\060\060\175\175\175\175\060\060\005\060\005\060\005\060\060\060\060\000\000\060\060\060\060\060\060\377\060\060\060\060"
                     "\377\060\377\377\060\060\057\060\060\057\060\060\060\000\000\060\060",
-                    2801, UNICODE_MODE, -1, ZINT_ERROR_TOO_LONG }, // Original OSS-Fuzz triggering data for binary_string buffer overrun
+                    2801, DATA_MODE, -1, ZINT_ERROR_TOO_LONG }, // Original OSS-Fuzz triggering data for binary_string buffer overrun
         /* 2*/ { BARCODE_AZTEC,
                     "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"
                     "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"

backend/tests/test_big5.c

@@ -0,0 +1,87 @@
+/*
+    libzint - the open source barcode library
+    Copyright (C) 2021 Robin Stuart <rstuart114@gmail.com>
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions
+    are met:
+
+    1. Redistributions of source code must retain the above copyright
+       notice, this list of conditions and the following disclaimer.
+    2. Redistributions in binary form must reproduce the above copyright
+       notice, this list of conditions and the following disclaimer in the
+       documentation and/or other materials provided with the distribution.
+    3. Neither the name of the project nor the names of its contributors
+       may be used to endorse or promote products derived from this software
+       without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+    ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+    IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+    ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+    FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+    OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+    HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+    LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+    OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+    SUCH DAMAGE.
+ */
+/* vim: set ts=4 sw=4 et : */
+
+#include "testcommon.h"
+#include "test_big5_tab.h"
+#include "../big5.h"
+
+// As control convert to Big5 using simple table generated from https://www.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/OTHER/BIG5.TXT plus simple processing
+static int big5_wctomb_zint2(unsigned int *r, unsigned int wc) {
+    if (wc < 0x80) {
+        return 0;
+    }
+    int tab_length = ARRAY_SIZE(test_big5_tab);
+    int start_i = test_big5_tab_ind[wc >> 10];
+    int end_i = start_i + 0x800 > tab_length ? tab_length : start_i + 0x800;
+    for (int i = start_i; i < end_i; i += 2) {
+        if (test_big5_tab[i + 1] == wc) {
+            *r = test_big5_tab[i];
+            return *r > 0xFF ? 2 : 1;
+        }
+    }
+    return 0;
+}
+
+static void test_big5_wctomb_zint(void) {
+
+    testStart("");
+
+    int ret, ret2;
+    unsigned int val, val2;
+
+    for (unsigned int i = 0; i < 0xFFFE; i++) {
+        if (i >= 0xD800 && i < 0xE000) { // UTF-16 surrogates
+            continue;
+        }
+        val = val2 = 0;
+        ret = big5_wctomb_zint(&val, i);
+        ret2 = big5_wctomb_zint2(&val2, i);
+        assert_equal(ret, ret2, "i:%d 0x%04X ret %d != ret2 %d, val 0x%04X, val2 0x%04X\n", i, i, ret, ret2, val, val2);
+        if (ret2) {
+            assert_equal(val, val2, "i:%d 0x%04X val 0x%04X != val2 0x%04X\n", i, i, val, val2);
+        }
+    }
+
+    testFinish();
+}
+
+int main(int argc, char *argv[]) {
+
+    testFunction funcs[] = { /* name, func, has_index, has_generate, has_debug */
+        { "test_big5_wctomb_zint", test_big5_wctomb_zint, 0, 0, 0 },
+    };
+
+    testRun(argc, argv, funcs, ARRAY_SIZE(funcs));
+
+    testReport();
+
+    return 0;
+}

backend/tests/test_dotcode.c

@@ -1,6 +1,6 @@
 /*
     libzint - the open source barcode library
-    Copyright (C) 2019 - 2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2019 - 2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -433,12 +433,12 @@ static void test_fuzz(int index, int debug) {
                     "\071\071\071\071\071\072\071\071\277\071\071\077\071\071\071\071\071\071\071\071\154\071\071\071\071\071\071\071\071\071\071\071"
                     "\071\071\071\011\071\071\071\071\071\071\071\071\071\071\071\071\071\071\105\105\105\105\105\105\105\105\105\105\105\105\105\071"
                     "\071\071\071\071\071", // Original OSS-Fuzz triggering data for index out of bounds (encoding of HT/FS/GS/RS when shifting to code set B)
-                    421, UNICODE_MODE, ZINT_WARN_USES_ECI },
+                    421, DATA_MODE, 0 },
-        /*  2*/ { "\233:", -1, UNICODE_MODE, ZINT_WARN_USES_ECI }, // Original OSS-Fuzz triggering data for codeword_array buffer overflow, L777
+        /*  2*/ { "\233:", -1, DATA_MODE, 0 }, // Original OSS-Fuzz triggering data for codeword_array buffer overflow, L777
-        /*  3*/ { "\241\034", -1, UNICODE_MODE, ZINT_WARN_USES_ECI }, // As above L793
+        /*  3*/ { "\241\034", -1, DATA_MODE, 0 }, // As above L793
-        /*  4*/ { "\270\036", -1, UNICODE_MODE, ZINT_WARN_USES_ECI }, // As above L799
+        /*  4*/ { "\270\036", -1, DATA_MODE, 0 }, // As above L799
-        /*  5*/ { "\237\032", -1, UNICODE_MODE, ZINT_WARN_USES_ECI }, // As above L904
+        /*  5*/ { "\237\032", -1, DATA_MODE, 0 }, // As above L904
-        /*  6*/ { "\237", -1, UNICODE_MODE, ZINT_WARN_USES_ECI }, // As above L1090
+        /*  6*/ { "\237", -1, DATA_MODE, 0 }, // As above L1090
     };
     int data_size = sizeof(data) / sizeof(struct item);
 

backend/tests/test_eci.c

@@ -1,6 +1,6 @@
 /*
     libzint - the open source barcode library
-    Copyright (C) 2019 - 2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2019 - 2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -27,9 +27,10 @@
     OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     SUCH DAMAGE.
  */
-/* vim: set ts=4 sw=4 et : */
+/* vim: set ts=4 sw=4 et norl : */
 
 #include "testcommon.h"
+#include "../eci.h"
 
 static void test_bom(int debug) {
 
@@ -154,66 +155,211 @@ static void test_reduced_charset_input(int index, int debug) {
         /* 18*/ { BARCODE_LOGMARS, UNICODE_MODE, 0, "é", ZINT_ERROR_INVALID_DATA, -1, "ASCII subset only" },
         /* 19*/ { BARCODE_PDF417, UNICODE_MODE, 0, "é", 0, 0, "" },
         /* 20*/ { BARCODE_PDF417, UNICODE_MODE, 3, "é", 0, 3, "Supports ECI" },
-        /* 21*/ { BARCODE_PDF417, UNICODE_MODE, 0, "β", ZINT_WARN_USES_ECI, 9, "" },
+        /* 21*/ { BARCODE_PDF417, UNICODE_MODE, 3, "\302\200", ZINT_ERROR_INVALID_DATA, -1, "U+0080" },
-        /* 22*/ { BARCODE_PDF417, UNICODE_MODE, 9, "β", 0, 9, "" },
+        /* 22*/ { BARCODE_PDF417, UNICODE_MODE, 3, "\302\237", ZINT_ERROR_INVALID_DATA, -1, "U+009F" },
-        /* 23*/ { BARCODE_PDF417, UNICODE_MODE, 0, "ก", ZINT_WARN_USES_ECI, 13, "" },
+        /* 23*/ { BARCODE_PDF417, UNICODE_MODE, 0, "˘", ZINT_WARN_USES_ECI, 4, "In ISO 8859-2 and ISO 8859-3 only of single-byte pages" },
-        /* 24*/ { BARCODE_PDF417, UNICODE_MODE, 13, "ก", 0, 13, "" },
+        /* 24*/ { BARCODE_PDF417, UNICODE_MODE, 4, "˘", 0, 4, "" },
-        /* 25*/ { BARCODE_PDF417, UNICODE_MODE, 0, "Ж", ZINT_WARN_USES_ECI, 7, "" },
+        /* 25*/ { BARCODE_PDF417, UNICODE_MODE, 0, "Ħ", ZINT_WARN_USES_ECI, 5, "In ISO 8859-3 only of single-byte pages" },
-        /* 26*/ { BARCODE_PDF417, UNICODE_MODE, 7, "Ж", 0, 7, "" },
+        /* 26*/ { BARCODE_PDF417, UNICODE_MODE, 5, "Ħ", 0, 5, "" },
-        /* 27*/ { BARCODE_PDF417, UNICODE_MODE, 0, "…", ZINT_WARN_USES_ECI, 21, "" },
+        /* 27*/ { BARCODE_PDF417, UNICODE_MODE, 0, "ĸ", ZINT_WARN_USES_ECI, 6, "In ISO 8859-4 and ISO 8859-6 only of single-byte pages" },
-        /* 28*/ { BARCODE_PDF417, UNICODE_MODE, 21, "…", 0, 21, "" },
+        /* 28*/ { BARCODE_PDF417, UNICODE_MODE, 6, "ĸ", 0, 6, "" },
-        /* 29*/ { BARCODE_PDF417, UNICODE_MODE, 0, "テ", ZINT_WARN_USES_ECI, 26, "Defaults to UTF-8 if not in any ISO 8859 or Win page" },
+        /* 29*/ { BARCODE_PDF417, UNICODE_MODE, 0, "Ж", ZINT_WARN_USES_ECI, 7, "In ISO 8859-5 and Win 1251 only of single-byte pages" },
-        /* 30*/ { BARCODE_PDF417, UNICODE_MODE, 26, "テ", 0, 26, "" },
+        /* 30*/ { BARCODE_PDF417, UNICODE_MODE, 7, "Ж", 0, 7, "" },
-        /* 31*/ { BARCODE_PDF417, UNICODE_MODE, 20, "テ", ZINT_ERROR_INVALID_DATA, -1, "テ in ECI 20 but that conversion not currently supported in Zint for non-extended barcodes" },
+        /* 31*/ { BARCODE_PDF417, UNICODE_MODE, 0, "غ", ZINT_WARN_USES_ECI, 8, "In ISO 8859-6 and Win 1256 only of single-byte pages" },
-        /* 32*/ { BARCODE_PDF417, UNICODE_MODE, 900, "é", 0, 900, "ECI > 899 ignored for character set conversion" },
+        /* 32*/ { BARCODE_PDF417, UNICODE_MODE, 8, "غ", 0, 8, "" },
-        /* 33*/ { BARCODE_PDF417, UNICODE_MODE, 900, "β", ZINT_ERROR_INVALID_DATA, 900, "But ECI > 899 suppresses auto-ECI `get_best_eci()`" },
+        /* 33*/ { BARCODE_PDF417, UNICODE_MODE, 0, "β", ZINT_WARN_USES_ECI, 9, "In ISO 8859-7 only of single-byte pages" },
-        /* 34*/ { BARCODE_PDF417COMP, UNICODE_MODE, 0, "é", 0, 0, "" },
+        /* 34*/ { BARCODE_PDF417, UNICODE_MODE, 9, "β", 0, 9, "" },
-        /* 35*/ { BARCODE_PDF417COMP, UNICODE_MODE, 3, "é", 0, 3, "Supports ECI" },
+        /* 35*/ { BARCODE_PDF417, UNICODE_MODE, 0, "‗", ZINT_WARN_USES_ECI, 10, "In ISO 8859-8 only of single-byte pages" },
-        /* 36*/ { BARCODE_PDF417COMP, UNICODE_MODE, 0, "β", ZINT_WARN_USES_ECI, 9, "" },
+        /* 36*/ { BARCODE_PDF417, UNICODE_MODE, 10, "‗", 0, 10, "" },
-        /* 37*/ { BARCODE_PDF417COMP, UNICODE_MODE, 9, "β", 0, 9, "" },
+        /* 37*/ { BARCODE_PDF417, UNICODE_MODE, 11, "Ğ", 0, 11, "In ISO 8859-9; Note no characters in ISO 8859-9 that aren't also in earlier ISO pages" },
-        /* 38*/ { BARCODE_PDF417COMP, UNICODE_MODE, 26, "テ", 0, 26, "" },
+        /* 38*/ { BARCODE_PDF417, UNICODE_MODE, 12, "Ĩ", 0, 12, "In ISO 8859-10; Note no characters in ISO 8859-10 that aren't also in earlier ISO pages" },
-        /* 39*/ { BARCODE_MAXICODE, UNICODE_MODE, 0, "é", 0, 0, "" },
+        /* 39*/ { BARCODE_PDF417, UNICODE_MODE, 0, "ก", ZINT_WARN_USES_ECI, 13, "" },
-        /* 40*/ { BARCODE_MAXICODE, UNICODE_MODE, 3, "é", 0, 3, "Supports ECI" },
+        /* 40*/ { BARCODE_PDF417, UNICODE_MODE, 13, "ก", 0, 13, "" },
-        /* 41*/ { BARCODE_MAXICODE, UNICODE_MODE, 0, "β", ZINT_WARN_USES_ECI, 9, "" },
+        /* 41*/ { BARCODE_PDF417, UNICODE_MODE, 14, "A", ZINT_ERROR_INVALID_DATA, -1, "Reserved ECI" },
-        /* 42*/ { BARCODE_MAXICODE, UNICODE_MODE, 9, "β", 0, 9, "" },
+        /* 42*/ { BARCODE_PDF417, UNICODE_MODE, 0, "„", ZINT_WARN_USES_ECI, 15, "" },
-        /* 43*/ { BARCODE_MAXICODE, UNICODE_MODE, 26, "テ", 0, 26, "" },
+        /* 43*/ { BARCODE_PDF417, UNICODE_MODE, 15, "„", 0, 15, "In ISO 8859-13 and ISO 8859-16 and Win 125x pages" },
-        /* 44*/ { BARCODE_CODE128B, UNICODE_MODE, 0, "é", 0, 0, "" },
+        /* 44*/ { BARCODE_PDF417, UNICODE_MODE, 0, "Ḃ", ZINT_WARN_USES_ECI, 16, "In ISO 8859-14 only of single-byte pages" },
-        /* 45*/ { BARCODE_CODE128B, UNICODE_MODE, 3, "é", ZINT_ERROR_INVALID_OPTION, -1, "Does not support ECI" },
+        /* 45*/ { BARCODE_PDF417, UNICODE_MODE, 16, "Ḃ", 0, 16, "" },
-        /* 46*/ { BARCODE_CODE128B, UNICODE_MODE, 0, "β", ZINT_ERROR_INVALID_DATA, -1, "β not in ISO 8859-1" },
+        /* 46*/ { BARCODE_PDF417, UNICODE_MODE, 17, "Ž", 0, 17, "In ISO 8859-15; Note no characters in ISO 8859-15 that aren't also in earlier ISO pages" },
-        /* 47*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 0, "é", 0, 0, "" },
+        /* 47*/ { BARCODE_PDF417, UNICODE_MODE, 0, "Ș", ZINT_WARN_USES_ECI, 18, "In ISO 8859-16 only of single-byte pages" },
-        /* 48*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 3, "é", 0, 3, "Supports ECI" },
+        /* 48*/ { BARCODE_PDF417, UNICODE_MODE, 18, "Ș", 0, 18, "" },
-        /* 49*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 0, "β", ZINT_WARN_USES_ECI, 9, "" },
+        /* 49*/ { BARCODE_PDF417, UNICODE_MODE, 0, "テ", ZINT_WARN_USES_ECI, 26, "Not in any single-byte page" },
-        /* 50*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 9, "β", 0, 9, "" },
+        /* 50*/ { BARCODE_PDF417, UNICODE_MODE, 19, "A", ZINT_ERROR_INVALID_DATA, -1, "Reserved ECI" },
-        /* 51*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 26, "テ", 0, 26, "" },
+        /* 51*/ { BARCODE_PDF417, UNICODE_MODE, 20, "テ", 0, 20, "In Shift JIS" },
-        /* 52*/ { BARCODE_CODABLOCKF, UNICODE_MODE, 0, "é", 0, 0, "" },
+        /* 52*/ { BARCODE_PDF417, UNICODE_MODE, 20, "テテ", 0, 20, "In Shift JIS" },
-        /* 53*/ { BARCODE_CODABLOCKF, UNICODE_MODE, 3, "é", ZINT_ERROR_INVALID_OPTION, -1, "Does not support ECI" },
+        /* 53*/ { BARCODE_PDF417, UNICODE_MODE, 20, "\\\\", 0, 20, "In Shift JIS" },
-        /* 54*/ { BARCODE_CODABLOCKF, UNICODE_MODE, 0, "β", ZINT_ERROR_INVALID_DATA, -1, "β not in ISO 8859-1" },
+        /* 54*/ { BARCODE_PDF417, UNICODE_MODE, 0, "…", ZINT_WARN_USES_ECI, 21, "In Win 1250 and other Win pages but not in ISO pages" },
-        /* 55*/ { BARCODE_NVE18, UNICODE_MODE, 0, "é", ZINT_ERROR_INVALID_DATA, -1, "Numbers only" },
+        /* 55*/ { BARCODE_PDF417, UNICODE_MODE, 21, "…", 0, 21, "" },
-        /* 56*/ { BARCODE_MICROPDF417, UNICODE_MODE, 0, "é", 0, 0, "" },
+        /* 56*/ { BARCODE_PDF417, UNICODE_MODE, 0, "Ґ", ZINT_WARN_USES_ECI, 22, "In Win 1251 only of single-byte pages" },
-        /* 57*/ { BARCODE_MICROPDF417, UNICODE_MODE, 3, "é", 0, 3, "Supports ECI" },
+        /* 57*/ { BARCODE_PDF417, UNICODE_MODE, 22, "Ґ", 0, 22, "" },
-        /* 58*/ { BARCODE_MICROPDF417, UNICODE_MODE, 0, "β", ZINT_WARN_USES_ECI, 9, "" },
+        /* 58*/ { BARCODE_PDF417, UNICODE_MODE, 0, "˜", ZINT_WARN_USES_ECI, 23, "In Win 1252 only of single-byte pages" },
-        /* 59*/ { BARCODE_MICROPDF417, UNICODE_MODE, 9, "β", 0, 9, "" },
+        /* 59*/ { BARCODE_PDF417, UNICODE_MODE, 23, "˜", 0, 23, "" },
-        /* 60*/ { BARCODE_MICROPDF417, UNICODE_MODE, 26, "テ", 0, 26, "" },
+        /* 60*/ { BARCODE_PDF417, UNICODE_MODE, 0, "پ", ZINT_WARN_USES_ECI, 24, "In Win 1256 only of single-byte pages" },
-        /* 61*/ { BARCODE_USPS_IMAIL, UNICODE_MODE, 0, "é", ZINT_ERROR_INVALID_DATA, -1, "Numbers/dash only" },
+        /* 61*/ { BARCODE_PDF417, UNICODE_MODE, 24, "پ", 0, 24, "" },
-        /* 62*/ { BARCODE_AZTEC, UNICODE_MODE, 0, "é", 0, 0, "" },
+        /* 62*/ { BARCODE_PDF417, UNICODE_MODE, 0, "က", ZINT_WARN_USES_ECI, 26, "Not in any single-byte page" },
-        /* 63*/ { BARCODE_AZTEC, UNICODE_MODE, 3, "é", 0, 3, "Supports ECI" },
+        /* 63*/ { BARCODE_PDF417, UNICODE_MODE, 25, "က", 0, 25, "In UCS-2BE" },
-        /* 64*/ { BARCODE_AZTEC, UNICODE_MODE, 0, "β", ZINT_WARN_USES_ECI, 9, "" },
+        /* 64*/ { BARCODE_PDF417, UNICODE_MODE, 25, "ကက", 0, 25, "In UCS-2BE" },
-        /* 65*/ { BARCODE_AZTEC, UNICODE_MODE, 9, "β", 0, 9, "" },
+        /* 65*/ { BARCODE_PDF417, UNICODE_MODE, 25, "12", 0, 25, "UCS-2BE ASCII" },
-        /* 66*/ { BARCODE_AZTEC, UNICODE_MODE, 26, "テ", 0, 26, "" },
+        /* 66*/ { BARCODE_PDF417, UNICODE_MODE, 0, "𐀀", ZINT_WARN_USES_ECI, 26, "Not in any single-byte page" },
-        /* 67*/ { BARCODE_HIBC_128, UNICODE_MODE, 0, "é", ZINT_ERROR_INVALID_DATA, -1, "HIBC ASCII subset only" },
+        /* 67*/ { BARCODE_PDF417, UNICODE_MODE, 25, "𐀀", ZINT_ERROR_INVALID_DATA, -1, "Not in UCS-2BE (in Supplementary Plane)" },
-        /* 68*/ { BARCODE_HIBC_AZTEC, UNICODE_MODE, 0, "é", ZINT_ERROR_INVALID_DATA, -1, "HIBC ASCII subset only" },
+        /* 68*/ { BARCODE_PDF417, UNICODE_MODE, 0, "テ", ZINT_WARN_USES_ECI, 26, "Defaults to UTF-8 if not in any ISO 8859 or Win page" },
-        /* 69*/ { BARCODE_DOTCODE, UNICODE_MODE, 0, "é", 0, 0, "" },
+        /* 69*/ { BARCODE_PDF417, UNICODE_MODE, 26, "テ", 0, 26, "" },
-        /* 70*/ { BARCODE_DOTCODE, UNICODE_MODE, 3, "é", 0, 3, "Supports ECI" },
+        /* 70*/ { BARCODE_PDF417, UNICODE_MODE, 26, "テテ", 0, 26, "" },
-        /* 71*/ { BARCODE_DOTCODE, UNICODE_MODE, 0, "β", ZINT_WARN_USES_ECI, 9, "" },
+        /* 71*/ { BARCODE_PDF417, UNICODE_MODE, 27, "@", 0, 27, "ASCII" },
-        /* 72*/ { BARCODE_DOTCODE, UNICODE_MODE, 9, "β", 0, 9, "" },
+        /* 72*/ { BARCODE_PDF417, UNICODE_MODE, 0, "龘", ZINT_WARN_USES_ECI, 26, "Not in any single-byte page" },
-        /* 73*/ { BARCODE_DOTCODE, UNICODE_MODE, 26, "テ", 0, 26, "" },
+        /* 73*/ { BARCODE_PDF417, UNICODE_MODE, 28, "龘", 0, 28, "U+9F98 in Big5 but not in GB2312" },
-        /* 74*/ { BARCODE_AZRUNE, UNICODE_MODE, 0, "é", ZINT_ERROR_INVALID_DATA, -1, "Numbers <= 255 only" },
+        /* 74*/ { BARCODE_PDF417, UNICODE_MODE, 28, "龘龘", 0, 28, "U+9F98 in Big5 but not in GB2312" },
-        /* 75*/ { BARCODE_CODE32, UNICODE_MODE, 0, "é", ZINT_ERROR_INVALID_DATA, -1, "Numbers only" },
+        /* 75*/ { BARCODE_PDF417, UNICODE_MODE, 0, "齄", ZINT_WARN_USES_ECI, 26, "Not in any single-byte page" },
-        /* 76*/ { BARCODE_CODEONE, UNICODE_MODE, 0, "é", 0, 0, "" },
+        /* 76*/ { BARCODE_PDF417, UNICODE_MODE, 29, "齄", 0, 29, "U+9F44 in GB2312 but not in Big5" },
-        /* 77*/ { BARCODE_CODEONE, UNICODE_MODE, 3, "é", ZINT_ERROR_INVALID_OPTION, -1, "Does not support ECI" },
+        /* 77*/ { BARCODE_PDF417, UNICODE_MODE, 29, "齄齄", 0, 29, "U+9F44 in GB2312 but not in Big5" },
-        /* 78*/ { BARCODE_CODEONE, UNICODE_MODE, 0, "β", ZINT_ERROR_INVALID_DATA, -1, "β not in ISO 8859-1" },
+        /* 78*/ { BARCODE_PDF417, UNICODE_MODE, 0, "가", ZINT_WARN_USES_ECI, 26, "Not in any single-byte page" },
+        /* 79*/ { BARCODE_PDF417, UNICODE_MODE, 30, "가", 0, 30, "U+AC00 in KS X 1001" },
+        /* 80*/ { BARCODE_PDF417, UNICODE_MODE, 30, "가가", 0, 30, "U+AC00 in KS X 1001" },
+        /* 81*/ { BARCODE_PDF417, UNICODE_MODE, 31, "A", 0, 31, "Undefined character set ECI - ignored for character set conversion" },
+        /* 82*/ { BARCODE_PDF417, UNICODE_MODE, 170, "?", 0, 170, "ASCII invariant" },
+        /* 83*/ { BARCODE_PDF417, UNICODE_MODE, 170, "@", ZINT_ERROR_INVALID_DATA, -1, "Not in ASCII invariant" },
+        /* 84*/ { BARCODE_PDF417, UNICODE_MODE, 0, "\200", ZINT_ERROR_INVALID_DATA, -1, "Not UTF-8" },
+        /* 85*/ { BARCODE_PDF417, DATA_MODE, 899, "\200", 0, 899, "8-bit binary" },
+        /* 86*/ { BARCODE_PDF417, UNICODE_MODE, 900, "é", 0, 900, "Non-character set ECIs > 899 ignored for character set conversion" },
+        /* 87*/ { BARCODE_PDF417, UNICODE_MODE, 900, "β", 0, 900, "Non-character set ECIs > 899 ignored for character set conversion" },
+        /* 88*/ { BARCODE_PDF417COMP, UNICODE_MODE, 0, "é", 0, 0, "" },
+        /* 89*/ { BARCODE_PDF417COMP, UNICODE_MODE, 3, "é", 0, 3, "Supports ECI" },
+        /* 90*/ { BARCODE_PDF417COMP, UNICODE_MODE, 0, "β", ZINT_WARN_USES_ECI, 9, "" },
+        /* 91*/ { BARCODE_PDF417COMP, UNICODE_MODE, 9, "β", 0, 9, "" },
+        /* 92*/ { BARCODE_PDF417COMP, UNICODE_MODE, 20, "テ", 0, 20, "In Shift JIS" },
+        /* 93*/ { BARCODE_PDF417COMP, UNICODE_MODE, 20, "テテ", 0, 20, "In Shift JIS" },
+        /* 94*/ { BARCODE_PDF417COMP, UNICODE_MODE, 25, "က", 0, 25, "In UCS-2BE" },
+        /* 95*/ { BARCODE_PDF417COMP, UNICODE_MODE, 25, "ကက", 0, 25, "In UCS-2BE" },
+        /* 96*/ { BARCODE_PDF417COMP, UNICODE_MODE, 25, "12", 0, 25, "ASCII" },
+        /* 97*/ { BARCODE_PDF417COMP, UNICODE_MODE, 26, "テ", 0, 26, "" },
+        /* 98*/ { BARCODE_PDF417COMP, UNICODE_MODE, 26, "テテ", 0, 26, "" },
+        /* 99*/ { BARCODE_PDF417COMP, UNICODE_MODE, 27, "@", 0, 27, "ASCII" },
+        /*100*/ { BARCODE_PDF417COMP, UNICODE_MODE, 28, "龘", 0, 28, "U+9F98 in Big5 but not in GB2312" },
+        /*101*/ { BARCODE_PDF417COMP, UNICODE_MODE, 28, "龘龘", 0, 28, "U+9F98 in Big5 but not in GB2312" },
+        /*102*/ { BARCODE_PDF417COMP, UNICODE_MODE, 29, "齄", 0, 29, "U+9F44 in GB2312 but not in Big5" },
+        /*103*/ { BARCODE_PDF417COMP, UNICODE_MODE, 29, "齄齄", 0, 29, "U+9F44 in GB2312 but not in Big5" },
+        /*104*/ { BARCODE_PDF417COMP, UNICODE_MODE, 30, "가", 0, 30, "U+AC00 in KS X 1001" },
+        /*105*/ { BARCODE_PDF417COMP, UNICODE_MODE, 30, "가가", 0, 30, "U+AC00 in KS X 1001" },
+        /*106*/ { BARCODE_MAXICODE, UNICODE_MODE, 0, "é", 0, 0, "" },
+        /*107*/ { BARCODE_MAXICODE, UNICODE_MODE, 3, "é", 0, 3, "Supports ECI" },
+        /*108*/ { BARCODE_MAXICODE, UNICODE_MODE, 0, "β", ZINT_WARN_USES_ECI, 9, "" },
+        /*109*/ { BARCODE_MAXICODE, UNICODE_MODE, 9, "β", 0, 9, "" },
+        /*110*/ { BARCODE_MAXICODE, UNICODE_MODE, 20, "テ", 0, 20, "In Shift JIS" },
+        /*111*/ { BARCODE_MAXICODE, UNICODE_MODE, 20, "テテ", 0, 20, "In Shift JIS" },
+        /*112*/ { BARCODE_MAXICODE, UNICODE_MODE, 25, "က", 0, 25, "In UCS-2BE" },
+        /*113*/ { BARCODE_MAXICODE, UNICODE_MODE, 25, "ကက", 0, 25, "In UCS-2BE" },
+        /*114*/ { BARCODE_MAXICODE, UNICODE_MODE, 25, "12", 0, 25, "ASCII" },
+        /*115*/ { BARCODE_MAXICODE, UNICODE_MODE, 26, "テ", 0, 26, "" },
+        /*116*/ { BARCODE_MAXICODE, UNICODE_MODE, 26, "テテ", 0, 26, "" },
+        /*117*/ { BARCODE_MAXICODE, UNICODE_MODE, 27, "@", 0, 27, "ASCII" },
+        /*118*/ { BARCODE_MAXICODE, UNICODE_MODE, 28, "龘", 0, 28, "U+9F98 in Big5 but not in GB2312" },
+        /*119*/ { BARCODE_MAXICODE, UNICODE_MODE, 28, "龘龘", 0, 28, "U+9F98 in Big5 but not in GB2312" },
+        /*120*/ { BARCODE_MAXICODE, UNICODE_MODE, 29, "齄", 0, 29, "U+9F44 in GB2312 but not in Big5" },
+        /*121*/ { BARCODE_MAXICODE, UNICODE_MODE, 29, "齄齄", 0, 29, "U+9F44 in GB2312 but not in Big5" },
+        /*122*/ { BARCODE_MAXICODE, UNICODE_MODE, 30, "가", 0, 30, "U+AC00 in KS X 1001" },
+        /*123*/ { BARCODE_MAXICODE, UNICODE_MODE, 30, "가가", 0, 30, "U+AC00 in KS X 1001" },
+        /*124*/ { BARCODE_CODE128B, UNICODE_MODE, 0, "é", 0, 0, "" },
+        /*125*/ { BARCODE_CODE128B, UNICODE_MODE, 3, "é", ZINT_ERROR_INVALID_OPTION, -1, "Does not support ECI" },
+        /*126*/ { BARCODE_CODE128B, UNICODE_MODE, 0, "β", ZINT_ERROR_INVALID_DATA, -1, "β not in ISO 8859-1" },
+        /*127*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 0, "é", 0, 0, "" },
+        /*128*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 3, "é", 0, 3, "Supports ECI" },
+        /*129*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 0, "β", ZINT_WARN_USES_ECI, 9, "" },
+        /*130*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 9, "β", 0, 9, "" },
+        /*131*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 20, "テ", 0, 20, "In Shift JIS" },
+        /*132*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 20, "テテ", 0, 20, "In Shift JIS" },
+        /*133*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 25, "က", 0, 25, "In UCS-2BE" },
+        /*134*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 25, "ကက", 0, 25, "In UCS-2BE" },
+        /*135*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 25, "12", 0, 25, "ASCII" },
+        /*136*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 26, "テ", 0, 26, "" },
+        /*137*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 26, "テテ", 0, 26, "" },
+        /*138*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 28, "龘", 0, 28, "U+9F98 in Big5 but not in GB2312" },
+        /*139*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 28, "龘龘", 0, 28, "U+9F98 in Big5 but not in GB2312" },
+        /*140*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 29, "齄", 0, 29, "U+9F44 in GB2312 but not in Big5" },
+        /*141*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 29, "齄齄", 0, 29, "U+9F44 in GB2312 but not in Big5" },
+        /*142*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 30, "가", 0, 30, "U+AC00 in KS X 1001" },
+        /*143*/ { BARCODE_DATAMATRIX, UNICODE_MODE, 30, "가가", 0, 30, "U+AC00 in KS X 1001" },
+        /*144*/ { BARCODE_CODABLOCKF, UNICODE_MODE, 0, "é", 0, 0, "" },
+        /*145*/ { BARCODE_CODABLOCKF, UNICODE_MODE, 3, "é", ZINT_ERROR_INVALID_OPTION, -1, "Does not support ECI" },
+        /*146*/ { BARCODE_CODABLOCKF, UNICODE_MODE, 0, "β", ZINT_ERROR_INVALID_DATA, -1, "β not in ISO 8859-1" },
+        /*147*/ { BARCODE_NVE18, UNICODE_MODE, 0, "é", ZINT_ERROR_INVALID_DATA, -1, "Numbers only" },
+        /*148*/ { BARCODE_MICROPDF417, UNICODE_MODE, 0, "é", 0, 0, "" },
+        /*149*/ { BARCODE_MICROPDF417, UNICODE_MODE, 3, "é", 0, 3, "Supports ECI" },
+        /*150*/ { BARCODE_MICROPDF417, UNICODE_MODE, 0, "β", ZINT_WARN_USES_ECI, 9, "" },
+        /*151*/ { BARCODE_MICROPDF417, UNICODE_MODE, 9, "β", 0, 9, "" },
+        /*152*/ { BARCODE_MICROPDF417, UNICODE_MODE, 20, "テ", 0, 20, "In Shift JIS" },
+        /*153*/ { BARCODE_MICROPDF417, UNICODE_MODE, 20, "テテ", 0, 20, "In Shift JIS" },
+        /*154*/ { BARCODE_MICROPDF417, UNICODE_MODE, 25, "က", 0, 25, "In UCS-2BE" },
+        /*155*/ { BARCODE_MICROPDF417, UNICODE_MODE, 25, "ကက", 0, 25, "In UCS-2BE" },
+        /*156*/ { BARCODE_MICROPDF417, UNICODE_MODE, 25, "12", 0, 25, "ASCII" },
+        /*157*/ { BARCODE_MICROPDF417, UNICODE_MODE, 26, "テ", 0, 26, "" },
+        /*158*/ { BARCODE_MICROPDF417, UNICODE_MODE, 26, "テテ", 0, 26, "" },
+        /*159*/ { BARCODE_MICROPDF417, UNICODE_MODE, 28, "龘", 0, 28, "U+9F98 in Big5 but not in GB2312" },
+        /*160*/ { BARCODE_MICROPDF417, UNICODE_MODE, 28, "龘龘", 0, 28, "U+9F98 in Big5 but not in GB2312" },
+        /*161*/ { BARCODE_MICROPDF417, UNICODE_MODE, 29, "齄", 0, 29, "U+9F44 in GB2312 but not in Big5" },
+        /*162*/ { BARCODE_MICROPDF417, UNICODE_MODE, 29, "齄齄", 0, 29, "U+9F44 in GB2312 but not in Big5" },
+        /*163*/ { BARCODE_MICROPDF417, UNICODE_MODE, 30, "가", 0, 30, "U+AC00 in KS X 1001" },
+        /*164*/ { BARCODE_MICROPDF417, UNICODE_MODE, 30, "가가", 0, 30, "U+AC00 in KS X 1001" },
+        /*165*/ { BARCODE_USPS_IMAIL, UNICODE_MODE, 0, "é", ZINT_ERROR_INVALID_DATA, -1, "Numbers/dash only" },
+        /*166*/ { BARCODE_AZTEC, UNICODE_MODE, 0, "é", 0, 0, "" },
+        /*167*/ { BARCODE_AZTEC, UNICODE_MODE, 3, "é", 0, 3, "Supports ECI" },
+        /*168*/ { BARCODE_AZTEC, UNICODE_MODE, 0, "β", ZINT_WARN_USES_ECI, 9, "" },
+        /*169*/ { BARCODE_AZTEC, UNICODE_MODE, 9, "β", 0, 9, "" },
+        /*170*/ { BARCODE_AZTEC, UNICODE_MODE, 20, "テ", 0, 20, "In Shift JIS" },
+        /*171*/ { BARCODE_AZTEC, UNICODE_MODE, 20, "テテ", 0, 20, "In Shift JIS" },
+        /*172*/ { BARCODE_AZTEC, UNICODE_MODE, 25, "က", 0, 25, "In UCS-2BE" },
+        /*173*/ { BARCODE_AZTEC, UNICODE_MODE, 25, "ကက", 0, 25, "In UCS-2BE" },
+        /*174*/ { BARCODE_AZTEC, UNICODE_MODE, 25, "12", 0, 25, "ASCII" },
+        /*175*/ { BARCODE_AZTEC, UNICODE_MODE, 26, "テ", 0, 26, "" },
+        /*176*/ { BARCODE_AZTEC, UNICODE_MODE, 26, "テテ", 0, 26, "" },
+        /*177*/ { BARCODE_AZTEC, UNICODE_MODE, 28, "龘", 0, 28, "U+9F98 in Big5 but not in GB2312" },
+        /*178*/ { BARCODE_AZTEC, UNICODE_MODE, 28, "龘龘", 0, 28, "U+9F98 in Big5 but not in GB2312" },
+        /*179*/ { BARCODE_AZTEC, UNICODE_MODE, 29, "齄", 0, 29, "U+9F44 in GB2312 but not in Big5" },
+        /*180*/ { BARCODE_AZTEC, UNICODE_MODE, 29, "齄齄", 0, 29, "U+9F44 in GB2312 but not in Big5" },
+        /*181*/ { BARCODE_AZTEC, UNICODE_MODE, 30, "가", 0, 30, "U+AC00 in KS X 1001" },
+        /*182*/ { BARCODE_AZTEC, UNICODE_MODE, 30, "가가", 0, 30, "U+AC00 in KS X 1001" },
+        /*183*/ { BARCODE_HIBC_128, UNICODE_MODE, 0, "é", ZINT_ERROR_INVALID_DATA, -1, "HIBC ASCII subset only" },
+        /*184*/ { BARCODE_HIBC_AZTEC, UNICODE_MODE, 0, "é", ZINT_ERROR_INVALID_DATA, -1, "HIBC ASCII subset only" },
+        /*185*/ { BARCODE_DOTCODE, UNICODE_MODE, 0, "é", 0, 0, "" },
+        /*186*/ { BARCODE_DOTCODE, UNICODE_MODE, 3, "é", 0, 3, "Supports ECI" },
+        /*187*/ { BARCODE_DOTCODE, UNICODE_MODE, 0, "β", ZINT_WARN_USES_ECI, 9, "" },
+        /*188*/ { BARCODE_DOTCODE, UNICODE_MODE, 9, "β", 0, 9, "" },
+        /*189*/ { BARCODE_DOTCODE, UNICODE_MODE, 20, "テ", 0, 20, "In Shift JIS" },
+        /*190*/ { BARCODE_DOTCODE, UNICODE_MODE, 20, "テテ", 0, 20, "In Shift JIS" },
+        /*191*/ { BARCODE_DOTCODE, UNICODE_MODE, 25, "က", 0, 25, "In UCS-2BE" },
+        /*192*/ { BARCODE_DOTCODE, UNICODE_MODE, 25, "ကက", 0, 25, "In UCS-2BE" },
+        /*193*/ { BARCODE_DOTCODE, UNICODE_MODE, 25, "12", 0, 25, "ASCII" },
+        /*194*/ { BARCODE_DOTCODE, UNICODE_MODE, 26, "テ", 0, 26, "" },
+        /*195*/ { BARCODE_DOTCODE, UNICODE_MODE, 26, "テテ", 0, 26, "" },
+        /*196*/ { BARCODE_DOTCODE, UNICODE_MODE, 28, "龘", 0, 28, "U+9F98 in Big5 but not in GB2312" },
+        /*197*/ { BARCODE_DOTCODE, UNICODE_MODE, 28, "龘龘", 0, 28, "U+9F98 in Big5 but not in GB2312" },
+        /*198*/ { BARCODE_DOTCODE, UNICODE_MODE, 29, "齄", 0, 29, "U+9F44 in GB2312 but not in Big5" },
+        /*199*/ { BARCODE_DOTCODE, UNICODE_MODE, 29, "齄齄", 0, 29, "U+9F44 in GB2312 but not in Big5" },
+        /*200*/ { BARCODE_DOTCODE, UNICODE_MODE, 30, "가", 0, 30, "U+AC00 in KS X 1001" },
+        /*201*/ { BARCODE_DOTCODE, UNICODE_MODE, 30, "가가", 0, 30, "U+AC00 in KS X 1001" },
+        /*202*/ { BARCODE_AZRUNE, UNICODE_MODE, 0, "é", ZINT_ERROR_INVALID_DATA, -1, "Numbers <= 255 only" },
+        /*203*/ { BARCODE_CODE32, UNICODE_MODE, 0, "é", ZINT_ERROR_INVALID_DATA, -1, "Numbers only" },
+        /*204*/ { BARCODE_CODEONE, UNICODE_MODE, 0, "é", 0, 0, "" },
+        /*205*/ { BARCODE_CODEONE, UNICODE_MODE, 3, "é", ZINT_ERROR_INVALID_OPTION, -1, "Does not support ECI" },
+        /*206*/ { BARCODE_CODEONE, UNICODE_MODE, 0, "β", ZINT_ERROR_INVALID_DATA, -1, "β not in ISO 8859-1" },
+        /*207*/ { BARCODE_ULTRA, UNICODE_MODE, 0, "é", 0, 0, "" },
+        /*208*/ { BARCODE_ULTRA, UNICODE_MODE, 3, "é", 0, 3, "Supports ECI" },
+        /*209*/ { BARCODE_ULTRA, UNICODE_MODE, 0, "β", ZINT_WARN_USES_ECI, 9, "" },
+        /*210*/ { BARCODE_ULTRA, UNICODE_MODE, 9, "β", 0, 9, "" },
+        /*211*/ { BARCODE_ULTRA, UNICODE_MODE, 20, "テ", 0, 20, "In Shift JIS" },
+        /*212*/ { BARCODE_ULTRA, UNICODE_MODE, 20, "テテ", 0, 20, "In Shift JIS" },
+        /*213*/ { BARCODE_ULTRA, UNICODE_MODE, 25, "က", 0, 25, "In UCS-2BE" },
+        /*214*/ { BARCODE_ULTRA, UNICODE_MODE, 25, "ကက", 0, 25, "In UCS-2BE" },
+        /*215*/ { BARCODE_ULTRA, UNICODE_MODE, 25, "12", 0, 25, "ASCII" },
+        /*216*/ { BARCODE_ULTRA, UNICODE_MODE, 26, "テ", 0, 26, "" },
+        /*217*/ { BARCODE_ULTRA, UNICODE_MODE, 26, "テテ", 0, 26, "" },
+        /*218*/ { BARCODE_ULTRA, UNICODE_MODE, 28, "龘", 0, 28, "U+9F98 in Big5 but not in GB2312" },
+        /*219*/ { BARCODE_ULTRA, UNICODE_MODE, 28, "龘龘", 0, 28, "U+9F98 in Big5 but not in GB2312" },
+        /*220*/ { BARCODE_ULTRA, UNICODE_MODE, 29, "齄", 0, 29, "U+9F44 in GB2312 but not in Big5" },
+        /*221*/ { BARCODE_ULTRA, UNICODE_MODE, 29, "齄齄", 0, 29, "U+9F44 in GB2312 but not in Big5" },
+        /*222*/ { BARCODE_ULTRA, UNICODE_MODE, 30, "가", 0, 30, "U+AC00 in KS X 1001" },
+        /*223*/ { BARCODE_ULTRA, UNICODE_MODE, 30, "가가", 0, 30, "U+AC00 in KS X 1001" },
     };
-    int data_size = sizeof(data) / sizeof(struct item);
+    int data_size = ARRAY_SIZE(data);
 
     for (int i = 0; i < data_size; i++) {
 
@@ -222,12 +368,7 @@ static void test_reduced_charset_input(int index, int debug) {
         struct zint_symbol *symbol = ZBarcode_Create();
         assert_nonnull(symbol, "Symbol not created\n");
 
-        symbol->symbology = data[i].symbology;
+        int length = testUtilSetSymbol(symbol, data[i].symbology, data[i].input_mode, data[i].eci, -1 /*option_1*/, -1, -1, -1 /*output_options*/, data[i].data, -1, debug);
-        symbol->input_mode = data[i].input_mode;
-        symbol->eci = data[i].eci;
-        symbol->debug |= debug;
-
-        int length = strlen(data[i].data);
 
         ret = ZBarcode_Encode(symbol, (unsigned char *) data[i].data, length);
         assert_equal(ret, data[i].ret, "i:%d ZBarcode_Encode ret %d != %d (%s)\n", i, ret, data[i].ret, symbol->errtxt);
@@ -242,12 +383,393 @@ static void test_reduced_charset_input(int index, int debug) {
     testFinish();
 }
 
+static int to_utf8(const unsigned int codepoint, unsigned char *buf) {
+    int length = 0;
+
+    if (codepoint < 0x80) {
+        buf[0] = (unsigned char) codepoint;
+        length = 1;
+    } else if (codepoint < 0x800) {
+        buf[0] = (unsigned char) (0xC0 | (codepoint >> 6));
+        buf[1] = (unsigned char) (0x80 | (codepoint & 0x3F));
+        length = 2;
+    } else if (codepoint < 0x10000) {
+        buf[0] = (unsigned char) (0xE0 | (codepoint >> 12));
+        buf[1] = (unsigned char) (0x80 | ((codepoint >> 6) & 0x3F));
+        buf[2] = (unsigned char) (0x80 | (codepoint & 0x3F));
+        length = 3;
+    } else {
+        buf[0] = (unsigned char) (0xF0 | (codepoint >> 18));
+        buf[1] = (unsigned char) (0x80 | ((codepoint >> 12) & 0x3F));
+        buf[2] = (unsigned char) (0x80 | ((codepoint >> 6) & 0x3F));
+        buf[3] = (unsigned char) (0x80 | (codepoint & 0x3F));
+        length = 4;
+    }
+    buf[length] = '\0';
+
+    return length;
+}
+
+// Original eci.h tables
+
+static const unsigned short int iso_8859_1[] = {// Latin alphabet No. 1
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
+    0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf,
+    0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
+    0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df,
+    0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
+    0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff
+};
+
+static const unsigned short int iso_8859_2[] = {// Latin alphabet No. 2
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x00a0, 0x0104, 0x02d8, 0x0141, 0x00a4, 0x013d, 0x015a, 0x00a7, 0x00a8, 0x0160, 0x015e, 0x0164, 0x0179, 0x00ad, 0x017d, 0x017b,
+    0x00b0, 0x0105, 0x02db, 0x0142, 0x00b4, 0x013e, 0x015b, 0x02c7, 0x00b8, 0x0161, 0x015f, 0x0165, 0x017a, 0x02dd, 0x017e, 0x017c,
+    0x0154, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0139, 0x0106, 0x00c7, 0x010c, 0x00c9, 0x0118, 0x00cb, 0x011a, 0x00cd, 0x00ce, 0x010e,
+    0x0110, 0x0143, 0x0147, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x00d7, 0x0158, 0x016e, 0x00da, 0x0170, 0x00dc, 0x00dd, 0x0162, 0x00df,
+    0x0155, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x013a, 0x0107, 0x00e7, 0x010d, 0x00e9, 0x0119, 0x00eb, 0x011b, 0x00ed, 0x00ee, 0x010f,
+    0x0111, 0x0144, 0x0148, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x00f7, 0x0159, 0x016f, 0x00fa, 0x0171, 0x00fc, 0x00fd, 0x0163, 0x02d9
+};
+
+static const unsigned short int iso_8859_3[] = {// Latin alphabet No. 3
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x00a0, 0x0126, 0x02d8, 0x00a3, 0x00a4, 0x0000, 0x0124, 0x00a7, 0x00a8, 0x0130, 0x015e, 0x011e, 0x0134, 0x00ad, 0x0000, 0x017b,
+    0x00b0, 0x0127, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x0125, 0x00b7, 0x00b8, 0x0131, 0x015f, 0x011f, 0x0135, 0x00bd, 0x0000, 0x017c,
+    0x00c0, 0x00c1, 0x00c2, 0x0000, 0x00c4, 0x010a, 0x0108, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
+    0x0000, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x0120, 0x00d6, 0x00d7, 0x011c, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x016c, 0x015c, 0x00df,
+    0x00e0, 0x00e1, 0x00e2, 0x0000, 0x00e4, 0x010b, 0x0109, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
+    0x0000, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x0121, 0x00f6, 0x00f7, 0x011d, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x016d, 0x015d, 0x02d9
+};
+
+static const unsigned short int iso_8859_4[] = {// Latin alphabet No. 4
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x00a0, 0x0104, 0x0138, 0x0156, 0x00a4, 0x0128, 0x013b, 0x00a7, 0x00a8, 0x0160, 0x0112, 0x0122, 0x0166, 0x00ad, 0x017d, 0x00af, // A5 0x012b -> 0x0128
+    0x00b0, 0x0105, 0x02db, 0x0157, 0x00b4, 0x0129, 0x013c, 0x02c7, 0x00b8, 0x0161, 0x0113, 0x0123, 0x0167, 0x014a, 0x017e, 0x014b,
+    0x0100, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x012e, 0x010c, 0x00c9, 0x0118, 0x00cb, 0x0116, 0x00cd, 0x00ce, 0x012a,
+    0x0110, 0x0145, 0x014c, 0x0136, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x0172, 0x00da, 0x00db, 0x00dc, 0x0168, 0x016a, 0x00df,
+    0x0101, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x012f, 0x010d, 0x00e9, 0x0119, 0x00eb, 0x0117, 0x00ed, 0x00ee, 0x012b,
+    0x0111, 0x0146, 0x014d, 0x0137, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x0173, 0x00fa, 0x00fb, 0x00fc, 0x0169, 0x016b, 0x02d9
+};
+
+static const unsigned short int iso_8859_5[] = {// Latin/Cyrillic alphabet
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x00a0, 0x0401, 0x0402, 0x0403, 0x0404, 0x0405, 0x0406, 0x0407, 0x0408, 0x0409, 0x040a, 0x040b, 0x040c, 0x00ad, 0x040e, 0x040f,
+    0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f,
+    0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f,
+    0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f,
+    0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f,
+    0x2116, 0x0451, 0x0452, 0x0453, 0x0454, 0x0455, 0x0456, 0x0457, 0x0458, 0x0459, 0x045a, 0x045b, 0x045c, 0x00a7, 0x045e, 0x045f
+};
+
+static const unsigned short int iso_8859_6[] = {// Latin/Arabic alphabet
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x00a0, 0x0000, 0x0000, 0x0000, 0x00a4, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x060c, 0x00ad, 0x0000, 0x0000,
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x061b, 0x0000, 0x0000, 0x0000, 0x061f,
+    0x0000, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, 0x0628, 0x0629, 0x062a, 0x062b, 0x062c, 0x062d, 0x062e, 0x062f,
+    0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x0637, 0x0638, 0x0639, 0x063a, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x0640, 0x0641, 0x0642, 0x0643, 0x0644, 0x0645, 0x0646, 0x0647, 0x0648, 0x0649, 0x064a, 0x064b, 0x064c, 0x064d, 0x064e, 0x064f,
+    0x0650, 0x0651, 0x0652, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000
+};
+
+static const unsigned short int iso_8859_7[] = {// Latin/Greek alphabet
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x00a0, 0x2018, 0x2019, 0x00a3, 0x20ac, 0x20af, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x037a, 0x00ab, 0x00ac, 0x00ad, 0x0000, 0x2015,
+    0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x0384, 0x0385, 0x0386, 0x00b7, 0x0388, 0x0389, 0x038a, 0x00bb, 0x038c, 0x00bd, 0x038e, 0x038f,
+    0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, 0x0398, 0x0399, 0x039a, 0x039b, 0x039c, 0x039d, 0x039e, 0x039f,
+    0x03a0, 0x03a1, 0x0000, 0x03a3, 0x03a4, 0x03a5, 0x03a6, 0x03a7, 0x03a8, 0x03a9, 0x03aa, 0x03ab, 0x03ac, 0x03ad, 0x03ae, 0x03af,
+    0x03b0, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf,
+    0x03c0, 0x03c1, 0x03c2, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, 0x03c8, 0x03c9, 0x03ca, 0x03cb, 0x03cc, 0x03cd, 0x03ce, 0x0000
+};
+
+static const unsigned short int iso_8859_8[] = {// Latin/Hebrew alphabet
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x00a0, 0x0000, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00d7, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
+    0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00f7, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x0000,
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2017,
+    0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x05d5, 0x05d6, 0x05d7, 0x05d8, 0x05d9, 0x05da, 0x05db, 0x05dc, 0x05dd, 0x05de, 0x05df,
+    0x05e0, 0x05e1, 0x05e2, 0x05e3, 0x05e4, 0x05e5, 0x05e6, 0x05e7, 0x05e8, 0x05e9, 0x05ea, 0x0000, 0x0000, 0x200e, 0x200f, 0x0000
+};
+
+static const unsigned short int iso_8859_9[] = {// Latin alphabet No. 5
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
+    0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf,
+    0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
+    0x011e, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x0130, 0x015e, 0x00df,
+    0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
+    0x011f, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x0131, 0x015f, 0x00ff
+};
+
+static const unsigned short int iso_8859_10[] = {// Latin alphabet No. 6
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x00a0, 0x0104, 0x0112, 0x0122, 0x012a, 0x0128, 0x0136, 0x00a7, 0x013b, 0x0110, 0x0160, 0x0166, 0x017d, 0x00ad, 0x016a, 0x014a, // A5 0x012b -> 0x0128
+    0x00b0, 0x0105, 0x0113, 0x0123, 0x012b, 0x0129, 0x0137, 0x00b7, 0x013c, 0x0111, 0x0161, 0x0167, 0x017e, 0x2015, 0x016b, 0x014b,
+    0x0100, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x012e, 0x010c, 0x00c9, 0x0118, 0x00cb, 0x0116, 0x00cd, 0x00ce, 0x00cf,
+    0x00d0, 0x0145, 0x014c, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x0168, 0x00d8, 0x0172, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df,
+    0x0101, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x012f, 0x010d, 0x00e9, 0x0119, 0x00eb, 0x0117, 0x00ed, 0x00ee, 0x00ef,
+    0x00f0, 0x0146, 0x014d, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x0169, 0x00f8, 0x0173, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x0138
+};
+
+static const unsigned short int iso_8859_11[] = {// Latin/Thai alphabet
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x00a0, 0x0e01, 0x0e02, 0x0e03, 0x0e04, 0x0e05, 0x0e06, 0x0e07, 0x0e08, 0x0e09, 0x0e0a, 0x0e0b, 0x0e0c, 0x0e0d, 0x0e0e, 0x0e0f,
+    0x0e10, 0x0e11, 0x0e12, 0x0e13, 0x0e14, 0x0e15, 0x0e16, 0x0e17, 0x0e18, 0x0e19, 0x0e1a, 0x0e1b, 0x0e1c, 0x0e1d, 0x0e1e, 0x0e1f,
+    0x0e20, 0x0e21, 0x0e22, 0x0e23, 0x0e24, 0x0e25, 0x0e26, 0x0e27, 0x0e28, 0x0e29, 0x0e2a, 0x0e2b, 0x0e2c, 0x0e2d, 0x0e2e, 0x0e2f,
+    0x0e30, 0x0e31, 0x0e32, 0x0e33, 0x0e34, 0x0e35, 0x0e36, 0x0e37, 0x0e38, 0x0e39, 0x0e3a, 0x0000, 0x0000, 0x0000, 0x0000, 0x0e3f, // D5 0x0e36 -> 0x0e35
+    0x0e40, 0x0e41, 0x0e42, 0x0e43, 0x0e44, 0x0e45, 0x0e46, 0x0e47, 0x0e48, 0x0e49, 0x0e4a, 0x0e4b, 0x0e4c, 0x0e4d, 0x0e4e, 0x0e4f,
+    0x0e50, 0x0e51, 0x0e52, 0x0e53, 0x0e54, 0x0e55, 0x0e56, 0x0e57, 0x0e58, 0x0e59, 0x0e5a, 0x0e5b, 0x0000, 0x0000, 0x0000, 0x0000
+};
+
+static const unsigned short int iso_8859_13[] = {// Latin alphabet No. 7
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x00a0, 0x201d, 0x00a2, 0x00a3, 0x00a4, 0x201e, 0x00a6, 0x00a7, 0x00d8, 0x00a9, 0x0156, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00c6,
+    0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x201c, 0x00b5, 0x00b6, 0x00b7, 0x00f8, 0x00b9, 0x0157, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00e6,
+    0x0104, 0x012e, 0x0100, 0x0106, 0x00c4, 0x00c5, 0x0118, 0x0112, 0x010c, 0x00c9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012a, 0x013b,
+    0x0160, 0x0143, 0x0145, 0x00d3, 0x014c, 0x00d5, 0x00d6, 0x00d7, 0x0172, 0x0141, 0x015a, 0x016a, 0x00dc, 0x017b, 0x017d, 0x00df,
+    0x0105, 0x012f, 0x0101, 0x0107, 0x00e4, 0x00e5, 0x0119, 0x0113, 0x010d, 0x00e9, 0x017a, 0x0117, 0x0123, 0x0137, 0x012b, 0x013c,
+    0x0161, 0x0144, 0x0146, 0x00f3, 0x014d, 0x00f5, 0x00f6, 0x00f7, 0x0173, 0x0142, 0x015b, 0x016b, 0x00fc, 0x017c, 0x017e, 0x2019
+};
+
+static const unsigned short int iso_8859_14[] = {// Latin alphabet No. 8 (Celtic)
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x00a0, 0x1e02, 0x1e03, 0x00a3, 0x010a, 0x010b, 0x1e0a, 0x00a7, 0x1e80, 0x00a9, 0x1e82, 0x1e0b, 0x1ef2, 0x00ad, 0x00ae, 0x0178,
+    0x1e1e, 0x1e1f, 0x0120, 0x0121, 0x1e40, 0x1e41, 0x00b6, 0x1e56, 0x1e81, 0x1e57, 0x1e83, 0x1e60, 0x1ef3, 0x1e84, 0x1e85, 0x1e61,
+    0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
+    0x0174, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x1e6a, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x0176, 0x00df,
+    0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
+    0x0175, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x1e6b, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x0177, 0x00ff
+};
+
+static const unsigned short int iso_8859_15[] = {// Latin alphabet No. 9
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x20ac, 0x00a5, 0x0160, 0x00a7, 0x0161, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
+    0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x017d, 0x00b5, 0x00b6, 0x00b7, 0x017e, 0x00b9, 0x00ba, 0x00bb, 0x0152, 0x0153, 0x0178, 0x00bf,
+    0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
+    0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df,
+    0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
+    0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff
+};
+
+static const unsigned short int iso_8859_16[] = {// Latin alphabet No. 10
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+    0x00a0, 0x0104, 0x0105, 0x0141, 0x20ac, 0x201e, 0x0160, 0x00a7, 0x0161, 0x00a9, 0x0218, 0x00ab, 0x0179, 0x00ad, 0x017a, 0x017b,
+    0x00b0, 0x00b1, 0x010c, 0x0142, 0x017d, 0x201d, 0x00b6, 0x00b7, 0x017e, 0x010d, 0x0219, 0x00bb, 0x0152, 0x0153, 0x0178, 0x017c,
+    0x00c0, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0106, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
+    0x0110, 0x0143, 0x00d2, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x015a, 0x0170, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x0118, 0x021a, 0x00df,
+    0x00e0, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x0107, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
+    0x0111, 0x0144, 0x00f2, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x015b, 0x0171, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x0119, 0x021b, 0x00ff
+};
+
+static const unsigned short int windows_1250[] = {
+    0x20ac, 0x0000, 0x201a, 0x0000, 0x201e, 0x2026, 0x2020, 0x2021, 0x0000, 0x2030, 0x0160, 0x2039, 0x015a, 0x0164, 0x017d, 0x0179,
+    0x0000, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x0000, 0x2122, 0x0161, 0x203a, 0x015b, 0x0165, 0x017e, 0x017a,
+    0x00a0, 0x02c7, 0x02d8, 0x0141, 0x00a4, 0x0104, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x015e, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x017b, // A2 0x02db -> 0x02d8
+    0x00b0, 0x00b1, 0x02db, 0x0142, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x0105, 0x015f, 0x00bb, 0x013d, 0x02dd, 0x013e, 0x017c,
+    0x0154, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0139, 0x0106, 0x00c7, 0x010c, 0x00c9, 0x0118, 0x00cb, 0x011a, 0x00cd, 0x00ce, 0x010e,
+    0x0110, 0x0143, 0x0147, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x00d7, 0x0158, 0x016e, 0x00da, 0x0170, 0x00dc, 0x00dd, 0x0162, 0x00df,
+    0x0155, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x013a, 0x0107, 0x00e7, 0x010d, 0x00e9, 0x0119, 0x00eb, 0x011b, 0x00ed, 0x00ee, 0x010f,
+    0x0111, 0x0144, 0x0148, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x00f7, 0x0159, 0x016f, 0x00fa, 0x0171, 0x00fc, 0x00fd, 0x0163, 0x02d9
+};
+
+static const unsigned short int windows_1251[] = {
+    0x0402, 0x0403, 0x201a, 0x0453, 0x201e, 0x2026, 0x2020, 0x2021, 0x20ac, 0x2030, 0x0409, 0x2039, 0x040a, 0x040c, 0x040b, 0x040f,
+    0x0452, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x0000, 0x2122, 0x0459, 0x203a, 0x045a, 0x045c, 0x045b, 0x045f,
+    0x00a0, 0x040e, 0x045e, 0x0408, 0x00a4, 0x0490, 0x00a6, 0x00a7, 0x0401, 0x00a9, 0x0404, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x0407,
+    0x00b0, 0x00b1, 0x0406, 0x0456, 0x0491, 0x00b5, 0x00b6, 0x00b7, 0x0451, 0x2116, 0x0454, 0x00bb, 0x0458, 0x0405, 0x0455, 0x0457,
+    0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f,
+    0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f,
+    0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f,
+    0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f
+};
+
+static const unsigned short int windows_1252[] = {
+    0x20ac, 0x0000, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0x02c6, 0x2030, 0x0160, 0x2039, 0x0152, 0x0000, 0x017d, 0x0000,
+    0x0000, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, 0x0000, 0x017e, 0x0178,
+    0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
+    0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf,
+    0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
+    0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df,
+    0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
+    0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff
+};
+
+static const unsigned short int windows_1256[] = {
+    0x20ac, 0x067e, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0x02c6, 0x2030, 0x0679, 0x2039, 0x0152, 0x0686, 0x0698, 0x0688,
+    0x06af, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x06a9, 0x2122, 0x0691, 0x203a, 0x0153, 0x200c, 0x200d, 0x06ba,
+    0x00a0, 0x060c, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x06be, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af,
+    0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x061b, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x061f,
+    0x06c1, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, 0x0628, 0x0629, 0x062a, 0x062b, 0x062c, 0x062d, 0x062e, 0x062f,
+    0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x00d7, 0x0637, 0x0638, 0x0639, 0x063a, 0x0640, 0x0641, 0x0642, 0x0643,
+    0x00e0, 0x0644, 0x00e2, 0x0645, 0x0646, 0x0647, 0x0648, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x0649, 0x064a, 0x00ee, 0x00ef,
+    0x064b, 0x064c, 0x064d, 0x064e, 0x00f4, 0x064f, 0x0650, 0x00f7, 0x0651, 0x00f9, 0x0652, 0x00fb, 0x00fc, 0x200e, 0x200f, 0x06d2
+};
+
+static void test_utf8_to_eci_sb(int index) {
+
+    testStart("");
+
+    int ret;
+    struct item {
+        int eci;
+        const unsigned short *tab;
+    };
+    // s/\/\*[ 0-9]*\*\//\=printf("\/*%3d*\/", line(".") - line("'<"))
+    struct item data[] = {
+        /*  0*/ { 3, iso_8859_1 },
+        /*  1*/ { 4, iso_8859_2 },
+        /*  2*/ { 5, iso_8859_3 },
+        /*  3*/ { 6, iso_8859_4 },
+        /*  4*/ { 7, iso_8859_5 },
+        /*  5*/ { 8, iso_8859_6 },
+        /*  6*/ { 9, iso_8859_7 },
+        /*  7*/ { 10, iso_8859_8 },
+        /*  8*/ { 11, iso_8859_9 },
+        /*  9*/ { 12, iso_8859_10 },
+        /* 10*/ { 13, iso_8859_11 },
+        /* 11*/ { 15, iso_8859_13 },
+        /* 12*/ { 16, iso_8859_14 },
+        /* 13*/ { 17, iso_8859_15 },
+        /* 14*/ { 18, iso_8859_16 },
+        /* 15*/ { 21, windows_1250 },
+        /* 16*/ { 22, windows_1251 },
+        /* 17*/ { 23, windows_1252 },
+        /* 18*/ { 24, windows_1256 },
+    };
+    int data_size = ARRAY_SIZE(data);
+
+    unsigned char source[5];
+    unsigned char dest[2] = {0};
+
+    for (int i = 0; i < data_size; i++) {
+
+        if (index != -1 && i != index) continue;
+
+        for (int j = 0; j < 128; j++) {
+            if (data[i].tab[j]) {
+                int k = j + 128;
+                int length = to_utf8(data[i].tab[j], source);
+                assert_nonzero(length, "i:%d to_utf8 length %d == 0\n", i, length);
+                ret = utf8_to_eci(data[i].eci, source, dest, &length);
+                assert_zero(ret, "i:%d utf8_to_eci ret %d != 0\n", i, ret);
+                assert_equal(*dest, k, "i:%d j:%d eci:%d codepoint:0x%x *dest 0x%X (%d) != 0x%X (%d)\n", i, j, data[i].eci, data[i].tab[j], *dest, *dest, k, k);
+            }
+        }
+    }
+
+    testFinish();
+}
+
+static void test_utf8_to_eci_ascii(void) {
+
+    testStart("");
+
+    int ret;
+    struct item {
+        int eci;
+        char *data;
+        int length;
+        int ret;
+    };
+    // s/\/\*[ 0-9]*\*\//\=printf("\/*%3d*\/", line(".") - line("'<"))
+    struct item data[] = {
+        /*  0*/ { 27, "\000\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037", 32, 0 },
+        /*  1*/ { 27, " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\177", 96, 0 },
+        /*  2*/ { 27, "\302\200", -1, ZINT_ERROR_INVALID_DATA },
+        /*  3*/ { 170, "\000\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037", 32, 0 },
+        /*  4*/ { 170, " !\"  %&'()*+,-./0123456789:;<=>? ABCDEFGHIJKLMNOPQRSTUVWXYZ    _ abcdefghijklmnopqrstuvwxyz    \177", 96, 0 },
+        /*  5*/ { 170, "#", -1, ZINT_ERROR_INVALID_DATA },
+        /*  6*/ { 170, "$", -1, ZINT_ERROR_INVALID_DATA },
+        /*  7*/ { 170, "@", -1, ZINT_ERROR_INVALID_DATA },
+        /*  8*/ { 170, "[", -1, ZINT_ERROR_INVALID_DATA },
+        /*  9*/ { 170, "\\", -1, ZINT_ERROR_INVALID_DATA },
+        /* 10*/ { 170, "]", -1, ZINT_ERROR_INVALID_DATA },
+        /* 11*/ { 170, "^", -1, ZINT_ERROR_INVALID_DATA },
+        /* 12*/ { 170, "`", -1, ZINT_ERROR_INVALID_DATA },
+        /* 13*/ { 170, "{", -1, ZINT_ERROR_INVALID_DATA },
+        /* 14*/ { 170, "|", -1, ZINT_ERROR_INVALID_DATA },
+        /* 15*/ { 170, "}", -1, ZINT_ERROR_INVALID_DATA },
+        /* 16*/ { 170, "~", -1, ZINT_ERROR_INVALID_DATA },
+        /* 17*/ { 170, "\302\200", -1, ZINT_ERROR_INVALID_DATA },
+    };
+    int data_size = ARRAY_SIZE(data);
+
+    char dest[128];
+
+    for (int i = 0; i < data_size; i++) {
+        int length = data[i].length != -1 ? data[i].length : (int) strlen(data[i].data);
+        int out_length = length;
+        ret = utf8_to_eci(data[i].eci, (const unsigned char *) data[i].data, (unsigned char *) dest, &out_length);
+        assert_equal(ret, data[i].ret, "i:%d utf8_to_eci ret %d != %d\n", i, ret, data[i].ret);
+        if (ret == 0) {
+            assert_equal(length, out_length, "i:%d length %d != %d\n", i, length, out_length);
+            assert_zero(memcmp(data[i].data, dest, length), "i:%d memcmp != 0\n", i);
+        }
+    }
+};
+
+static void test_utf8_to_eci_ucs2be(void) {
+
+    testStart("");
+
+    int ret;
+    struct item {
+        int eci;
+        char *data;
+        int length;
+        int ret;
+        int expected_length;
+    };
+    // s/\/\*[ 0-9]*\*\//\=printf("\/*%3d*\/", line(".") - line("'<"))
+    struct item data[] = {
+        /*  0*/ { 25, "\000\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037", 32, 0, 32 * 2 },
+        /*  1*/ { 25, " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\177", 96, 0, 96 * 2 },
+        /*  2*/ { 25, "\302\200\357\277\277", -1, 0, 4 }, // U+0080 U+FFFF
+        /*  3*/ { 25, "\357\277\276", -1, ZINT_ERROR_INVALID_DATA, -1 }, // U+FFFE (reversed BOM) not allowed
+        /*  4*/ { 25, "\355\240\200", -1, ZINT_ERROR_INVALID_DATA, -1 }, // U+D800 surrogate not allowed
+    };
+    int data_size = ARRAY_SIZE(data);
+
+    for (int i = 0; i < data_size; i++) {
+        int length = data[i].length != -1 ? data[i].length : (int) strlen(data[i].data);
+        int out_length = length;
+        int eci_length = get_eci_length(data[i].eci, (const unsigned char *) data[i].data, length);
+        char dest[eci_length + 1];
+
+        ret = utf8_to_eci(data[i].eci, (const unsigned char *) data[i].data, (unsigned char *) dest, &out_length);
+        assert_equal(ret, data[i].ret, "i:%d utf8_to_eci ret %d != %d\n", i, ret, data[i].ret);
+        if (ret == 0) {
+            assert_equal(out_length, data[i].expected_length, "i:%d length %d != %d\n", i, out_length, data[i].expected_length);
+            assert_nonzero(out_length <= eci_length, "i:%d out_length %d > eci_length %d\n", i, out_length, eci_length);
+        }
+    }
+};
+
 int main(int argc, char *argv[]) {
 
     testFunction funcs[] = { /* name, func, has_index, has_generate, has_debug */
         { "test_bom", test_bom, 0, 0, 1 },
         { "test_iso_8859_16", test_iso_8859_16, 0, 0, 1 },
         { "test_reduced_charset_input", test_reduced_charset_input, 1, 0, 1 },
+        { "test_utf8_to_eci_sb", test_utf8_to_eci_sb, 1, 0, 0 },
+        { "test_utf8_to_eci_ascii", test_utf8_to_eci_ascii, 0, 0, 0 },
+        { "test_utf8_to_eci_ucs2be", test_utf8_to_eci_ucs2be, 0, 0, 0 },
     };
 
     testRun(argc, argv, funcs, ARRAY_SIZE(funcs));

backend/tests/test_gb18030.c

@@ -1,6 +1,6 @@
 /*
     libzint - the open source barcode library
-    Copyright (C) 2019 - 2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2019 - 2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -107,7 +107,9 @@ static int gb18030_wctomb_zint2(unsigned int *r1, unsigned int *r2, unsigned int
         return 4;
     }
     int tab_length = sizeof(test_gb18030_tab) / sizeof(unsigned int);
-    for (int i = test_gb18030_tab_ind[wc >> 12]; i < tab_length; i += 2) {
+    int start_i = test_gb18030_tab_ind[wc >> 10];
+    int end_i = start_i + 0x800 > tab_length ? tab_length : start_i + 0x800;
+    for (int i = start_i; i < end_i; i += 2) {
         if (test_gb18030_tab[i + 1] == wc) {
             c = test_gb18030_tab[i];
             if (c <= 0xFFFF) {
@@ -146,7 +148,7 @@ static void test_gb18030_wctomb_zint(void) {
     testFinish();
 }
 
-static void test_gb18030_utf8tomb(int index) {
+static void test_gb18030_utf8(int index) {
 
     testStart("");
 
@@ -192,7 +194,7 @@ static void test_gb18030_utf8tomb(int index) {
         int length = data[i].length == -1 ? (int) strlen(data[i].data) : data[i].length;
         int ret_length = length;
 
-        ret = gb18030_utf8tomb(&symbol, (unsigned char *) data[i].data, &ret_length, gbdata);
+        ret = gb18030_utf8(&symbol, (unsigned char *) data[i].data, &ret_length, gbdata);
         assert_equal(ret, data[i].ret, "i:%d ret %d != %d (%s)\n", i, ret, data[i].ret, symbol.errtxt);
         if (ret == 0) {
             assert_equal(ret_length, data[i].ret_length, "i:%d ret_length %d != %d\n", i, ret_length, data[i].ret_length);
@@ -205,7 +207,7 @@ static void test_gb18030_utf8tomb(int index) {
     testFinish();
 }
 
-static void test_gb18030_utf8tosb(int index) {
+static void test_gb18030_utf8_to_eci(int index) {
 
     testStart("");
 
@@ -228,32 +230,58 @@ static void test_gb18030_utf8tosb(int index) {
     // 9 U+0039 in ASCII 0x39, outside first byte range, outside double-byte second byte range and quad-byte third byte range, in quad-byte second/fourth byte ranges
     // s/\/\*[ 0-9]*\*\//\=printf("\/*%3d*\/", line(".") - line("'<"))
     struct item data[] = {
-        /*  1*/ { 3, 0, "é", -1, 0, 1, { 0xE9 }, "Not full multibyte" },
+        /*  0*/ { 3, 0, "é", -1, 0, 1, { 0xE9 }, "Not full multibyte" },
-        /*  2*/ { 3, 1, "é", -1, 0, 1, { 0xE9 }, "First byte in range but only one byte" },
+        /*  1*/ { 3, 1, "é", -1, 0, 1, { 0xE9 }, "First byte in range but only one byte" },
-        /*  3*/ { 3, 0, "β", -1, ZINT_ERROR_INVALID_DATA, -1, {}, "Not full multibyte" },
+        /*  2*/ { 3, 0, "β", -1, ZINT_ERROR_INVALID_DATA, -1, {}, "Not full multibyte" },
-        /*  4*/ { 3, 1, "β", -1, ZINT_ERROR_INVALID_DATA, -1, {}, "Not in ECI 3 (ISO 8859-1)" },
+        /*  3*/ { 3, 1, "β", -1, ZINT_ERROR_INVALID_DATA, -1, {}, "Not in ECI 3 (ISO 8859-1)" },
-        /*  5*/ { 9, 0, "β", -1, 0, 1, { 0xE2 }, "Not full multibyte" },
+        /*  4*/ { 9, 0, "β", -1, 0, 1, { 0xE2 }, "Not full multibyte" },
-        /*  6*/ { 9, 1, "β", -1, 0, 1, { 0xE2 }, "In ECI 9 (ISO 8859-7)" },
+        /*  5*/ { 9, 1, "β", -1, 0, 1, { 0xE2 }, "In ECI 9 (ISO 8859-7)" },
-        /*  7*/ { 3, 0, "¥", -1, 0, 1, { 0xA5 }, "Not full multibyte" },
+        /*  6*/ { 3, 0, "¥", -1, 0, 1, { 0xA5 }, "Not full multibyte" },
-        /*  8*/ { 3, 1, "¥", -1, 0, 1, { 0xA5 }, "First byte in range but only one byte" },
+        /*  7*/ { 3, 1, "¥", -1, 0, 1, { 0xA5 }, "First byte in range but only one byte" },
-        /*  9*/ { 3, 0, "¥é", -1, 0, 2, { 0xA5, 0xE9 }, "Not full multibyte" },
+        /*  8*/ { 3, 0, "¥é", -1, 0, 2, { 0xA5, 0xE9 }, "Not full multibyte" },
-        /* 10*/ { 3, 1, "¥é", -1, 0, 1, { 0xA5E9 }, "In double-byte range" },
+        /*  9*/ { 3, 1, "¥é", -1, 0, 1, { 0xA5E9 }, "In double-byte range" },
-        /* 11*/ { 3, 0, "¥ÿ", -1, 0, 2, { 0xA5, 0xFF }, "Not full multibyte" },
+        /* 10*/ { 3, 0, "¥ÿ", -1, 0, 2, { 0xA5, 0xFF }, "Not full multibyte" },
-        /* 12*/ { 3, 1, "¥ÿ", -1, 0, 2, { 0xA5, 0xFF }, "First byte in range but not second" },
+        /* 11*/ { 3, 1, "¥ÿ", -1, 0, 2, { 0xA5, 0xFF }, "First byte in range but not second" },
-        /* 13*/ { 3, 0, "¥9é9", -1, 0, 4, { 0xA5, 0x39, 0xE9, 0x39 }, "Not full multibyte" },
+        /* 12*/ { 3, 0, "¥9é9", -1, 0, 4, { 0xA5, 0x39, 0xE9, 0x39 }, "Not full multibyte" },
-        /* 14*/ { 3, 1, "¥9é9", -1, 0, 2, { 0xA539, 0xE939 }, "In quad-byte range" },
+        /* 13*/ { 3, 1, "¥9é9", -1, 0, 2, { 0xA539, 0xE939 }, "In quad-byte range" },
-        /* 15*/ { 3, 0, "¥9", -1, 0, 2, { 0xA5, 0x39 }, "Not full multibyte" },
+        /* 14*/ { 3, 0, "¥9", -1, 0, 2, { 0xA5, 0x39 }, "Not full multibyte" },
-        /* 16*/ { 3, 1, "¥9", -1, 0, 2, { 0xA5, 0x39 }, "In quad-byte first/second range but only 2 bytes, not in double-byte range" },
+        /* 15*/ { 3, 1, "¥9", -1, 0, 2, { 0xA5, 0x39 }, "In quad-byte first/second range but only 2 bytes, not in double-byte range" },
-        /* 17*/ { 3, 0, "¥9é", -1, 0, 3, { 0xA5, 0x39, 0xE9 }, "Not full multibyte" },
+        /* 16*/ { 3, 0, "¥9é", -1, 0, 3, { 0xA5, 0x39, 0xE9 }, "Not full multibyte" },
-        /* 18*/ { 3, 1, "¥9é", -1, 0, 3, { 0xA5, 0x39, 0xE9 }, "In quad-byte first/second/third range but only 3 bytes, no bytes in double-byte range" },
+        /* 17*/ { 3, 1, "¥9é", -1, 0, 3, { 0xA5, 0x39, 0xE9 }, "In quad-byte first/second/third range but only 3 bytes, no bytes in double-byte range" },
-        /* 19*/ { 3, 0, "¥9é@", -1, 0, 4, { 0xA5, 0x39, 0xE9, 0x40 }, "Not full multibyte" },
+        /* 18*/ { 3, 0, "¥9é@", -1, 0, 4, { 0xA5, 0x39, 0xE9, 0x40 }, "Not full multibyte" },
-        /* 20*/ { 3, 1, "¥9é@", -1, 0, 3, { 0xA5, 0x39, 0xE940 }, "In quad-byte first/second/third range but not fourth, second 2 bytes in double-byte range" },
+        /* 19*/ { 3, 1, "¥9é@", -1, 0, 3, { 0xA5, 0x39, 0xE940 }, "In quad-byte first/second/third range but not fourth, second 2 bytes in double-byte range" },
-        /* 21*/ { 3, 0, "¥@é9", -1, 0, 4, { 0xA5, 0x40, 0xE9, 0x39 }, "Not full multibyte" },
+        /* 20*/ { 3, 0, "¥@é9", -1, 0, 4, { 0xA5, 0x40, 0xE9, 0x39 }, "Not full multibyte" },
-        /* 22*/ { 3, 1, "¥@é9", -1, 0, 3, { 0xA540, 0xE9, 0x39 }, "In quad-byte first/third/fourth range but not second, first 2 bytes in double-byte range" },
+        /* 21*/ { 3, 1, "¥@é9", -1, 0, 3, { 0xA540, 0xE9, 0x39 }, "In quad-byte first/third/fourth range but not second, first 2 bytes in double-byte range" },
-        /* 23*/ { 3, 0, "¥9@9", -1, 0, 4, { 0xA5, 0x39, 0x40, 0x39 }, "Not full multibyte" },
+        /* 22*/ { 3, 0, "¥9@9", -1, 0, 4, { 0xA5, 0x39, 0x40, 0x39 }, "Not full multibyte" },
-        /* 24*/ { 3, 1, "¥9@9", -1, 0, 4, { 0xA5, 0x39, 0x40, 0x39 }, "In quad-byte first/second/fourth range but not third, no bytes in double-byte range" },
+        /* 23*/ { 3, 1, "¥9@9", -1, 0, 4, { 0xA5, 0x39, 0x40, 0x39 }, "In quad-byte first/second/fourth range but not third, no bytes in double-byte range" },
-        /* 25*/ { 3, 0, "é9éé¥9é@¥9é9¥9é0é@@¥¥é0é1", -1, 0, 25, { 0xE9, 0x39, 0xE9, 0xE9, 0xA5, 0x39, 0xE9, 0x40, 0xA5, 0x39, 0xE9, 0x39, 0xA5, 0x39, 0xE9, 0x30, 0xE9, 0x40, 0x40, 0xA5, 0xA5, 0xE9, 0x30, 0xE9, 0x31 }, "" },
+        /* 24*/ { 3, 0, "é9éé¥9é@¥9é9¥9é0é@@¥¥é0é1", -1, 0, 25, { 0xE9, 0x39, 0xE9, 0xE9, 0xA5, 0x39, 0xE9, 0x40, 0xA5, 0x39, 0xE9, 0x39, 0xA5, 0x39, 0xE9, 0x30, 0xE9, 0x40, 0x40, 0xA5, 0xA5, 0xE9, 0x30, 0xE9, 0x31 }, "" },
-        /* 26*/ { 3, 1, "é9éé¥9é@¥9é9¥9é0é@@¥¥é0é1", -1, 0, 15, { 0xE9, 0x39, 0xE9E9, 0xA5, 0x39, 0xE940, 0xA539, 0xE939, 0xA539, 0xE930, 0xE940, 0x40, 0xA5A5, 0xE930, 0xE931 }, "" },
+        /* 25*/ { 3, 1, "é9éé¥9é@¥9é9¥9é0é@@¥¥é0é1", -1, 0, 15, { 0xE9, 0x39, 0xE9E9, 0xA5, 0x39, 0xE940, 0xA539, 0xE939, 0xA539, 0xE930, 0xE940, 0x40, 0xA5A5, 0xE930, 0xE931 }, "" },
+        /* 26*/ { 20, 0, "\\\\", -1, 0, 4, { 0x81, 0x5F, 0x81, 0x5F }, "Shift JIS reverse solidus (backslash) mapping from ASCII to double byte" },
+        /* 27*/ { 20, 1, "\\\\", -1, 0, 2, { 0x815F, 0x815F }, "Shift JIS in GB 18030 Hanzi mode range" },
+        /* 28*/ { 20, 0, "爍", -1, 0, 2, { 0xE0, 0xA1 }, "Shift JIS U+720D" },
+        /* 29*/ { 20, 1, "爍", -1, 0, 1, { 0xE0A1 }, "Shift JIS in GB 18030 Hanzi mode range" },
+        /* 30*/ { 25, 0, "12", -1, 0, 4, { 0x00, 0x31, 0x00, 0x32 }, "UCS-2BE ASCII" },
+        /* 31*/ { 25, 0, "", -1, 0, 4, { 0x00, 0x81, 0x00, 0x81 }, "UCS-2BE U+0081" },
+        /* 32*/ { 25, 1, "", -1, 0, 4, { 0x00, 0x81, 0x00, 0x81 }, "UCS-2BE outside GB 18030 Hanzi mode range" },
+        /* 33*/ { 25, 0, "ꆩꆩ", -1, 0, 4, { 0xA1, 0xA9, 0xA1, 0xA9 }, "UCS-2BE U+A1A9" },
+        /* 34*/ { 25, 1, "ꆩꆩ", -1, 0, 2, { 0xA1A9, 0xA1A9 }, "UCS-2BE in GB 18030 Hanzi mode range" },
+        /* 35*/ { 25, 0, "膀膀", -1, 0, 4, { 0x81, 0x80, 0x81, 0x80 }, "UCS-2BE U+8180" },
+        /* 36*/ { 25, 1, "膀膀", -1, 0, 2, { 0x8180, 0x8180 }, "UCS-2BE in GB 18030 Hanzi mode range (but outside GB 2312 range)" },
+        /* 37*/ { 28, 0, "¢¢", -1, 0, 4, { 0xA2, 0x46, 0xA2, 0x46 }, "Big5 U+00A2" },
+        /* 38*/ { 28, 1, "¢¢", -1, 0, 2, { 0xA246, 0xA246 }, "Big5 in GB 18030 Hanzi mode range (but outside GB 2312 range)" },
+        /* 39*/ { 28, 0, "陛", -1, 0, 2, { 0xB0, 0xA1 }, "Big5 U+965B" },
+        /* 40*/ { 28, 1, "陛", -1, 0, 1, { 0xB0A1 }, "Big5 in GB 18030 Hanzi mode range" },
+        /* 41*/ { 29, 0, "¨¨", -1, 0, 4, { 0xA1, 0xA7, 0xA1, 0xA7 }, "GB 2312 U+00A8" },
+        /* 42*/ { 29, 1, "¨¨", -1, 0, 2, { 0xA1A7, 0xA1A7 }, "GB 2312" },
+        /* 43*/ { 29, 0, "崂", -1, 0, 2, { 0xE1, 0xC0 }, "GB 2312 U+5D02" },
+        /* 44*/ { 29, 1, "崂", -1, 0, 1, { 0xE1C0 }, "GB 2312" },
+        /* 45*/ { 29, 0, "・", -1, 0, 2, { 0xA1, 0xA4 }, "GB 2312 U+30FB" },
+        /* 46*/ { 29, 1, "・", -1, 0, 1, { 0xA1A4 }, "GB 2312" },
+        /* 47*/ { 29, 0, "釦", -1, ZINT_ERROR_INVALID_DATA, -1, {}, "GB 18030 U+91E6 not in GB 2312" },
+        /* 48*/ { 30, 0, "¡¡", -1, 0, 4, { 0x22, 0x2E, 0x22, 0x2E }, "KS X 1001 U+00A1" },
+        /* 49*/ { 30, 1, "¡¡", -1, 0, 4, { 0x22, 0x2E, 0x22, 0x2E }, "KS X 1001 outside GB 18030 Hanzi mode range" },
+        /* 50*/ { 30, 0, "詰", -1, 0, 2, { 0x7D, 0x7E }, "KS X 1001 U+8A70" },
+        /* 51*/ { 30, 1, "詰", -1, 0, 2, { 0x7D, 0x7E }, "KS X 1001 <= 0x7D7E so none in GB 18030 Hanzi mode range" },
     };
 
     int data_size = sizeof(data) / sizeof(struct item);
@@ -267,7 +295,7 @@ static void test_gb18030_utf8tosb(int index) {
         int length = data[i].length == -1 ? (int) strlen(data[i].data) : data[i].length;
         int ret_length = length;
 
-        ret = gb18030_utf8tosb(data[i].eci, (unsigned char *) data[i].data, &ret_length, gbdata, data[i].full_multibyte);
+        ret = gb18030_utf8_to_eci(data[i].eci, (unsigned char *) data[i].data, &ret_length, gbdata, data[i].full_multibyte);
         assert_equal(ret, data[i].ret, "i:%d ret %d != %d\n", i, ret, data[i].ret);
         if (ret == 0) {
             assert_equal(ret_length, data[i].ret_length, "i:%d ret_length %d != %d\n", i, ret_length, data[i].ret_length);
@@ -332,8 +360,8 @@ int main(int argc, char *argv[]) {
 
     testFunction funcs[] = { /* name, func, has_index, has_generate, has_debug */
         { "test_gb18030_wctomb_zint", test_gb18030_wctomb_zint, 0, 0, 0 },
-        { "test_gb18030_utf8tomb", test_gb18030_utf8tomb, 1, 0, 0 },
+        { "test_gb18030_utf8", test_gb18030_utf8, 1, 0, 0 },
-        { "test_gb18030_utf8tosb", test_gb18030_utf8tosb, 1, 0, 0 },
+        { "test_gb18030_utf8_to_eci", test_gb18030_utf8_to_eci, 1, 0, 0 },
         { "test_gb18030_cpy", test_gb18030_cpy, 1, 0, 0 },
     };
 

backend/tests/test_gb18030_tab.h

@@ -63491,20 +63491,68 @@ static const unsigned int test_gb18030_tab[] = {
 };
 
 static const unsigned int test_gb18030_tab_ind[] = {
-0,
+    0,
-8192,
+    2048,
-16384,
+    4096,
-24576,
+    6144,
-32768,
+    8192,
-40960,
+    10240,
-49152,
+    12288,
-57344,
+    14336,
-65536,
+    16384,
-73728,
+    18432,
-81920,
+    20480,
-90112,
+    22528,
-98304,
+    24576,
-106496,
+    26624,
-110592,
+    28672,
-118784,
+    30720,
+    32768,
+    34816,
+    36864,
+    38912,
+    40960,
+    43008,
+    45056,
+    47104,
+    49152,
+    51200,
+    53248,
+    55296,
+    57344,
+    59392,
+    61440,
+    63488,
+    65536,
+    67584,
+    69632,
+    71680,
+    73728,
+    75776,
+    77824,
+    79872,
+    81920,
+    83968,
+    86016,
+    88064,
+    90112,
+    92160,
+    94208,
+    96256,
+    98304,
+    100352,
+    102400,
+    104448,
+    106496,
+    108544,
+    110592,
+    110592,
+    110592,
+    112640,
+    114688,
+    116736,
+    118784,
+    120832,
+    122880,
+    124928,
 };

backend/tests/test_gb2312.c

@@ -1,6 +1,6 @@
 /*
     libzint - the open source barcode library
-    Copyright (C) 2019 - 2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2019 - 2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -41,7 +41,9 @@ static int gb2312_wctomb_zint2(unsigned int *r, unsigned int wc) {
         return 0;
     }
     int tab_length = sizeof(test_gb2312_tab) / sizeof(unsigned int);
-    for (int i = test_gb2312_tab_ind[wc >> 12]; i < tab_length; i += 2) {
+    int start_i = test_gb2312_tab_ind[wc >> 10];
+    int end_i = start_i + 0x800 > tab_length ? tab_length : start_i + 0x800;
+    for (int i = start_i; i < end_i; i += 2) {
         if (test_gb2312_tab[i + 1] == wc) {
             *r = test_gb2312_tab[i] + 0x8080; // Table in GB 2312 not EUC-CN
             return 2;
@@ -86,7 +88,7 @@ static void test_gb2312_wctomb_zint(void) {
     testFinish();
 }
 
-static void test_gb2312_utf8tomb(int index) {
+static void test_gb2312_utf8(int index) {
 
     testStart("");
 
@@ -132,7 +134,7 @@ static void test_gb2312_utf8tomb(int index) {
         int length = data[i].length == -1 ? (int) strlen(data[i].data) : data[i].length;
         int ret_length = length;
 
-        ret = gb2312_utf8tomb(&symbol, (unsigned char *) data[i].data, &ret_length, gbdata);
+        ret = gb2312_utf8(&symbol, (unsigned char *) data[i].data, &ret_length, gbdata);
         assert_equal(ret, data[i].ret, "i:%d ret %d != %d (%s)\n", i, ret, data[i].ret, symbol.errtxt);
         if (ret == 0) {
             assert_equal(ret_length, data[i].ret_length, "i:%d ret_length %d != %d\n", i, ret_length, data[i].ret_length);
@@ -145,7 +147,7 @@ static void test_gb2312_utf8tomb(int index) {
     testFinish();
 }
 
-static void test_gb2312_utf8tosb(int index) {
+static void test_gb2312_utf8_to_eci(int index) {
 
     testStart("");
 
@@ -192,6 +194,32 @@ static void test_gb2312_utf8tosb(int index) {
         /* 15*/ { 3, 1, "©ÿ", -1, 0, 2, { 0xA9, 0xFF }, "First byte in range but not second" },
         /* 16*/ { 3, 0, "éaé驪ª©¯é°°é÷éø", -1, 0, 16, { 0xE9, 0x61, 0xE9, 0xE9, 0xA9, 0xAA, 0xAA, 0xA9, 0xAF, 0xE9, 0xB0, 0xB0, 0xE9, 0xF7, 0xE9, 0xF8 }, "" },
         /* 17*/ { 3, 1, "éaé驪ª©¯é°°é÷éø", -1, 0, 10, { 0xE9, 0x61, 0xE9E9, 0xA9AA, 0xAA, 0xA9AF, 0xE9B0, 0xB0E9, 0xF7E9, 0xF8 }, "" },
+        /* 18*/ { 20, 0, "\\\\", -1, 0, 4, { 0x81, 0x5F, 0x81, 0x5F }, "Shift JIS reverse solidus (backslash) mapping from ASCII to double byte" },
+        /* 19*/ { 20, 1, "\\\\", -1, 0, 4, { 0x81, 0x5F, 0x81, 0x5F }, "Shift JIS outside GB 2312 Hanzi mode range" },
+        /* 20*/ { 20, 0, "爍", -1, 0, 2, { 0xE0, 0xA1 }, "Shift JIS U+720D" },
+        /* 21*/ { 20, 1, "爍", -1, 0, 1, { 0xE0A1 }, "Shift JIS in GB 2312 Hanzi mode range" },
+        /* 22*/ { 25, 0, "12", -1, 0, 4, { 0x00, 0x31, 0x00, 0x32 }, "UCS-2BE ASCII" },
+        /* 23*/ { 25, 0, "", -1, 0, 4, { 0x00, 0x81, 0x00, 0x81 }, "UCS-2BE U+0081" },
+        /* 24*/ { 25, 1, "", -1, 0, 4, { 0x00, 0x81, 0x00, 0x81 }, "UCS-2BE outside GB 2312 Hanzi mode range" },
+        /* 25*/ { 25, 0, "ꆩꆩ", -1, 0, 4, { 0xA1, 0xA9, 0xA1, 0xA9 }, "UCS-2BE U+A1A9" },
+        /* 26*/ { 25, 1, "ꆩꆩ", -1, 0, 2, { 0xA1A9, 0xA1A9 }, "UCS-2BE in GB 2312 Hanzi mode range" },
+        /* 27*/ { 25, 0, "膀膀", -1, 0, 4, { 0x81, 0x80, 0x81, 0x80 }, "UCS-2BE U+8180" },
+        /* 28*/ { 25, 1, "膀膀", -1, 0, 4, { 0x81, 0x80, 0x81, 0x80 }, "UCS-2BE outside GB 2312 Hanzi mode range (but in GB 18030 range)" },
+        /* 29*/ { 28, 0, "¢¢", -1, 0, 4, { 0xA2, 0x46, 0xA2, 0x46 }, "Big5 U+00A2" },
+        /* 30*/ { 28, 1, "¢¢", -1, 0, 4, { 0xA2, 0x46, 0xA2, 0x46 }, "Big5 outside GB 2312 Hanzi mode range (but in GB 18030 range)" },
+        /* 31*/ { 28, 0, "陛", -1, 0, 2, { 0xB0, 0xA1 }, "Big5 U+965B" },
+        /* 32*/ { 28, 1, "陛", -1, 0, 1, { 0xB0A1 }, "Big5 in GB 2312 Hanzi mode range" },
+        /* 33*/ { 29, 0, "¨¨", -1, 0, 4, { 0xA1, 0xA7, 0xA1, 0xA7 }, "GB 2312 U+00A8" },
+        /* 34*/ { 29, 1, "¨¨", -1, 0, 2, { 0xA1A7, 0xA1A7 }, "GB 2312" },
+        /* 35*/ { 29, 0, "崂", -1, 0, 2, { 0xE1, 0xC0 }, "GB 2312 U+5D02" },
+        /* 36*/ { 29, 1, "崂", -1, 0, 1, { 0xE1C0 }, "GB 2312" },
+        /* 37*/ { 29, 0, "・", -1, 0, 2, { 0xA1, 0xA4 }, "GB 2312 U+30FB" },
+        /* 38*/ { 29, 1, "・", -1, 0, 1, { 0xA1A4 }, "GB 2312" },
+        /* 39*/ { 29, 0, "釦", -1, ZINT_ERROR_INVALID_DATA, -1, {}, "GB 18030 U+91E6 not in GB 2312" },
+        /* 40*/ { 30, 0, "¡¡", -1, 0, 4, { 0x22, 0x2E, 0x22, 0x2E }, "KS X 1001 U+00A1" },
+        /* 41*/ { 30, 1, "¡¡", -1, 0, 4, { 0x22, 0x2E, 0x22, 0x2E }, "KS X 1001 outside GB 2312 Hanzi mode range" },
+        /* 42*/ { 30, 0, "詰", -1, 0, 2, { 0x7D, 0x7E }, "KS X 1001 U+8A70" },
+        /* 43*/ { 30, 1, "詰", -1, 0, 2, { 0x7D, 0x7E }, "KS X 1001 <= 0x7D7E so none in GB 2312 Hanzi mode range" },
     };
 
     int data_size = sizeof(data) / sizeof(struct item);
@@ -205,7 +233,7 @@ static void test_gb2312_utf8tosb(int index) {
         int length = data[i].length == -1 ? (int) strlen(data[i].data) : data[i].length;
         int ret_length = length;
 
-        ret = gb2312_utf8tosb(data[i].eci, (unsigned char *) data[i].data, &ret_length, gbdata, data[i].full_multibyte);
+        ret = gb2312_utf8_to_eci(data[i].eci, (unsigned char *) data[i].data, &ret_length, gbdata, data[i].full_multibyte);
         assert_equal(ret, data[i].ret, "i:%d ret %d != %d\n", i, ret, data[i].ret);
         if (ret == 0) {
             assert_equal(ret_length, data[i].ret_length, "i:%d ret_length %d != %d\n", i, ret_length, data[i].ret_length);
@@ -270,8 +298,8 @@ int main(int argc, char *argv[]) {
 
     testFunction funcs[] = { /* name, func, has_index, has_generate, has_debug */
         { "test_gb2312_wctomb_zint", test_gb2312_wctomb_zint, 0, 0, 0 },
-        { "test_gb2312_utf8tomb", test_gb2312_utf8tomb, 1, 0, 0 },
+        { "test_gb2312_utf8", test_gb2312_utf8, 1, 0, 0 },
-        { "test_gb2312_utf8tosb", test_gb2312_utf8tosb, 1, 0, 0 },
+        { "test_gb2312_utf8_to_eci", test_gb2312_utf8_to_eci, 1, 0, 0 },
         { "test_gb2312_cpy", test_gb2312_cpy, 1, 0, 0 },
     };
 

backend/tests/test_gb2312_tab.h

@@ -7448,20 +7448,68 @@ static const unsigned int test_gb2312_tab[] = {
 };
 
 static const unsigned int test_gb2312_tab_ind[] = {
-0,
+    0,
-298,
+    166,
-298,
+    298,
-694,
+    298,
-1168,
+    298,
-1708,
+    298,
-4686,
+    298,
-7508,
+    298,
-9962,
+    298,
-12638,
+    416,
-14694,
+    694,
-14694,
+    694,
-14694,
+    694,
-14694,
+    1168,
-14694,
+    1168,
-14694,
+    1168,
+    1168,
+    1168,
+    1168,
+    1168,
+    1708,
+    2482,
+    3340,
+    3970,
+    4686,
+    5510,
+    6218,
+    6740,
+    7508,
+    8064,
+    8756,
+    9404,
+    9962,
+    10872,
+    11464,
+    11924,
+    12638,
+    13038,
+    13870,
+    14280,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
+    14694,
 };

backend/tests/test_gridmtx.c

@@ -1,6 +1,6 @@
 /*
     libzint - the open source barcode library
-    Copyright (C) 2008-2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2008-2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -27,7 +27,7 @@
     OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     SUCH DAMAGE.
  */
-/* vim: set ts=4 sw=4 et : */
+/* vim: set ts=4 sw=4 et norl : */
 
 #include "testcommon.h"
 
@@ -110,7 +110,7 @@ static void test_options(int index, int debug) {
         /* 10*/ { "123456789012345678", 5, 0, 0, 0, 30 }, // Version not specified so increased to allow for ECC level
         /* 11*/ { "123456789012345678", 6, 0, 0, 0, 30 }, // ECC > max ECC 5 so ignored and auto-settings version 2, ECC 4 used
     };
-    int data_size = sizeof(data) / sizeof(struct item);
+    int data_size = ARRAY_SIZE(data);
 
     for (int i = 0; i < data_size; i++) {
 
@@ -119,12 +119,7 @@ static void test_options(int index, int debug) {
         struct zint_symbol *symbol = ZBarcode_Create();
         assert_nonnull(symbol, "Symbol not created\n");
 
-        symbol->symbology = BARCODE_GRIDMATRIX;
+        int length = testUtilSetSymbol(symbol, BARCODE_GRIDMATRIX, -1 /*input_mode*/, -1 /*eci*/, data[i].option_1, data[i].option_2, -1, -1 /*output_options*/, data[i].data, -1, debug);
-        symbol->option_1 = data[i].option_1;
-        symbol->option_2 = data[i].option_2;
-        symbol->debug |= debug;
-
-        int length = strlen(data[i].data);
 
         ret = ZBarcode_Encode(symbol, (unsigned char *) data[i].data, length);
         assert_equal(ret, data[i].ret_encode, "i:%d ZBarcode_Encode ret %d != %d (%s)\n", i, ret, data[i].ret_encode, symbol->errtxt);
@@ -167,17 +162,17 @@ static void test_input(int index, int generate, int debug) {
         /*  1*/ { UNICODE_MODE, 3, -1, "é", 0, 3, "60 01 58 00 74 40", "ECI-3 B1 (ISO 8859-1)" },
         /*  2*/ { UNICODE_MODE, 29, -1, "é", 0, 29, "60 0E 44 2A 37 7C 00", "ECI-29 H1 (GB 2312)" },
         /*  3*/ { UNICODE_MODE, 26, -1, "é", 0, 26, "60 0D 18 01 61 6A 20", "ECI-26 B2 (UTF-8)" },
-        /*  4*/ { UNICODE_MODE, 26, 200, "é", 0, 26, "60 0D 05 28 4F 7C 00", "ECI-26 H1 (UTF-8) (full multibyte)" },
+        /*  4*/ { UNICODE_MODE, 26, ZINT_FULL_MULTIBYTE, "é", 0, 26, "60 0D 05 28 4F 7C 00", "ECI-26 H1 (UTF-8) (full multibyte)" },
         /*  5*/ { DATA_MODE, 0, -1, "é", 0, 0, "30 03 43 54 40", "B2 (UTF-8)" },
-        /*  6*/ { DATA_MODE, 0, 200, "é", 0, 0, "0A 51 1F 78 00", "H1 (UTF-8) (full multibyte)" },
+        /*  6*/ { DATA_MODE, 0, ZINT_FULL_MULTIBYTE, "é", 0, 0, "0A 51 1F 78 00", "H1 (UTF-8) (full multibyte)" },
         /*  7*/ { DATA_MODE, 0, -1, "\351", 0, 0, "30 01 69 00", "B1 (ISO 8859-1) (0xE9)" },
         /*  8*/ { UNICODE_MODE, 0, -1, "β", 0, 0, "08 40 2F 78 00", "H1 (GB 2312)" },
         /*  9*/ { UNICODE_MODE, 9, -1, "β", 0, 9, "60 04 58 00 71 00", "ECI-9 B1 (ISO 8859-7)" },
         /* 10*/ { UNICODE_MODE, 29, -1, "β", 0, 29, "60 0E 44 20 17 7C 00", "ECI-29 H1 (GB 2312)" },
         /* 11*/ { UNICODE_MODE, 26, -1, "β", 0, 26, "60 0D 18 01 67 2C 40", "ECI-26 H1 (UTF-8)" },
-        /* 12*/ { UNICODE_MODE, 26, 200, "β", 0, 26, "60 0D 05 6B 17 7C 00", "ECI-26 H1 (UTF-8) (full multibyte)" },
+        /* 12*/ { UNICODE_MODE, 26, ZINT_FULL_MULTIBYTE, "β", 0, 26, "60 0D 05 6B 17 7C 00", "ECI-26 H1 (UTF-8) (full multibyte)" },
         /* 13*/ { DATA_MODE, 0, -1, "β", 0, 0, "30 03 4E 59 00", "B2 (UTF-8)" },
-        /* 14*/ { DATA_MODE, 0, 200, "β", 0, 0, "0B 56 2F 78 00", "H1 (UTF-8) (full multibyte)" },
+        /* 14*/ { DATA_MODE, 0, ZINT_FULL_MULTIBYTE, "β", 0, 0, "0B 56 2F 78 00", "H1 (UTF-8) (full multibyte)" },
         /* 15*/ { UNICODE_MODE, 0, -1, "ÿ", 0, 0, "30 01 7F 00", "B1 (ISO 8859-1)" },
         /* 16*/ { UNICODE_MODE, 0, -1, "ÿÿÿ", 0, 0, "30 05 7F 7F 7F 60", "B3 (ISO 8859-1)" },
         /* 17*/ { UNICODE_MODE, 0, -1, "㈩一", 0, 0, "08 15 68 0E 7F 70 00", "H2 (GB 2312)" },
@@ -215,8 +210,44 @@ static void test_input(int index, int generate, int debug) {
         /* 49*/ { UNICODE_MODE, 0, -1, "::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::\177", 0, 0, "(591) 37 68 68 68 68 68 74 7E 74 74 74 74 74 3A 3A 3A 3A 3A 3A 3A 1D 1D 1D 1D 1D 1D 1D 0E", "B513 (ASCII)" },
         /* 50*/ { UNICODE_MODE, 0, -1, ":::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::至", 0, 0, "(591) 37 68 68 68 68 68 74 7C 74 74 74 74 74 3A 3A 3A 3A 3A 3A 3A 1D 1D 1D 1D 1D 1D 1D 0E", "B511 H1 (GB 2312)" },
         /* 51*/ { UNICODE_MODE, 0, -1, ":::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::至:", 0, 0, "(592) 37 68 68 68 68 68 74 7E 74 74 74 74 74 3A 3A 3A 3A 3A 3A 3A 1D 1D 1D 1D 1D 1D 1D 0E", "B513 (GB 2312)" },
+        /* 52*/ { UNICODE_MODE, 0, -1, "˘", ZINT_WARN_USES_ECI, 4, "Warning 60 02 18 00 51 00", "ECI-4 B1 (ISO 8859-2)" },
+        /* 53*/ { UNICODE_MODE, 4, -1, "˘", 0, 4, "60 02 18 00 51 00", "ECI-4 B1 (ISO 8859-2)" },
+        /* 54*/ { UNICODE_MODE, 0, -1, "Ħ", ZINT_WARN_USES_ECI, 5, "Warning 60 02 58 00 50 40", "ECI-5 B1 (ISO 8859-3)" },
+        /* 55*/ { UNICODE_MODE, 5, -1, "Ħ", 0, 5, "60 02 58 00 50 40", "ECI-5 B1 (ISO 8859-3)" },
+        /* 56*/ { UNICODE_MODE, 6, -1, "ĸ", 0, 6, "60 03 18 00 51 00", "ECI-6 B1 (ISO 8859-4)" },
+        /* 57*/ { UNICODE_MODE, 7, -1, "Ж", 0, 7, "60 03 58 00 5B 00", "ECI-7 B1 (ISO 8859-5)" },
+        /* 58*/ { UNICODE_MODE, 0, -1, "Ș", ZINT_WARN_USES_ECI, 18, "Warning 60 09 18 00 55 00", "ECI-18 B1 (ISO 8859-16)" },
+        /* 59*/ { UNICODE_MODE, 18, -1, "Ș", 0, 18, "60 09 18 00 55 00", "ECI-18 B1 (ISO 8859-16)" },
+        /* 60*/ { UNICODE_MODE, 0, -1, "テ", 0, 0, "08 34 6F 78 00", "H1 (GB 2312)" },
+        /* 61*/ { UNICODE_MODE, 20, -1, "テ", 0, 20, "60 0A 18 01 41 59 20", "ECI-20 B2 (SHIFT JIS)" },
+        /* 62*/ { UNICODE_MODE, 20, -1, "テテ", 0, 20, "60 0A 18 03 41 59 30 36 28 00", "ECI-20 B4 (SHIFT JIS)" },
+        /* 63*/ { UNICODE_MODE, 20, -1, "\\\\", 0, 20, "60 0A 18 03 40 57 70 15 78 00", "ECI-20 B4 (SHIFT JIS)" },
+        /* 64*/ { UNICODE_MODE, 0, -1, "…", 0, 0, "08 01 5F 78 00", "H1 (GB 2312)" },
+        /* 65*/ { UNICODE_MODE, 21, -1, "…", 0, 21, "60 0A 58 00 42 40", "ECI-21 B1 (Win 1250)" },
+        /* 66*/ { UNICODE_MODE, 0, -1, "Ґ", ZINT_WARN_USES_ECI, 22, "Warning 60 0B 18 00 52 40", "ECI-22 B1 (Win 1251)" },
+        /* 67*/ { UNICODE_MODE, 22, -1, "Ґ", 0, 22, "60 0B 18 00 52 40", "ECI-22 B1 (Win 1251)" },
+        /* 68*/ { UNICODE_MODE, 0, -1, "˜", ZINT_WARN_USES_ECI, 23, "Warning 60 0B 58 00 4C 00", "ECI-23 B1 (Win 1252)" },
+        /* 69*/ { UNICODE_MODE, 23, -1, "˜", 0, 23, "60 0B 58 00 4C 00", "ECI-23 B1 (Win 1252)" },
+        /* 70*/ { UNICODE_MODE, 24, -1, "پ", 0, 24, "60 0C 18 00 40 40", "ECI-24 B1 (Win 1256)" },
+        /* 71*/ { UNICODE_MODE, 0, -1, "က", ZINT_WARN_USES_ECI, 26, "Warning 60 0D 18 02 70 60 10 00", "ECI-26 B3 (UTF-8)" },
+        /* 72*/ { UNICODE_MODE, 25, -1, "က", 0, 25, "60 0C 58 01 08 00 00", "ECI-25 B2 (UCS-2BE)" },
+        /* 73*/ { UNICODE_MODE, 25, -1, "ကက", 0, 25, "60 0C 58 03 08 00 02 00 00 00", "ECI-25 B4 (UCS-2BE)" },
+        /* 74*/ { UNICODE_MODE, 25, -1, "12", 0, 25, "60 0C 58 03 00 0C 20 03 10 00", "ECI-25 B4 (UCS-2BE ASCII)" },
+        /* 75*/ { UNICODE_MODE, 27, -1, "@", 0, 27, "60 0D 4F 77 2E 60", "ECI-27 L1 (ASCII)" },
+        /* 76*/ { UNICODE_MODE, 0, -1, "龘", ZINT_WARN_USES_ECI, 26, "Warning 60 0D 18 02 74 6F 53 00", "ECI-26 B3 (UTF-8)" },
+        /* 77*/ { UNICODE_MODE, 28, -1, "龘", 0, 28, "60 0E 18 01 7C 75 20", "ECI-28 B2 (Big5)" },
+        /* 78*/ { UNICODE_MODE, 28, -1, "龘龘", 0, 28, "60 0E 18 03 7C 75 3F 1D 28 00", "ECI-28 B4 (Big5)" },
+        /* 79*/ { UNICODE_MODE, 0, -1, "齄", 0, 0, "0F 4B 6F 78 00", "H1 (GB 2312)" },
+        /* 80*/ { UNICODE_MODE, 29, -1, "齄", 0, 29, "60 0E 47 65 77 7C 00", "ECI-29 H1 (GB 2312)" },
+        /* 81*/ { UNICODE_MODE, 29, -1, "齄齄", 0, 29, "60 0E 47 65 77 4B 6F 78 00", "ECI-29 H2 (GB 2312)" },
+        /* 82*/ { UNICODE_MODE, 0, -1, "가", ZINT_WARN_USES_ECI, 26, "Warning 60 0D 18 02 75 2C 10 00", "ECI-26 B3 (UTF-8)" },
+        /* 83*/ { UNICODE_MODE, 30, -1, "가", 0, 30, "60 0F 18 01 18 08 20", "ECI-30 B2 (KS X 1001)" },
+        /* 84*/ { UNICODE_MODE, 30, -1, "가가", 0, 30, "60 0F 18 03 18 08 26 02 08 00", "ECI-30 B4 (KS X 1001)" },
+        /* 85*/ { UNICODE_MODE, 170, -1, "?", 0, 170, "60 55 0F 77 26 60", "ECI-170 L1 (ASCII invariant)" },
+        /* 86*/ { DATA_MODE, 899, -1, "\200", 0, 899, "63 41 58 00 40 00", "ECI-899 B1 (8-bit binary)" },
+        /* 87*/ { UNICODE_MODE, 900, -1, "é", 0, 900, "63 42 18 01 61 6A 20", "ECI-900 B2 (no conversion)" },
     };
-    int data_size = sizeof(data) / sizeof(struct item);
+    int data_size = ARRAY_SIZE(data);
 
     char escaped[1024];
 
@@ -227,23 +258,17 @@ static void test_input(int index, int generate, int debug) {
         struct zint_symbol *symbol = ZBarcode_Create();
         assert_nonnull(symbol, "Symbol not created\n");
 
-        symbol->symbology = BARCODE_GRIDMATRIX;
+        debug |= ZINT_DEBUG_TEST; // Needed to get codeword dump in errtxt
-        symbol->input_mode = data[i].input_mode;
-        symbol->eci = data[i].eci;
-        if (data[i].option_3 != -1) {
-            symbol->option_3 = data[i].option_3;
-        }
-        symbol->debug = ZINT_DEBUG_TEST; // Needed to get codeword dump in errtxt
-        symbol->debug |= debug;
 
-        int length = strlen(data[i].data);
+        int length = testUtilSetSymbol(symbol, BARCODE_GRIDMATRIX, data[i].input_mode, data[i].eci, -1 /*option_1*/, -1, data[i].option_3, -1 /*output_options*/, data[i].data, -1, debug);
 
         ret = ZBarcode_Encode(symbol, (unsigned char *) data[i].data, length);
         assert_equal(ret, data[i].ret, "i:%d ZBarcode_Encode ret %d != %d\n", i, ret, data[i].ret);
 
         if (generate) {
-            printf("        /*%3d*/ { %s, %d, %d, \"%s\", %s, %d, \"%s\", \"%s\" },\n",
+            printf("        /*%3d*/ { %s, %d, %s, \"%s\", %s, %d, \"%s\", \"%s\" },\n",
-                    i, testUtilInputModeName(data[i].input_mode), data[i].eci, data[i].option_3, testUtilEscape(data[i].data, length, escaped, sizeof(escaped)),
+                    i, testUtilInputModeName(data[i].input_mode), data[i].eci, testUtilOption3Name(data[i].option_3),
+                    testUtilEscape(data[i].data, length, escaped, sizeof(escaped)),
                     testUtilErrorName(data[i].ret), ret < 5 ? symbol->eci : -1, symbol->errtxt, data[i].comment);
         } else {
             if (ret < 5) {
@@ -374,7 +399,7 @@ static void test_encode(int index, int generate, int debug) {
                     "111111000000111111000000111111000000111111"
                },
     };
-    int data_size = sizeof(data) / sizeof(struct item);
+    int data_size = ARRAY_SIZE(data);
 
     for (int i = 0; i < data_size; i++) {
 
@@ -383,17 +408,7 @@ static void test_encode(int index, int generate, int debug) {
         struct zint_symbol *symbol = ZBarcode_Create();
         assert_nonnull(symbol, "Symbol not created\n");
 
-        symbol->symbology = BARCODE_GRIDMATRIX;
+        int length = testUtilSetSymbol(symbol, BARCODE_GRIDMATRIX, data[i].input_mode, -1 /*eci*/, data[i].option_1, data[i].option_2, -1, -1 /*output_options*/, data[i].data, -1, debug);
-        symbol->input_mode = data[i].input_mode;
-        if (data[i].option_1 != -1) {
-            symbol->option_1 = data[i].option_1;
-        }
-        if (data[i].option_2 != -1) {
-            symbol->option_2 = data[i].option_2;
-        }
-        symbol->debug |= debug;
-
-        int length = strlen(data[i].data);
 
         ret = ZBarcode_Encode(symbol, (unsigned char *) data[i].data, length);
         assert_equal(ret, data[i].ret, "i:%d ZBarcode_Encode ret %d != %d (%s)\n", i, ret, data[i].ret, symbol->errtxt);

backend/tests/test_hanxin.c

@@ -1,6 +1,6 @@
 /*
     libzint - the open source barcode library
-    Copyright (C) 2019 - 2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2019 - 2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -27,7 +27,7 @@
     OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     SUCH DAMAGE.
  */
-/* vim: set ts=4 sw=4 et : */
+/* vim: set ts=4 sw=4 et norl : */
 
 #include "testcommon.h"
 
@@ -196,7 +196,7 @@ static void test_input(int index, int generate, int debug) {
         /* 30*/ { DATA_MODE, 0, -1, "Summer Palace Ticket for 6 June 2015 13:00;2015年6月6日夜01時00分PM頤和園のチケット;2015년6월6일13시오후여름궁전티켓.2015年6月6号下午13:00的颐和园门票;", -1, 0, 0, "(209) 27 38 C3 0A 35 F9 CF 99 92 F9 26 A3 E7 3E 76 C9 AE A3 7F CC 15 04 0C CD EE 44 06 C4", "T20 B117 (UTF-8)" },
         /* 31*/ { UNICODE_MODE, 0, -1, "\000\014\033 #/059:<@AMZ", 15, 0, 0, "2F 80 31 B7 1F AF E0 05 27 EB 2E CB E2 96 8F F0 00", "T15 (ASCII)" },
         /* 32*/ { UNICODE_MODE, 0, -1, "Z[\\`alz{~\177", -1, 0, 0, "28 FE CF 4E 3E 92 FF 7E E7 CF 7F 00 00", "T10 (ASCII)" },
-        /* 33*/ { UNICODE_MODE, 26, ZINT_FULL_MULTIBYTE, "\202\061\203\063", -1, 0, 26, "81 A7 01 B1 D8 00 00 00 00", "ECI-26 H(f)1 (GB 18030) (Invalid UTF-8, forces GB 2312/18030 utf8tosb() difference)" },
+        /* 33*/ { DATA_MODE, 26, ZINT_FULL_MULTIBYTE, "\202\061\203\063", -1, 0, 26, "81 A7 01 B1 D8 00 00 00 00", "ECI-26 H(f)1 (GB 18030) (Invalid UTF-8, forces GB 2312/18030 utf8tosb() difference) NOTE: 2021-01-10 now UTF-8 is checked and mode -> DATA_MODE this test no longer shows difference" },
         /* 34*/ { UNICODE_MODE, 128, 0, "A", -1, 0, 128, "88 08 02 2B F0 00 00 00 00", "ECI > 127" },
         /* 35*/ { UNICODE_MODE, 16364, 0, "A", -1, 0, 16364, "8B FE C2 2B F0 00 00 00 00", "ECI > 16363" },
         /* 36*/ { UNICODE_MODE, 0, -1, "啊啊啊亍", -1, 0, 0, "40 00 00 00 00 FF E0 00 FF F0 00 00 00", "Region 1 (FFE terminator) -> Region 2 (no indicator)" },
@@ -204,6 +204,44 @@ static void test_input(int index, int generate, int debug) {
         /* 38*/ { UNICODE_MODE, 0, -1, "啊啊啊啊亍亍啊", -1, 0, 0, "40 00 00 00 00 00 0F FE 00 00 00 FF E0 00 FF F0 00", "Region 1 (FFE) -> Region 2 (FFE) -> Region 1" },
         /* 39*/ { UNICODE_MODE, 0, -1, "亍亍亍亍啊啊亍", -1, 0, 0, "50 00 00 00 00 00 0F FE 00 00 00 FF E0 00 FF F0 00", "Region 2 (FFE) -> Region 1 (FFE) -> Region 2" },
         /* 40*/ { DATA_MODE, 0, ZINT_FULL_MULTIBYTE | (2 << 8), "é", -1, 0, 0, "47 02 FF F0 00 00 00 00 00", "H(1)1 (UTF-8) (Region One) (full multibyte with mask)" },
+        /* 41*/ { UNICODE_MODE, 0, -1, "˘", -1, 0, 0, "70 01 16 80 00 00 00 00 00", "H(f)1 (GB 18030)" },
+        /* 42*/ { UNICODE_MODE, 4, -1, "˘", -1, 0, 4, "80 43 00 0D 10 00 00 00 00", "ECI-4 B1 (ISO 8859-2)" },
+        /* 43*/ { UNICODE_MODE, 0, -1, "Ħ", -1, 0, 0, "70 00 47 80 00 00 00 00 00", "H(f)1 (GB 18030)" },
+        /* 44*/ { UNICODE_MODE, 5, -1, "Ħ", -1, 0, 5, "80 53 00 0D 08 00 00 00 00", "ECI-5 B1 (ISO 8859-3)" },
+        /* 45*/ { UNICODE_MODE, 0, -1, "ĸ", -1, 0, 0, "70 00 50 00 00 00 00 00 00", "H(f)1 (GB 18030)" },
+        /* 46*/ { UNICODE_MODE, 6, -1, "ĸ", -1, 0, 6, "80 63 00 0D 10 00 00 00 00", "ECI-6 B1 (ISO 8859-4)" },
+        /* 47*/ { UNICODE_MODE, 0, -1, "Ж", -1, 0, 0, "30 01 53 D4 00 00 00 00 00", "B2 (GB 18030)" },
+        /* 48*/ { UNICODE_MODE, 7, -1, "Ж", -1, 0, 7, "80 73 00 0D B0 00 00 00 00", "ECI-7 B1 (ISO 8859-5)" },
+        /* 49*/ { UNICODE_MODE, 0, -1, "Ș", -1, 0, 0, "70 00 B9 80 00 00 00 00 00", "H(f)1 (GB 18030)" },
+        /* 50*/ { UNICODE_MODE, 18, -1, "Ș", -1, 0, 18, "81 23 00 0D 50 00 00 00 00", "ECI-18 B1 (ISO 8859-16)" },
+        /* 51*/ { UNICODE_MODE, 0, -1, "テ", -1, 0, 0, "30 01 52 E3 00 00 00 00 00", "B2 (GB 18030)" },
+        /* 52*/ { UNICODE_MODE, 20, -1, "テ", -1, 0, 20, "81 43 00 14 1B 28 00 00 00", "ECI-20 B2 (SHIFT JIS)" },
+        /* 53*/ { UNICODE_MODE, 20, -1, "テテ", -1, 0, 20, "81 43 00 24 1B 2C 1B 28 00", "ECI-20 B4 (SHIFT JIS)" },
+        /* 54*/ { UNICODE_MODE, 20, -1, "\\\\", -1, 0, 20, "81 43 00 24 0A FC 0A F8 00", "ECI-20 B4 (SHIFT JIS)" },
+        /* 55*/ { UNICODE_MODE, 0, -1, "…", -1, 0, 0, "4E BC FF F0 00 00 00 00 00", "H(1)1 (GB 18030)" },
+        /* 56*/ { UNICODE_MODE, 21, -1, "…", -1, 0, 21, "81 53 00 0C 28 00 00 00 00", "ECI-21 B1 (Win 1250)" },
+        /* 57*/ { UNICODE_MODE, 0, -1, "Ґ", -1, 0, 0, "70 01 B9 00 00 00 00 00 00", "H(f)1 (GB 18030)" },
+        /* 58*/ { UNICODE_MODE, 22, -1, "Ґ", -1, 0, 22, "81 63 00 0D 28 00 00 00 00", "ECI-22 B1 (Win 1251)" },
+        /* 59*/ { UNICODE_MODE, 0, -1, "˜", -1, 0, 0, "70 01 18 00 00 00 00 00 00", "H(f)1 (GB 18030)" },
+        /* 60*/ { UNICODE_MODE, 23, -1, "˜", -1, 0, 23, "81 73 00 0C C0 00 00 00 00", "ECI-23 B1 (Win 1252)" },
+        /* 61*/ { UNICODE_MODE, 24, -1, "پ", -1, 0, 24, "81 83 00 0C 08 00 00 00 00", "ECI-24 B1 (Win 1256)" },
+        /* 62*/ { UNICODE_MODE, 0, -1, "က", -1, 0, 0, "70 07 71 00 00 00 00 00 00", "H(f)1 (GB 18030)" },
+        /* 63*/ { UNICODE_MODE, 25, -1, "က", -1, 0, 25, "81 92 F9 00 3F 00 00 00 00", "ECI-25 T2 (UCS-2BE)" },
+        /* 64*/ { UNICODE_MODE, 25, -1, "ကက", -1, 0, 25, "81 92 F9 00 10 03 F0 00 00", "ECI-25 T4 (UCS-2BE)" },
+        /* 65*/ { UNICODE_MODE, 25, -1, "12", -1, 0, 25, "81 93 00 20 01 88 01 90 00", "ECI-25 B4 (UCS-2BE ASCII)" },
+        /* 66*/ { UNICODE_MODE, 27, -1, "@", -1, 0, 27, "81 B2 FB 2F C0 00 00 00 00", "ECI-27 T1 (ASCII)" },
+        /* 67*/ { UNICODE_MODE, 0, -1, "龘", -1, 0, 0, "30 01 7E C9 80 00 00 00 00", "B2 (GB 18030)" },
+        /* 68*/ { UNICODE_MODE, 28, -1, "龘", -1, 0, 28, "81 C3 00 17 CE A8 00 00 00", "ECI-28 B2 (Big5)" },
+        /* 69*/ { UNICODE_MODE, 28, -1, "龘龘", -1, 0, 28, "81 C3 00 27 CE AF CE A8 00", "ECI-28 B4 (Big5)" },
+        /* 70*/ { UNICODE_MODE, 0, -1, "齄", -1, 0, 0, "5B BF FF F0 00 00 00 00 00", "H(2)1 (GB 18030)" },
+        /* 71*/ { UNICODE_MODE, 29, -1, "齄", -1, 0, 29, "81 D5 BB FF FF 00 00 00 00", "ECI-29 H(2)1 (GB 2312)" },
+        /* 72*/ { UNICODE_MODE, 29, -1, "齄齄", -1, 0, 29, "81 D5 BB FB BF FF F0 00 00", "ECI-29 H(2)2 (GB 2312)" },
+        /* 73*/ { UNICODE_MODE, 0, -1, "가", -1, 0, 0, "70 2B 5E 80 00 00 00 00 00", "H(f)1 (GB 18030)" },
+        /* 74*/ { UNICODE_MODE, 30, -1, "가", -1, 0, 30, "81 E2 03 E7 7F 00 00 00 00", "ECI-30 T2 (KS X 1001)" },
+        /* 75*/ { UNICODE_MODE, 30, -1, "가가", -1, 0, 30, "81 E3 00 21 81 09 81 08 00", "ECI-30 B4 (KS X 1001)" },
+        /* 76*/ { UNICODE_MODE, 170, -1, "?", -1, 0, 170, "88 0A A2 FB 1F C0 00 00 00", "ECI-170 L1 (ASCII invariant)" },
+        /* 77*/ { DATA_MODE, 899, -1, "\200", -1, 0, 899, "88 38 33 00 0C 00 00 00 00", "ECI-899 B1 (8-bit binary)" },
+        /* 78*/ { UNICODE_MODE, 900, -1, "é", -1, 0, 900, "88 38 43 00 16 1D 48 00 00", "ECI-900 B2 (no conversion)" },
     };
     int data_size = ARRAY_SIZE(data);
 

backend/tests/test_ksx1001.c

@@ -0,0 +1,99 @@
+/*
+    libzint - the open source barcode library
+    Copyright (C) 2021 Robin Stuart <rstuart114@gmail.com>
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions
+    are met:
+
+    1. Redistributions of source code must retain the above copyright
+       notice, this list of conditions and the following disclaimer.
+    2. Redistributions in binary form must reproduce the above copyright
+       notice, this list of conditions and the following disclaimer in the
+       documentation and/or other materials provided with the distribution.
+    3. Neither the name of the project nor the names of its contributors
+       may be used to endorse or promote products derived from this software
+       without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+    ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+    IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+    ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+    FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+    OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+    HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+    LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+    OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+    SUCH DAMAGE.
+ */
+/* vim: set ts=4 sw=4 et : */
+
+#include "testcommon.h"
+#include "test_ksx1001_tab.h"
+#include "../ksx1001.h"
+
+// As control convert to KS X 1001 using simple table generated from https://www.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/KSC/KSX1001.TXT plus simple processing
+static int ksx1001_wctomb_zint2(unsigned int *r, unsigned int wc) {
+    if (wc < 0x80) {
+        return 0;
+    }
+    if (wc == 0x20AC) { // Euro sign added KS X 1001:1998
+        *r = 0x2266;
+        return 2;
+    }
+    if (wc == 0xAE) { // Registered trademark added KS X 1001:1998
+        *r = 0x2267;
+        return 2;
+    }
+    if (wc == 0x327E) { // Korean postal code symbol added KS X 1001:2002
+        *r = 0x2268;
+        return 2;
+    }
+    int tab_length = ARRAY_SIZE(test_ksx1001_tab);
+    int start_i = test_ksx1001_tab_ind[wc >> 10];
+    int end_i = start_i + 0x800 > tab_length ? tab_length : start_i + 0x800;
+    for (int i = start_i; i < end_i; i += 2) {
+        if (test_ksx1001_tab[i + 1] == wc) {
+            *r = test_ksx1001_tab[i];
+            return *r > 0xFF ? 2 : 1;
+        }
+    }
+    return 0;
+}
+
+static void test_ksx1001_wctomb_zint(void) {
+
+    testStart("");
+
+    int ret, ret2;
+    unsigned int val, val2;
+
+    for (unsigned int i = 0; i < 0xFFFE; i++) {
+        if (i >= 0xD800 && i <= 0xDFFF) { // UTF-16 surrogates
+            continue;
+        }
+        val = val2 = 0;
+        ret = ksx1001_wctomb_zint(&val, i);
+        ret2 = ksx1001_wctomb_zint2(&val2, i);
+        assert_equal(ret, ret2, "i:%d 0x%04X ret %d != ret2 %d, val 0x%04X, val2 0x%04X\n", i, i, ret, ret2, val, val2);
+        if (ret2) {
+            assert_equal(val, val2, "i:%d 0x%04X val 0x%04X != val2 0x%04X\n", i, i, val, val2);
+        }
+    }
+
+    testFinish();
+}
+
+int main(int argc, char *argv[]) {
+
+    testFunction funcs[] = { /* name, func, has_index, has_generate, has_debug */
+        { "test_ksx1001_wctomb_zint", test_ksx1001_wctomb_zint, 0, 0, 0 },
+    };
+
+    testRun(argc, argv, funcs, ARRAY_SIZE(funcs));
+
+    testReport();
+
+    return 0;
+}

backend/tests/test_library.c

@@ -1,6 +1,6 @@
 /*
     libzint - the open source barcode library
-    Copyright (C) 2019 - 2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2019 - 2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -262,24 +262,51 @@ static void test_cap(int index) {
 }
 
 // #181 Nico Gunkel OSS-Fuzz
-static void test_encode_file_zero_length(void) {
+static void test_encode_file_length(void) {
 
     testStart("");
 
     int ret;
     char filename[] = "in.bin";
+    char buf[ZINT_MAX_DATA_LEN + 1] = {0};
     int fd;
 
     struct zint_symbol *symbol = ZBarcode_Create();
     assert_nonnull(symbol, "Symbol not created\n");
 
     (void)remove(filename); // In case junk hanging around
+
+    // Empty file
     fd = creat(filename, S_IRUSR);
-    assert_nonzero(fd, "Input file not created\n");
+    assert_nonzero(fd, "Empty input file not created\n");
-    assert_zero(close(fd), "close(%s) != 0\n", filename);
+    assert_zero(close(fd), "Empty close(%s) != 0\n", filename);
 
     ret = ZBarcode_Encode_File(symbol, filename);
-    assert_equal(ret, ZINT_ERROR_INVALID_DATA, "ret %d != ZINT_ERROR_INVALID_DATA\n", ret);
+    assert_equal(ret, ZINT_ERROR_INVALID_DATA, "ZBarcode_Encode_File empty ret %d != ZINT_ERROR_INVALID_DATA (%s)\n", ret, symbol->errtxt);
+
+    assert_zero(remove(filename), "remove(%s) != 0\n", filename);
+
+    // Too large file
+    fd = creat(filename, S_IRUSR | S_IWUSR);
+    assert_nonzero(fd, "Too large input file not created\n");
+    ret = write(fd, buf, sizeof(buf));
+    assert_equal(ret, sizeof(buf), "Too large write ret %d != %d\n", ret, (int) sizeof(buf));
+    assert_zero(close(fd), "Too large close(%s) != 0\n", filename);
+
+    ret = ZBarcode_Encode_File(symbol, filename);
+    assert_equal(ret, ZINT_ERROR_TOO_LONG, "ZBarcode_Encode_File too large ret %d != ZINT_ERROR_TOO_LONG (%s)\n", ret, symbol->errtxt);
+
+    assert_zero(remove(filename), "remove(%s) != 0\n", filename);
+
+    // Unreadable file
+    fd = creat(filename, S_IWUSR);
+    assert_nonzero(fd, "Unreadable input file not created\n");
+    ret = write(fd, buf, 1);
+    assert_equal(ret, 1, "Unreadable write ret %d != 1\n", ret);
+    assert_zero(close(fd), "Unreadable close(%s) != 0\n", filename);
+
+    ret = ZBarcode_Encode_File(symbol, filename);
+    assert_equal(ret, ZINT_ERROR_INVALID_DATA, "ZBarcode_Encode_File unreadable ret %d != ZINT_ERROR_INVALID_DATA (%s)\n", ret, symbol->errtxt);
 
     assert_zero(remove(filename), "remove(%s) != 0\n", filename);
 
@@ -430,7 +457,7 @@ int main(int argc, char *argv[]) {
         { "test_input_mode", test_input_mode, 1, 0, 1 },
         { "test_escape_char_process", test_escape_char_process, 1, 1, 1 },
         { "test_cap", test_cap, 1, 0, 0 },
-        { "test_encode_file_zero_length", test_encode_file_zero_length, 0, 0, 0 },
+        { "test_encode_file_length", test_encode_file_length, 0, 0, 0 },
         { "test_encode_file_directory", test_encode_file_directory, 0, 0, 0 },
         { "test_bad_args", test_bad_args, 0, 0, 0 },
         { "test_valid_id", test_valid_id, 0, 0, 0 },

backend/tests/test_qr.c

@@ -1,6 +1,6 @@
 /*
     libzint - the open source barcode library
-    Copyright (C) 2019 - 2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2019 - 2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -27,7 +27,7 @@
     OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     SUCH DAMAGE.
  */
-/* vim: set ts=4 sw=4 et : */
+/* vim: set ts=4 sw=4 et norl : */
 
 #include "testcommon.h"
 
@@ -235,6 +235,42 @@ static void test_qr_input(int index, int generate, int debug) {
         /* 87*/ { UNICODE_MODE, 0, -1, "áA", 0, 0, "40 2E 14 10 EC 11 EC 11 EC", "B2 (ISO 8859-1)" },
         /* 88*/ { UNICODE_MODE, 0, ZINT_FULL_MULTIBYTE, "áA", 0, 0, "80 1C 00 80 EC 11 EC 11 EC", "K1 (ISO 8859-1) (full multibyte)" },
         /* 89*/ { UNICODE_MODE, 0, -1, "A0B1C2D3E4F5G6H7I8J9KLMNOPQRSTUVWXYZ $%*+-./:", 0, 0, "(34) 21 69 C2 3E 08 79 26 27 A5 50 B5 98 23 32 6C 0E 65 FA C5 19 5B 42 6B 2D C1 C3 B9 E7", "A45" },
+        /* 90*/ { UNICODE_MODE, 0, -1, "˘", ZINT_WARN_USES_ECI, 4, "Warning 70 44 01 A2 00 EC 11 EC 11", "ECI-4 B1 (ISO 8859-2)" },
+        /* 91*/ { UNICODE_MODE, 4, -1, "˘", 0, 4, "70 44 01 A2 00 EC 11 EC 11", "ECI-4 B1 (ISO 8859-2)" },
+        /* 92*/ { UNICODE_MODE, 0, -1, "Ħ", ZINT_WARN_USES_ECI, 5, "Warning 70 54 01 A1 00 EC 11 EC 11", "ECI-5 B1 (ISO 8859-3)" },
+        /* 93*/ { UNICODE_MODE, 5, -1, "Ħ", 0, 5, "70 54 01 A1 00 EC 11 EC 11", "ECI-5 B1 (ISO 8859-3)" },
+        /* 94*/ { UNICODE_MODE, 0, -1, "ĸ", ZINT_WARN_USES_ECI, 6, "Warning 70 64 01 A2 00 EC 11 EC 11", "ECI-6 B1 (ISO 8859-4)" },
+        /* 95*/ { UNICODE_MODE, 6, -1, "ĸ", 0, 6, "70 64 01 A2 00 EC 11 EC 11", "ECI-6 B1 (ISO 8859-4)" },
+        /* 96*/ { UNICODE_MODE, 0, -1, "Ș", ZINT_WARN_USES_ECI, 18, "Warning 71 24 01 AA 00 EC 11 EC 11", "ECI-18 B1 (ISO 8859-16)" },
+        /* 97*/ { UNICODE_MODE, 18, -1, "Ș", 0, 18, "71 24 01 AA 00 EC 11 EC 11", "ECI-18 B1 (ISO 8859-16)" },
+        /* 98*/ { UNICODE_MODE, 0, -1, "テ", 0, 0, "80 10 D2 80 EC 11 EC 11 EC", "K1 (SHIFT JIS)" },
+        /* 99*/ { UNICODE_MODE, 20, -1, "テ", 0, 20, "71 48 01 0D 28 00 EC 11 EC", "ECI-20 K1 (SHIFT JIS)" },
+        /*100*/ { UNICODE_MODE, 20, -1, "テテ", 0, 20, "71 48 02 0D 28 69 40 EC 11", "ECI-20 K2 (SHIFT JIS)" },
+        /*101*/ { UNICODE_MODE, 20, -1, "\\\\", 0, 20, "71 48 02 00 F8 07 C0 EC 11", "ECI-20 K2 (SHIFT JIS)" },
+        /*102*/ { UNICODE_MODE, 0, -1, "…", 0, 0, "80 10 11 80 EC 11 EC 11 EC", "K1 (SHIFT JIS)" },
+        /*103*/ { UNICODE_MODE, 21, -1, "…", 0, 21, "71 54 01 85 00 EC 11 EC 11", "ECI-21 B1 (Win 1250)" },
+        /*104*/ { UNICODE_MODE, 0, -1, "Ґ", ZINT_WARN_USES_ECI, 22, "Warning 71 64 01 A5 00 EC 11 EC 11", "ECI-22 B1 (Win 1251)" },
+        /*105*/ { UNICODE_MODE, 22, -1, "Ґ", 0, 22, "71 64 01 A5 00 EC 11 EC 11", "ECI-22 B1 (Win 1251)" },
+        /*106*/ { UNICODE_MODE, 0, -1, "˜", ZINT_WARN_USES_ECI, 23, "Warning 71 74 01 98 00 EC 11 EC 11", "ECI-23 B1 (Win 1252)" },
+        /*107*/ { UNICODE_MODE, 23, -1, "˜", 0, 23, "71 74 01 98 00 EC 11 EC 11", "ECI-23 B1 (Win 1252)" },
+        /*108*/ { UNICODE_MODE, 24, -1, "پ", 0, 24, "71 84 01 81 00 EC 11 EC 11", "ECI-24 B1 (Win 1256)" },
+        /*109*/ { UNICODE_MODE, 0, -1, "က", ZINT_WARN_USES_ECI, 26, "Warning 71 A4 03 E1 80 80 00 EC 11", "ECI-26 B3 (UTF-8)" },
+        /*110*/ { UNICODE_MODE, 25, -1, "က", 0, 25, "71 94 02 10 00 00 EC 11 EC", "ECI-25 B2 (UCS-2BE)" },
+        /*111*/ { UNICODE_MODE, 25, -1, "ကက", 0, 25, "71 94 04 10 00 10 00 00 EC", "ECI-25 B4 (UCS-2BE)" },
+        /*112*/ { UNICODE_MODE, 25, -1, "12", 0, 25, "71 94 04 00 31 00 32 00 EC", "ECI-25 B4 (UCS-2BE ASCII)" },
+        /*113*/ { UNICODE_MODE, 27, -1, "@", 0, 27, "71 B4 01 40 00 EC 11 EC 11", "ECI-27 B1 (ASCII)" },
+        /*114*/ { UNICODE_MODE, 0, -1, "龘", ZINT_WARN_USES_ECI, 26, "Warning 71 A4 03 E9 BE 98 00 EC 11", "ECI-26 B3 (UTF-8)" },
+        /*115*/ { UNICODE_MODE, 28, -1, "龘", 0, 28, "71 C4 02 F9 D5 00 EC 11 EC", "ECI-28 B2 (Big5)" },
+        /*116*/ { UNICODE_MODE, 28, -1, "龘龘", 0, 28, "71 C4 04 F9 D5 F9 D5 00 EC", "ECI-28 B4 (Big5)" },
+        /*117*/ { UNICODE_MODE, 0, -1, "齄", ZINT_WARN_USES_ECI, 26, "Warning 71 A4 03 E9 BD 84 00 EC 11", "ECI-26 B3 (UTF-8)" },
+        /*118*/ { UNICODE_MODE, 29, -1, "齄", 0, 29, "71 D4 02 F7 FE 00 EC 11 EC", "ECI-29 B2 (GB 2312)" },
+        /*119*/ { UNICODE_MODE, 29, -1, "齄齄", 0, 29, "71 D4 04 F7 FE F7 FE 00 EC", "ECI-29 B4 (GB 2312)" },
+        /*120*/ { UNICODE_MODE, 0, -1, "가", ZINT_WARN_USES_ECI, 26, "Warning 71 A4 03 EA B0 80 00 EC 11", "ECI-26 B3 (UTF-8)" },
+        /*121*/ { UNICODE_MODE, 30, -1, "가", 0, 30, "71 E4 02 30 21 00 EC 11 EC", "ECI-30 B2 (KS X 1001)" },
+        /*122*/ { UNICODE_MODE, 30, -1, "가가", 0, 30, "71 E4 04 30 21 30 21 00 EC", "ECI-30 B4 (KS X 1001)" },
+        /*123*/ { UNICODE_MODE, 170, -1, "?", 0, 170, "78 0A A4 01 3F 00 EC 11 EC", "ECI-170 B1 (ASCII invariant)" },
+        /*124*/ { DATA_MODE, 899, -1, "\200", 0, 899, "78 38 34 01 80 00 EC 11 EC", "ECI-899 B1 (8-bit binary)" },
+        /*125*/ { UNICODE_MODE, 900, -1, "é", 0, 900, "78 38 44 02 C3 A9 00 EC 11", "ECI-900 B2 (no conversion)" },
     };
     int data_size = ARRAY_SIZE(data);
 
@@ -1412,7 +1448,7 @@ static void test_microqr_options(int index, int debug) {
         /* 47*/ { "ABCDEFGHIJABCDEFGH", 3, 4, ZINT_ERROR_TOO_LONG, -1, 0, -1 },
         /* 48*/ { "ABCDEFGHIJABC", 3, 4, 0, 0, 17, -1 }, // 13 alphanumerics, ECC 3 (Q), version 4
     };
-    int data_size = sizeof(data) / sizeof(struct item);
+    int data_size = ARRAY_SIZE(data);
 
     struct zint_symbol previous_symbol;
 
@@ -1423,16 +1459,7 @@ static void test_microqr_options(int index, int debug) {
         struct zint_symbol *symbol = ZBarcode_Create();
         assert_nonnull(symbol, "Symbol not created\n");
 
-        symbol->symbology = BARCODE_MICROQR;
+        int length = testUtilSetSymbol(symbol, BARCODE_MICROQR, -1 /*input_mode*/, -1 /*eci*/, data[i].option_1, data[i].option_2, -1, -1 /*output_options*/, data[i].data, -1, debug);
-        if (data[i].option_1 != -1) {
-            symbol->option_1 = data[i].option_1;
-        }
-        if (data[i].option_2 != -1) {
-            symbol->option_2 = data[i].option_2;
-        }
-        symbol->debug |= debug;
-
-        int length = strlen(data[i].data);
 
         ret = ZBarcode_Encode(symbol, (unsigned char *) data[i].data, length);
         assert_equal(ret, data[i].ret_encode, "i:%d ZBarcode_Encode ret %d != %d (%s)\n", i, ret, data[i].ret_encode, symbol->errtxt);
@@ -1524,7 +1551,7 @@ static void test_microqr_input(int index, int generate, int debug) {
         /* 39*/ { UNICODE_MODE, -1, "áA", 0, "8B 85 04 00 EC 11 EC 11 00", "B2 (ISO 8859-1)" },
         /* 40*/ { UNICODE_MODE, 200, "áA", 0, "CE 00 40 00 EC 11 EC 11 00", "K1 (ISO 8859-1) (full multibyte)" },
     };
-    int data_size = sizeof(data) / sizeof(struct item);
+    int data_size = ARRAY_SIZE(data);
 
     char escaped[1024];
 
@@ -1535,15 +1562,9 @@ static void test_microqr_input(int index, int generate, int debug) {
         struct zint_symbol *symbol = ZBarcode_Create();
         assert_nonnull(symbol, "Symbol not created\n");
 
-        symbol->symbology = BARCODE_MICROQR;
+        debug |= ZINT_DEBUG_TEST; // Needed to get codeword dump in errtxt
-        symbol->input_mode = data[i].input_mode;
-        if (data[i].option_3 != -1) {
-            symbol->option_3 = data[i].option_3;
-        }
-        symbol->debug = ZINT_DEBUG_TEST; // Needed to get codeword dump in errtxt
-        symbol->debug |= debug;
 
-        int length = strlen(data[i].data);
+        int length = testUtilSetSymbol(symbol, BARCODE_MICROQR, data[i].input_mode, -1 /*eci*/, -1 /*option_1*/, -1, data[i].option_3, -1 /*output_options*/, data[i].data, -1, debug);
 
         ret = ZBarcode_Encode(symbol, (unsigned char *) data[i].data, length);
         assert_equal(ret, data[i].ret, "i:%d ZBarcode_Encode ret %d != %d\n", i, ret, data[i].ret);

backend/tests/test_sjis.c

@@ -1,6 +1,6 @@
 /*
     libzint - the open source barcode library
-    Copyright (C) 2019 - 2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2019 - 2021 Robin Stuart <rstuart114@gmail.com>
 
     Redistribution and use in source and binary forms, with or without
     modification, are permitted provided that the following conditions
@@ -68,7 +68,9 @@ static int sjis_wctomb_zint2(unsigned int *r, unsigned int wc) {
         return 2;
     }
     int tab_length = sizeof(test_sjis_tab) / sizeof(unsigned int);
-    for (int i = test_sjis_tab_ind[wc >> 12]; i < tab_length; i += 2) {
+    int start_i = test_sjis_tab_ind[wc >> 10];
+    int end_i = start_i + 0x800 > tab_length ? tab_length : start_i + 0x800;
+    for (int i = start_i; i < end_i; i += 2) {
         if (test_sjis_tab[i + 1] == wc) {
             *r = test_sjis_tab[i];
             return *r > 0xFF ? 2 : 1;
@@ -106,7 +108,7 @@ static void test_sjis_wctomb_zint(void) {
     testFinish();
 }
 
-static void test_sjis_utf8tomb(int index) {
+static void test_sjis_utf8(int index) {
 
     testStart("");
 
@@ -151,7 +153,7 @@ static void test_sjis_utf8tomb(int index) {
         int length = data[i].length == -1 ? (int) strlen(data[i].data) : data[i].length;
         int ret_length = length;
 
-        ret = sjis_utf8tomb(&symbol, (unsigned char *) data[i].data, &ret_length, jisdata);
+        ret = sjis_utf8(&symbol, (unsigned char *) data[i].data, &ret_length, jisdata);
         assert_equal(ret, data[i].ret, "i:%d ret %d != %d (%s)\n", i, ret, data[i].ret, symbol.errtxt);
         if (ret == 0) {
             assert_equal(ret_length, data[i].ret_length, "i:%d ret_length %d != %d\n", i, ret_length, data[i].ret_length);
@@ -164,7 +166,7 @@ static void test_sjis_utf8tomb(int index) {
     testFinish();
 }
 
-static void test_sjis_utf8tosb(int index) {
+static void test_sjis_utf8_to_eci(int index) {
 
     testStart("");
 
@@ -204,6 +206,29 @@ static void test_sjis_utf8tosb(int index) {
         /* 11*/ { 3, 1, "éaúbàcëdìeµ", -1, 0, 8, { 0xE961, 0xFA, 0x62, 0xE063, 0xEB64, 0xEC, 0x65, 0xB5 }, "" },
         /* 12*/ { 3, 0, "ëÀ", -1, 0, 2, { 0xEB, 0xC0 }, "Not full multibyte" },
         /* 13*/ { 3, 1, "ëÀ", -1, 0, 2, { 0xEB, 0xC0 }, "Outside QR Kanji mode range" },
+        /* 14*/ { 20, 0, "\\\\", -1, 0, 4, { 0x81, 0x5F, 0x81, 0x5F }, "Shift JIS reverse solidus (backslash) mapping from ASCII to double byte" },
+        /* 15*/ { 20, 1, "\\\\", -1, 0, 2, { 0x815F, 0x815F }, "Shift JIS reverse solidus (backslash) mapping from ASCII to double byte" },
+        /* 16*/ { 20, 0, "爍", -1, 0, 2, { 0xE0, 0xA1 }, "Shift JIS U+720D" },
+        /* 17*/ { 20, 1, "爍", -1, 0, 1, { 0xE0A1 }, "Shift JIS" },
+        /* 18*/ { 20, 0, "~", -1, ZINT_ERROR_INVALID_DATA, -1, {}, "ASCII tilde not in Shift JIS" },
+        /* 19*/ { 25, 0, "12", -1, 0, 4, { 0x00, 0x31, 0x00, 0x32 }, "UCS-2BE ASCII" },
+        /* 20*/ { 25, 0, "", -1, 0, 4, { 0x00, 0x81, 0x00, 0x81 }, "UCS-2BE U+0081" },
+        /* 21*/ { 25, 1, "", -1, 0, 4, { 0x00, 0x81, 0x00, 0x81 }, "UCS-2BE outside QR Kanji mode range" },
+        /* 22*/ { 25, 0, "腀", -1, 0, 2, { 0x81, 0x40 }, "UCS-2BE U+8140" },
+        /* 23*/ { 25, 1, "腀", -1, 0, 1, { 0x8140 }, "UCS-2BE in QR Kanji mode range" },
+        /* 24*/ { 28, 0, "¢¢", -1, 0, 4, { 0xA2, 0x46, 0xA2, 0x46 }, "Big5 U+00A2" },
+        /* 25*/ { 28, 1, "¢¢", -1, 0, 4, { 0xA2, 0x46, 0xA2, 0x46 }, "Big5 outside QR Kanji mode range" },
+        /* 26*/ { 28, 0, "觡", -1, 0, 2, { 0xE0, 0x40 }, "Big5 U+89E1" },
+        /* 27*/ { 28, 1, "觡", -1, 0, 1, { 0xE040 }, "Big5 in QR Kanji mode range" },
+        /* 28*/ { 29, 0, "¨¨", -1, 0, 4, { 0xA1, 0xA7, 0xA1, 0xA7 }, "GB 2312 U+00A8" },
+        /* 29*/ { 29, 1, "¨¨", -1, 0, 4, { 0xA1, 0xA7, 0xA1, 0xA7 }, "GB 2312 outside QR Kanji mode range" },
+        /* 30*/ { 29, 0, "崂", -1, 0, 2, { 0xE1, 0xC0 }, "GB 2312 U+5D02" },
+        /* 31*/ { 29, 0, "釦", -1, ZINT_ERROR_INVALID_DATA, -1, {}, "GB 18030 U+91E6 not in GB 2312" },
+        /* 32*/ { 29, 1, "崂", -1, 0, 1, { 0xE1C0 }, "GB 2312 in QR Kanji mode range" },
+        /* 33*/ { 30, 0, "¡¡", -1, 0, 4, { 0x22, 0x2E, 0x22, 0x2E }, "KS X 1001 U+00A1" },
+        /* 34*/ { 30, 1, "¡¡", -1, 0, 4, { 0x22, 0x2E, 0x22, 0x2E }, "KS X 1001 outside QR Kanji mode range" },
+        /* 35*/ { 30, 0, "詰", -1, 0, 2, { 0x7D, 0x7E }, "KS X 1001 U+8A70" },
+        /* 36*/ { 30, 1, "詰", -1, 0, 2, { 0x7D, 0x7E }, "KS X 1001 <= 0x7D7E so none in QR Kanji mode range" },
     };
 
     int data_size = sizeof(data) / sizeof(struct item);
@@ -217,7 +242,7 @@ static void test_sjis_utf8tosb(int index) {
         int length = data[i].length == -1 ? (int) strlen(data[i].data) : data[i].length;
         int ret_length = length;
 
-        ret = sjis_utf8tosb(data[i].eci, (unsigned char *) data[i].data, &ret_length, jisdata, data[i].full_multibyte);
+        ret = sjis_utf8_to_eci(data[i].eci, (unsigned char *) data[i].data, &ret_length, jisdata, data[i].full_multibyte);
         assert_equal(ret, data[i].ret, "i:%d ret %d != %d\n", i, ret, data[i].ret);
         if (ret == 0) {
             assert_equal(ret_length, data[i].ret_length, "i:%d ret_length %d != %d\n", i, ret_length, data[i].ret_length);
@@ -281,8 +306,8 @@ int main(int argc, char *argv[]) {
 
     testFunction funcs[] = { /* name, func, has_index, has_generate, has_debug */
         { "test_sjis_wctomb_zint", test_sjis_wctomb_zint, 0, 0, 0 },
-        { "test_sjis_utf8tomb", test_sjis_utf8tomb, 1, 0, 0 },
+        { "test_sjis_utf8", test_sjis_utf8, 1, 0, 0 },
-        { "test_sjis_utf8tosb", test_sjis_utf8tosb, 1, 0, 0 },
+        { "test_sjis_utf8_to_eci", test_sjis_utf8_to_eci, 1, 0, 0 },
         { "test_sjis_cpy", test_sjis_cpy, 1, 0, 0 },
     };
 

backend/tests/test_sjis_tab.h

@@ -7040,20 +7040,68 @@ static const unsigned int test_sjis_tab[] = {
 };
 
 static const unsigned int test_sjis_tab_ind[] = {
-0,
+    0,
-440,
+    308,
-440,
+    440,
-656,
+    440,
-1054,
+    440,
-1466,
+    440,
-4228,
+    440,
-7042,
+    440,
-9404,
+    440,
-11812,
+    554,
-13766,
+    656,
-13766,
+    656,
-13766,
+    656,
-13766,
+    1054,
-13766,
+    1054,
-13766,
+    1054,
+    1054,
+    1054,
+    1054,
+    1054,
+    1466,
+    2286,
+    2856,
+    3498,
+    4228,
+    5006,
+    5734,
+    6408,
+    7042,
+    7520,
+    8144,
+    8760,
+    9404,
+    10084,
+    10628,
+    11256,
+    11812,
+    12376,
+    12876,
+    13372,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
+    13766,
 };

backend/tests/tools/gen_test_tab.php

@@ -2,7 +2,7 @@
 /* Generate lookup table from unicode.org mapping file (SHIFTJIS.TXT by default). */
 /*
     libzint - the open source barcode library
-    Copyright (C) 2008-2019 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2019-2021 Robin Stuart <rstuart114@gmail.com>
 */
 /* To create backend/tests/test_sjis_tab.h (from backend/tests/build directory):
  *
@@ -17,6 +17,15 @@
  * using the version libiconv-1.11/GB18030.TXT):
  *
  *   php ../tools/gen_test_tab.php -f GB18030.TXT -s gb18030_tab
+ *
+ * To create backend/tests/test_big5_tab.h;
+ *
+ *   php ../tools/gen_test_tab.php -f BIG5.TXT -s big5_tab
+ *
+ * To create backend/tests/test_ksx1001_tab.h;
+ *
+ *   php ../tools/gen_test_tab.php -f KSX1001.TXT -s ksx1001_tab
+ *
  */
 /* vim: set ts=4 sw=4 et : */
 
@@ -46,7 +55,7 @@ $tab_lines = array();
 $sort = array();
 foreach ($lines as $line) {
     $line = trim($line);
-    if ($line === '' || strncmp($line, '0x', 2) !== 0) {
+    if ($line === '' || strncmp($line, '0x', 2) !== 0 || strpos($line, "*** NO MAPPING ***") !== false) {
         continue;
     }
     if (preg_match('/^0x([0-9A-F]{2,8})[ \t]+0x([0-9A-F]{5})/', $line)) { // Exclude U+10000..10FFFF to save space
@@ -75,9 +84,9 @@ $out[] = '';
 $out[] = 'static const unsigned int test_' . $suffix_name . '_ind[] = {';
 $first = 0;
 foreach ($sort as $ind => $unicode) {
-    $div = (int)($unicode / 0x1000);
+    $div = (int)($unicode / 0x400);
     while ($div >= $first) {
-        $out[] = ($ind * 2) . ',';
+        $out[] = '    ' . ($ind * 2) . ',';
         $first++;
     }
 }

docs/manual.txt

@@ -196,7 +196,7 @@ output file will be out.gif.
 
 The data input to Zint is assumed to be encoded in Unicode (UTF-8) format. If
 you are encoding characters beyond the 7-bit ASCII set using a scheme other than
-Unicode then you will need to set the appropriate input options as shown in
+UTF-8 then you will need to set the appropriate input options as shown in
 section 4.11 below.
 
 Non-printing characters can be entered on the command line using the backslash
@@ -449,11 +449,11 @@ example for PNG images a scale of 5 will increase the X-dimension to 10 pixels.
 4.10 Input modes
 ----------------
 By default all input data is assumed to be encoded in Unicode (UTF-8) format.
-Many barcode symbologies encode data using Latin-1 (ISO-8859-1) character
+Many barcode symbologies encode data using Latin-1 (ISO/IEC 8859-1) character
-encoding, so input is converted from Unicode to Latin-1 before being put in the
+encoding, so input is converted from UTF-8 to Latin-1 before being put in the
 symbol. In addition QR Code, Micro QR Code, Rectangular Micro QR Code, Han Xin
 Code and Grid Matrix can encode Japanese or Chinese characters which are also
-converted from Unicode. If Zint encounters characters which can not be encoded
+converted from UTF-8. If Zint encounters characters which can not be encoded
 using the default character encoding then it will take advantage of the ECI
 (Extended Channel Interpretations) mechanism to encode the data. Be aware that
 not all barcode readers support ECI mode, so this can sometimes lead to
@@ -476,8 +476,8 @@ Identification Code (HIBC LIC). For HIBC Provider Applications Standard
 (HIBC PAS), preface the data with a slash "/".
 
 The --binary option encodes the input data as given. Automatic code page
-translations to ECI pages is disabled. This may be used for raw binary or binary
+translations to ECI pages is disabled, and no validation of the data's encoding
-encrypted data.
+takes place. This may be used for raw binary or binary encrypted data.
 This switch plays together with the built-in ECI logic and examples may
 be found in that section.
 
@@ -497,7 +497,7 @@ The ECI information is added to your code symbol as prefix data.
 The ECI value may be specified with the --eci switch, followed by the value in
 the column "ECI Code".
 The ECI value of 0 does not encode any ECI information in the code symbol. In
-this case, the default encoding applies for the data which is "ISO-8859-1 -
+this case, the default encoding applies for the data which is "ISO/IEC 8859-1 -
 Latin alphabet No. 1".
 
 The first row of the table (ECI code 3) is the default value and does not lead
@@ -505,65 +505,59 @@ to any ECI information being included in the symbol.
 
 The input data should be UTF-8 formatted. Zint automatically translates the
 data into the target encoding.
-The rows marked with a star (*) do not do this transformation. The data must be
+
-specified as binary data (--binary switch) with the data in the encoding given
+The row marked with a star (*) translates GB 2312 codepoints, except when using
-by the "Character Encoding Scheme" column.
+Han Xin Code, which translates GB 18030 codepoints, a superset of GB 2312.
-The row marked with a double star (**) only does this transformation for QR
-Code, Micro QR Code and Rectangular Micro QR Code.
-The row marked with a triple star (***) only does this transformation for Han
-Xin Code and Grid Matrix. Han Xin Code can encode GB 18030. Grid Matrix can
-encode the subset GB 2312.
 
 Note: the "--eci 3" specification should only be used for special purposes.
 Using this parameter, the ECI information is explicitly added to the code
 symbol. Nevertheless, for ECI Code 3, this is not required, as this is the
 default encoding, which is also active without any ECI information.
 
---------------------------------------------------------
+------------------------------------------------------------
 ECI Code  |  Character Encoding Scheme
---------------------------------------------------------
+------------------------------------------------------------
-3         |  ISO-8859-1 - Latin alphabet No. 1
+3         |  ISO/IEC 8859-1 - Latin alphabet No. 1
-4         |  ISO-8859-2 - Latin alphabet No. 2
+4         |  ISO/IEC 8859-2 - Latin alphabet No. 2
-5         |  ISO-8859-3 - Latin alphabet No. 3
+5         |  ISO/IEC 8859-3 - Latin alphabet No. 3
-6         |  ISO-8859-4 - Latin alphabet No. 4
+6         |  ISO/IEC 8859-4 - Latin alphabet No. 4
-7         |  ISO-8859-5 - Latin/Cyrillic alphabet
+7         |  ISO/IEC 8859-5 - Latin/Cyrillic alphabet
-8         |  ISO-8859-6 - Latin/Arabic alphabet
+8         |  ISO/IEC 8859-6 - Latin/Arabic alphabet
-9         |  ISO-8859-7 - Latin/Greek alphabet
+9         |  ISO/IEC 8859-7 - Latin/Greek alphabet
-10        |  ISO-8859-8 - Latin/Hebrew alphabet
+10        |  ISO/IEC 8859-8 - Latin/Hebrew alphabet
-11        |  ISO-8859-9 - Latin alphabet No. 5
+11        |  ISO/IEC 8859-9 - Latin alphabet No. 5 (Turkish)
-12        |  ISO-8859-10 - Latin alphabet No. 6
+12        |  ISO/IEC 8859-10 - Latin alphabet No. 6 (Nordic)
-13        |  ISO-8859-11 - Latin/Thai alphabet
+13        |  ISO/IEC 8859-11 - Latin/Thai alphabet
-15        |  ISO-8859-13 - Latin alphabet No. 7
+15        |  ISO/IEC 8859-13 - Latin alphabet No. 7 (Baltic)
-16        |  ISO-8859-14 - Latin alphabet No. 8 (Celtic)
+16        |  ISO/IEC 8859-14 - Latin alphabet No. 8 (Celtic)
-17        |  ISO-8859-15 - Latin alphabet No. 9
+17        |  ISO/IEC 8859-15 - Latin alphabet No. 9
-18        |  ISO-8859-16 - Latin alphabet No. 10
+18        |  ISO/IEC 8859-16 - Latin alphabet No. 10
-20 **     |  Shift-JIS (JISX 0208 amd JISX 0201)
+20        |  Shift JIS (JIS X 0208 amd JIS X 0201)
 21        |  Windows-1250 - Latin 2 (Central Europe)
 22        |  Windows-1251 - Cyrillic
 23        |  Windows-1252 - Latin 1
 24        |  Windows-1256 - Arabic
-25 *      |  UCS-2 Unicode (High order byte first)
+25        |  UCS-2BE (High order byte first) (Unicode BMP)
-26        |  Unicode (UTF-8)
+26        |  UTF-8 (Unicode)
-27        |  ISO-646:1991 7-bit character set
+27        |  ISO/IEC 646:1991 7-bit character set (ASCII)
-28 *      |  Big5 (Taiwan) Chinese Character Set
+28        |  Big5 (Taiwan) Chinese Character Set
-29 ***    |  GB (PRC) Chinese Character Set
+29 *      |  GB (PRC) Chinese Character Set
-30 *      |  Korean Character Set (KSX1001:1998)
+30        |  Korean Character Set (KS X 1001:2002)
---------------------------------------------------------
+899       |  8-bit binary data
+------------------------------------------------------------
 
 Three examples:
-Ex1: The Euro sign can be encoded in ISO-8859-15.
+Ex1: The Euro sign U+20AC can be encoded in ISO/IEC 8859-15.
-The Euro sign has the ISO-8859-15 codepoint hex A4.
+The Euro sign has the ISO/IEC 8859-15 codepoint hex A4.
 It is encoded in UTF-8 as the hex sequence: e2 82 ac
 Those 3 bytes are contained in the file "utf8euro.txt"
 This command will generate the corresponding code:
 
 zint.exe -b 71 --square --scale 10 --eci 17 -i utf8euro.txt
 
-Ex2: The Chinese character with Unicode codepoint hex 5E38 can be encoded in
+Ex2: The Chinese character with Unicode codepoint U+5E38 can be encoded in Big5
-Big5 encoding. The Big5 ECI is marked in the upper table to require input data
+encoding. The Big5 representation of this character is the two hex bytes: 9C 75
-in Big5 instead of UTF-8. The Big5 representation of this character is the two
+(contained in the file big5char.txt). The generation command for Data Matrix is:
-hex bytes: 9C 75 (contained in the file big5char.txt).
-The generation command for Data Matrix is:
 
 zint -b 71 --square --scale 10 --eci 28 --binary -i big5char.txt
 
@@ -2062,8 +2056,8 @@ When using automatic symbol sizes you can force Zint to use square symbols
 (versions 1-24) at the command line by using the option --square and when
 using the API by setting the value option_3 = DM_SQUARE.
 
-Data Matrix Rectangular Extension (ISO/IEC21471) codes may be generated with the
+Data Matrix Rectangular Extension (ISO/IEC 21471) codes may be generated with
-following values as before:
+the following values as before:
 
 ---------------------
 Input  |  Symbol Size
@@ -2162,10 +2156,10 @@ Input  |  Symbol Size
 The maximum capacity of a (version 40) QR Code symbol is 7089 numeric digits,
 4296 alphanumeric characters or 2953 bytes of data. QR Code symbols can also be
 used to encode GS1 data. QR Code symbols can by default encode characters in
-the Latin-1 set and Kanji characters which are members of the Shift-JIS
+the Latin-1 set and Kanji characters which are members of the Shift JIS
 encoding scheme. In addition QR Code supports using other character sets using
 the ECI mechanism. Input should usually be entered as Unicode (UTF-8) with
-conversion to Shift-JIS being carried out by Zint. A separate symbology ID can
+conversion to Shift JIS being carried out by Zint. A separate symbology ID can
 be used to encode Health Industry Barcode (HIBC) data which adds a leading '+'
 character and a modulo-49 check digit to the encoded data.
 
@@ -2183,8 +2177,8 @@ ZINT_FULL_MULTIBYTE | (N + 1) << 8.
 -------------------------------
 A miniature version of the QR Code symbol for short messages. ECC levels can be
 selected as for QR Code (above). QR Code symbols can encode characters in the
-Latin-1 set and Kanji characters which are members of the Shift-JIS encoding
+Latin-1 set and Kanji characters which are members of the Shift JIS encoding
-scheme. Input should be entered as a UTF-8 stream with conversion to Shift-JIS
+scheme. Input should be entered as a UTF-8 stream with conversion to Shift JIS
 being carried out automatically by Zint. A preferred symbol size can be
 selected by using the --vers= option or by setting option_2 although the actual
 version used by Zint may be different if required by the input data. The table
@@ -2211,11 +2205,12 @@ ZINT_FULL_MULTIBYTE | (N + 1) << 8.
 6.6.4 Rectangular Micro QR Code (rMQR)
 --------------------------------------
 A rectangular version of QR Code. Like QR code rMQR supports encoding of GS1
-data, Latin-1 and Kanji characters in the Shift-JIS encoding scheme.
+data, Latin-1 and Kanji characters in the Shift JIS encoding scheme. It does not
-It does not support other ISO 8859 character sets or Unicode. As with other
+support other ISO/IEC 8859 character sets or encodings. As with other
-symbologies data should be entered as UTF-8 with the conversion to Shift-JIS
+symbologies data should be entered as UTF-8 with the conversion to Shift JIS
-being handled by Zint. The amount of ECC codewords can be adjusted using
+being handled by Zint. The amount of ECC codewords can be adjusted using the
---secure=, however only ECC levels M and H are valid for this type of symbol.
+--secure= option (API option_1), however only ECC levels M and H are valid for
+this type of symbol.
 
 -------------------------------------------------------------------------
 Input  |  ECC Level    |  Error Correction Capacity  |  Recovery Capacity
@@ -2224,9 +2219,9 @@ Input  |  ECC Level    |  Error Correction Capacity  |  Recovery Capacity
 4      |  H            |  Approx 65% of symbol       |  Approx 30%
 -------------------------------------------------------------------------
 
-The preferred symbol sizes can be selected using the --vers= option as shown
+The preferred symbol sizes can be selected using the --vers= option (API
-in the table below. Input values between 33 and 38 fix the height of the
+option_2) as shown in the table below. Input values between 33 and 38 fix the
-symbol while allowing Zint to determine the minimum symbol width.
+height of the symbol while allowing Zint to determine the minimum symbol width.
 
 ---------------------------------
 Input  |  Version  |  Symbol Size
@@ -2279,12 +2274,13 @@ using the --fullmultibyte switch or by setting option_3 to ZINT_FULL_MULTIBYTE.
 ------------------------------------------------
 A variation of QR Code used by Združenje Bank Slovenije (Bank Association of
 Slovenia). The size, error correction level and ECI are set by Zint and do not
-need to be specified. UPNQR is unusual in that it uses ISO-8859-2 formatted
+need to be specified. UPNQR is unusual in that it uses ISO/IEC 8859-2 formatted
-data. Zint will accept UTF-8 data and convert it to ISO-8859-2, or if your data
+data. Zint will accept UTF-8 data and convert it to ISO/IEC 8859-2, or if your
-is already ISO-8859-2 formatted use the --binary switch or if using the API set
+data is already ISO/IEC 8859-2 formatted use the --binary switch or if using the
-symbol->input_mode = DATA MODE;
+API set symbol->input_mode = DATA MODE;
 
-The following example creates a symbol from data saved as an ISO-8859-2 file:
+The following example creates a symbol from data saved as an ISO/IEC 8859-2
+file:
 
 zint -o upnqr.png -b 143 --border=5 --scale=3 --binary -i ./upn.txt
 
@@ -2719,7 +2715,7 @@ are ignored.
 ================================
 7.1 License
 -----------
-Zint, libzint and Zint Barcode Studio are Copyright © 2020 Robin Stuart. All
+Zint, libzint and Zint Barcode Studio are Copyright © 2021 Robin Stuart. All
 historical versions are distributed under the GNU General Public License
 version 3 or later. Version 2.5 is released under a dual license: the encoding
 library is released under the BSD license whereas the GUI, Zint Barcode Studio,
@@ -3085,11 +3081,11 @@ E   |  SO    |  RS   |  .      |  >  |  N  |  ^  |  n  |  ~
 F   |  SI    |  US   |  /      |  ?  |  O  |  _  |  o  |  DEL
 -------------------------------------------------------------
 
-A.2 Latin Alphabet No 1 (ISO 8859-1)
+A.2 Latin Alphabet No 1 (ISO/IEC 8859-1)
-------------------------------------
+----------------------------------------
 A common extension to the ASCII standard, Latin-1 is used to expand the range
 of Code 128, PDF417 and other symbols. Input strings should be in Unicode
-format
+(UTF-8) format
 
 ------------------------------------------------------
 Hex |  8  |  9  |  A      |  B  |  C  |  D  |  E  |  F
@@ -3109,6 +3105,6 @@ B   |     |     |  «      |  »  |  Ë  |  Û  |  ë  |  û
 C   |     |     |  ¬      |  ¼  |  Ì  |  Ü  |  ì  |  ü
 D   |     |     |  SHY    |  ½  |  Í  |  Ý  |  í  |  ý
 E   |     |     |  ®      |  ¾  |  Î  |  Þ  |  î  |  þ
-F   |     |     |  ¯      |  ¿  |  Ï  |  ß  |  î  |  ÿ
+F   |     |     |  ¯      |  ¿  |  Ï  |  ß  |  ï  |  ÿ
 ------------------------------------------------------
 

frontend_qt/datawindow.cpp

@@ -1,6 +1,6 @@
 /*
     Zint Barcode Generator - the open source barcode generator
-    Copyright (C) 2009 - 2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2009 - 2021 Robin Stuart <rstuart114@gmail.com>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -73,7 +73,9 @@ void DataWindow::okay()
 void DataWindow::from_file()
 {
     QSettings settings;
+#if QT_VERSION < 0x60000
     settings.setIniCodec("UTF-8");
+#endif
     QFileDialog open_dialog;
     QString filename;
     QFile file;

frontend_qt/exportwindow.cpp

@@ -1,6 +1,6 @@
 /*
     Zint Barcode Generator - the open source barcode generator
-    Copyright (C) 2009 - 2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2009 - 2021 Robin Stuart <rstuart114@gmail.com>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -29,7 +29,9 @@
 ExportWindow::ExportWindow()
 {
     QSettings settings;
+#if QT_VERSION < 0x60000
     settings.setIniCodec("UTF-8");
+#endif
     setupUi(this);
 
     linDestPath->setText(settings.value("studio/export/destination", QDir::toNativeSeparators(QDir::homePath())).toString());
@@ -45,7 +47,9 @@ ExportWindow::ExportWindow()
 ExportWindow::~ExportWindow()
 {
     QSettings settings;
+#if QT_VERSION < 0x60000
     settings.setIniCodec("UTF-8");
+#endif
 
     settings.setValue("studio/export/destination", linDestPath->text());
     settings.setValue("studio/export/file_prefix", linPrefix->text());
@@ -61,7 +65,9 @@ void ExportWindow::quit_now()
 void ExportWindow::get_directory()
 {
     QSettings settings;
+#if QT_VERSION < 0x60000
     settings.setIniCodec("UTF-8");
+#endif
     QString directory;
     QFileDialog fdialog;
 

frontend_qt/frontend_qt.pro

@@ -20,6 +20,7 @@ HEADERS += barcodeitem.h \
          sequencewindow.h \
          qzint.h \
         ..\backend\aztec.h \
+        ..\backend\big5.h \
         ..\backend\bmp.h \
         ..\backend\channel_precalcs.h \
         ..\backend\code1.h \
@@ -29,6 +30,7 @@ HEADERS += barcodeitem.h \
         ..\backend\composite.h \
         ..\backend\dmatrix.h \
         ..\backend\eci.h \
+        ..\backend\eci_sb.h \
         ..\backend\emf.h \
         ..\backend\font.h \
         ..\backend\gb18030.h \
@@ -37,6 +39,7 @@ HEADERS += barcodeitem.h \
         ..\backend\gridmtx.h \
         ..\backend\gs1.h \
         ..\backend\hanxin.h \
+        ..\backend\ksx1001.h \
         ..\backend\large.h \
         ..\backend\maxicode.h \
         ..\backend\ms_stdint.h \

frontend_qt/mainWindow.ui

@@ -394,82 +394,82 @@ p, li { white-space: pre-wrap; }
                </item>
                <item>
                 <property name="text">
-                 <string>3: ISO-8859-1 Latin-1</string>
+                 <string>3: ISO 8859-1 Latin-1</string>
                 </property>
                </item>
                <item>
                 <property name="text">
-                 <string>4: ISO-8859-2 Latin-2</string>
+                 <string>4: ISO 8859-2 Latin-2</string>
                 </property>
                </item>
                <item>
                 <property name="text">
-                 <string>5: ISO-8859-3 Latin-3</string>
+                 <string>5: ISO 8859-3 Latin-3</string>
                 </property>
                </item>
                <item>
                 <property name="text">
-                 <string>6: ISO-8859-4 Latin-4</string>
+                 <string>6: ISO 8859-4 Latin-4</string>
                 </property>
                </item>
                <item>
                 <property name="text">
-                 <string>7: ISO-8859-5 Cyrillic</string>
+                 <string>7: ISO 8859-5 Cyrillic</string>
                 </property>
                </item>
                <item>
                 <property name="text">
-                 <string>8: ISO-8859-6 Arabic</string>
+                 <string>8: ISO 8859-6 Arabic</string>
                 </property>
                </item>
                <item>
                 <property name="text">
-                 <string>9: ISO-8859-7 Greek</string>
+                 <string>9: ISO 8859-7 Greek</string>
                 </property>
                </item>
                <item>
                 <property name="text">
-                 <string>10: ISO-8859-8 Hebrew</string>
+                 <string>10: ISO 8859-8 Hebrew</string>
                 </property>
                </item>
                <item>
                 <property name="text">
-                 <string>11: ISO-8859-9 Turkish</string>
+                 <string>11: ISO 8859-9 Turkish</string>
                 </property>
                </item>
                <item>
                 <property name="text">
-                 <string>12: ISO-8859-10 Nordic</string>
+                 <string>12: ISO 8859-10 Nordic</string>
                 </property>
                </item>
                <item>
                 <property name="text">
-                 <string>13: ISO-8859-11 Thai</string>
+                 <string>13: ISO 8859-11 Thai</string>
                 </property>
                </item>
                <item>
                 <property name="text">
-                 <string>15: ISO-8859-13 Baltic</string>
+                 <string>15: ISO 8859-13 Baltic</string>
                 </property>
                </item>
                <item>
                 <property name="text">
-                 <string>16: ISO-8859-14 Celtic</string>
+                 <string>16: ISO 8859-14 Celtic</string>
                 </property>
                </item>
                <item>
                 <property name="text">
-                 <string>17: ISO-8859-15 Latin-9</string>
+                 <string>17: ISO 8859-15 Latin-9</string>
                 </property>
                </item>
                <item>
                 <property name="text">
-                 <string>18: ISO-8859-16 Latin-10</string>
+                 <string>18: ISO 8859-16 Latin-10</string>
                 </property>
                </item>
                <item>
                 <property name="text">
-                 <string>20: Shift-JIS </string>
+                 <string>20: Shift JIS </string>
                 </property>
                </item>
                <item>
@@ -494,7 +494,7 @@ p, li { white-space: pre-wrap; }
                </item>
                <item>
                 <property name="text">
-                 <string>25: UCS-2 (BE)</string>
+                 <string>25: UCS-2BE</string>
                 </property>
                </item>
                <item>
@@ -504,7 +504,7 @@ p, li { white-space: pre-wrap; }
                </item>
                <item>
                 <property name="text">
-                 <string>27: ISO-646 ASCII</string>
+                 <string>27: ISO 646 ASCII</string>
                 </property>
                </item>
                <item>
@@ -514,12 +514,17 @@ p, li { white-space: pre-wrap; }
                </item>
                <item>
                 <property name="text">
-                 <string>29: GB (PRC)</string>
+                 <string>29: GB 2312 (PRC)</string>
                 </property>
                </item>
                <item>
                 <property name="text">
-                 <string>30: Korean</string>
+                 <string>30: Korean KS X 1001</string>
+                </property>
+               </item>
+               <item>
+                <property name="text">
+                 <string>899: 8-bit binary data</string>
                 </property>
                </item>
               </widget>

frontend_qt/mainwindow.cpp

@@ -1,6 +1,6 @@
 /***************************************************************************
  *   Copyright (C) 2008 by BogDan Vatra <bogdan@licentia.eu>               *
- *   Copyright (C) 2009-2020 by Robin Stuart <rstuart114@gmail.com>        *
+ *   Copyright (C) 2009-2021 by Robin Stuart <rstuart114@gmail.com>        *
  *                                                                         *
  *   This program is free software: you can redistribute it and/or modify  *
  *   it under the terms of the GNU General Public License as published by  *
@@ -46,7 +46,9 @@ MainWindow::MainWindow(QWidget* parent, Qt::WindowFlags fl)
     QCoreApplication::setApplicationName("Barcode Studio");
 
     QSettings settings;
+#if QT_VERSION < 0x60000
     settings.setIniCodec("UTF-8");
+#endif
 
     char bstyle_text[][50] = {
         "Australia Post Redirect Code",
@@ -225,7 +227,9 @@ MainWindow::MainWindow(QWidget* parent, Qt::WindowFlags fl)
 MainWindow::~MainWindow()
 {
     QSettings settings;
+#if QT_VERSION < 0x60000
     settings.setIniCodec("UTF-8");
+#endif
 
     settings.setValue("studio/window_geometry", saveGeometry());
     settings.setValue("studio/tab_index", tabMain->currentIndex());
@@ -278,7 +282,9 @@ void MainWindow::reset_view()
 bool MainWindow::save()
 {
     QSettings settings;
+#if QT_VERSION < 0x60000
     settings.setIniCodec("UTF-8");
+#endif
     QFileDialog save_dialog;
     QString filename;
     QString suffix;
@@ -367,7 +373,7 @@ void MainWindow::about()
            "<p>A free barcode generator"
            "<p>Instruction manual is available at the project homepage:<br>"
            "<a href=\"http://www.zint.org.uk\">http://www.zint.org.uk</a>"
-           "<p>Copyright &copy; 2006-2020 Robin Stuart and others.<br>"
+           "<p>Copyright &copy; 2006-2021 Robin Stuart and others.<br>"
            "Qt back end by BogDan Vatra<br>"
            "Windows port by Harald Oehlmann</p>"
            "<p>Qt version %2"
@@ -583,7 +589,9 @@ void MainWindow::change_options()
 {
     QUiLoader uiload;
     QSettings settings;
+#if QT_VERSION < 0x60000
     settings.setIniCodec("UTF-8");
+#endif
 
     bool initial_load = m_symbology == 0;
     int original_tab_count = tabMain->count();
@@ -598,6 +606,7 @@ void MainWindow::change_options()
         tabMain->removeTab(1);
 
     chkComposite->setText(tr("Add &2D Component"));
+    cmbECI->setItemText(25, tr("29: GB 2312 (PRC)"));
     btype->setItemText(0, tr("No border"));
     combobox_item_enabled(cmbFontSetting, 1, true);
     cmbFontSetting->setItemText(2, tr("Small"));
@@ -841,6 +850,7 @@ void MainWindow::change_options()
         m_optionWidget=uiload.load(&file);
         file.close();
         tabMain->insertTab(1,m_optionWidget,tr("Han Xin Cod&e"));
+        cmbECI->setItemText(25, tr("29: GB 18030 (PRC)"));
         connect(m_optionWidget->findChild<QObject*>("cmbHXSize"), SIGNAL(currentIndexChanged( int )), SLOT(update_preview()));
         connect(m_optionWidget->findChild<QObject*>("cmbHXECC"), SIGNAL(currentIndexChanged( int )), SLOT(update_preview()));
         connect(m_optionWidget->findChild<QObject*>("cmbHXMask"), SIGNAL(currentIndexChanged( int )), SLOT(update_preview()));

frontend_qt/qzint.cpp

@@ -306,7 +306,7 @@ namespace Zint {
         } else if (ECIIndex >= 16 && ECIIndex <= 26) {
             m_eci = ECIIndex + 4;
         } else if (ECIIndex == 27) {
-            m_eci = 899; /* 8-bit binary data TODO: support */
+            m_eci = 899; /* 8-bit binary data */
         } else {
             m_eci = 0;
         }

frontend_qt/sequencewindow.cpp

@@ -1,6 +1,6 @@
 /*
     Zint Barcode Generator - the open source barcode generator
-    Copyright (C) 2009 - 2020 Robin Stuart <rstuart114@gmail.com>
+    Copyright (C) 2009 - 2021 Robin Stuart <rstuart114@gmail.com>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -32,7 +32,9 @@ SequenceWindow::SequenceWindow()
 {
     setupUi(this);
     QSettings settings;
+#if QT_VERSION < 0x60000
     settings.setIniCodec("UTF-8");
+#endif
     QValidator *intvalid = new QIntValidator(this);
 
     linStartVal->setText(settings.value("studio/sequence/start_value", "1").toString());
@@ -54,7 +56,9 @@ SequenceWindow::SequenceWindow()
 SequenceWindow::~SequenceWindow()
 {
     QSettings settings;
+#if QT_VERSION < 0x60000
     settings.setIniCodec("UTF-8");
+#endif
 
     settings.setValue("studio/sequence/start_value", linStartVal->text());
     settings.setValue("studio/sequence/end_value", linEndVal->text());
@@ -170,7 +174,9 @@ void SequenceWindow::check_generate()
 void SequenceWindow::import()
 {
     QSettings settings;
+#if QT_VERSION < 0x60000
     settings.setIniCodec("UTF-8");
+#endif
     QFileDialog import_dialog;
     QString filename;
     QFile file;

win32/libzint.vcxproj

@@ -174,6 +174,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\backend\aztec.h" />
+    <ClInclude Include="..\backend\big5.h" />
     <ClInclude Include="..\backend\bmp.h" />
     <ClInclude Include="..\backend\channel_precalcs.h" />
     <ClInclude Include="..\backend\code1.h" />
@@ -183,6 +184,7 @@
     <ClInclude Include="..\backend\composite.h" />
     <ClInclude Include="..\backend\dmatrix.h" />
     <ClInclude Include="..\backend\eci.h" />
+    <ClInclude Include="..\backend\eci_sb.h" />
     <ClInclude Include="..\backend\emf.h" />
     <ClInclude Include="..\backend\font.h" />
     <ClInclude Include="..\backend\gb18030.h" />
@@ -191,6 +193,7 @@
     <ClInclude Include="..\backend\gridmtx.h" />
     <ClInclude Include="..\backend\gs1.h" />
     <ClInclude Include="..\backend\hanxin.h" />
+    <ClInclude Include="..\backend\ksx1001.h" />
     <ClInclude Include="..\backend\large.h" />
     <ClInclude Include="..\backend\maxicode.h" />
     <ClInclude Include="..\backend\ms_stdint.h" />

win32/vs2008/libzint.vcproj

@@ -473,6 +473,14 @@
 				RelativePath="..\backend\aztec.h"
 				>
 			</File>
+			<File
+				RelativePath="..\backend\big5.h"
+				>
+			</File>
+			<File
+				RelativePath="..\backend\bmp.h"
+				>
+			</File>
 			<File
 				RelativePath="..\backend\channel_precalcs.h"
 				>
@@ -505,6 +513,10 @@
 				RelativePath="..\backend\eci.h"
 				>
 			</File>
+			<File
+				RelativePath="..\backend\eci_sb.h"
+				>
+			</File>
 			<File
 				RelativePath="..\backend\emf.h"
 				>
@@ -537,6 +549,10 @@
 				RelativePath="..\backend\hanxin.h"
 				>
 			</File>
+			<File
+				RelativePath="..\backend\ksx1001.h"
+				>
+			</File>
 			<File
 				RelativePath="..\backend\large.h"
 				>

win32/vs2015/libzint.vcxproj

@@ -353,6 +353,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\backend\aztec.h" />
+    <ClInclude Include="..\..\backend\big5.h" />
     <ClInclude Include="..\..\backend\bmp.h" />
     <ClInclude Include="..\..\backend\channel_precalcs.h" />
     <ClInclude Include="..\..\backend\code1.h" />
@@ -362,6 +363,7 @@
     <ClInclude Include="..\..\backend\composite.h" />
     <ClInclude Include="..\..\backend\dmatrix.h" />
     <ClInclude Include="..\..\backend\eci.h" />
+    <ClInclude Include="..\..\backend\eci_sb.h" />
     <ClInclude Include="..\..\backend\emf.h" />
     <ClInclude Include="..\..\backend\font.h" />
     <ClInclude Include="..\..\backend\gb18030.h" />
@@ -370,6 +372,7 @@
     <ClInclude Include="..\..\backend\gridmtx.h" />
     <ClInclude Include="..\..\backend\gs1.h" />
     <ClInclude Include="..\..\backend\hanxin.h" />
+    <ClInclude Include="..\..\backend\ksx1001.h" />
     <ClInclude Include="..\..\backend\large.h" />
     <ClInclude Include="..\..\backend\maxicode.h" />
     <ClInclude Include="..\..\backend\ms_stdint.h" />

win32/vs2015/vsx/libzintMD.vcxproj

@@ -121,6 +121,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\backend\aztec.h" />
+    <ClInclude Include="..\..\backend\big5.h" />
     <ClInclude Include="..\..\backend\bmp.h" />
     <ClInclude Include="..\..\backend\channel_precalcs.h" />
     <ClInclude Include="..\..\backend\code1.h" />
@@ -130,6 +131,7 @@
     <ClInclude Include="..\..\backend\composite.h" />
     <ClInclude Include="..\..\backend\dmatrix.h" />
     <ClInclude Include="..\..\backend\eci.h" />
+    <ClInclude Include="..\..\backend\eci_sb.h" />
     <ClInclude Include="..\..\backend\emf.h" />
     <ClInclude Include="..\..\backend\font.h" />
     <ClInclude Include="..\..\backend\gb18030.h" />
@@ -138,6 +140,7 @@
     <ClInclude Include="..\..\backend\gridmtx.h" />
     <ClInclude Include="..\..\backend\gs1.h" />
     <ClInclude Include="..\..\backend\hanxin.h" />
+    <ClInclude Include="..\..\backend\ksx1001.h" />
     <ClInclude Include="..\..\backend\large.h" />
     <ClInclude Include="..\..\backend\maxicode.h" />
     <ClInclude Include="..\..\backend\ms_stdint.h" />

win32/vs2019/libzint.vcxproj

@@ -174,6 +174,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\backend\aztec.h" />
+    <ClInclude Include="..\..\backend\big5.h" />
     <ClInclude Include="..\..\backend\bmp.h" />
     <ClInclude Include="..\..\backend\channel_precalcs.h" />
     <ClInclude Include="..\..\backend\code1.h" />
@@ -183,6 +184,7 @@
     <ClInclude Include="..\..\backend\composite.h" />
     <ClInclude Include="..\..\backend\dmatrix.h" />
     <ClInclude Include="..\..\backend\eci.h" />
+    <ClInclude Include="..\..\backend\eci_sb.h" />
     <ClInclude Include="..\..\backend\emf.h" />
     <ClInclude Include="..\..\backend\font.h" />
     <ClInclude Include="..\..\backend\gb18030.h" />
@@ -191,6 +193,7 @@
     <ClInclude Include="..\..\backend\gridmtx.h" />
     <ClInclude Include="..\..\backend\gs1.h" />
     <ClInclude Include="..\..\backend\hanxin.h" />
+    <ClInclude Include="..\..\backend\ksx1001.h" />
     <ClInclude Include="..\..\backend\large.h" />
     <ClInclude Include="..\..\backend\maxicode.h" />
     <ClInclude Include="..\..\backend\ms_stdint.h" />