Support for Unicode UTF-8 Identifiers

pxr/base/tf/CMakeLists.txt

@@ -81,6 +81,7 @@ pxr_library(tf
         type
         typeFunctions
         typeNotice
+        unicodeUtils
         warning
         weakBase
         weakPtr
@@ -172,6 +173,7 @@ pxr_library(tf
     CPPFILES
         initConfig.cpp
         preprocessorUtils.cpp
+        unicodeCharacterClasses.cpp
         pxrDoubleConversion/double-conversion.cc
         pxrDoubleConversion/bignum-dtoa.cc
         pxrDoubleConversion/bignum.cc

pxr/base/tf/stringUtils.cpp

@@ -63,6 +63,37 @@ using std::vector;
 
 PXR_NAMESPACE_OPEN_SCOPE
 
+// environment setting for defaulting back to standard ASCII rules for 
+// identifiers and prim names.  By default this value is true indicating 
+// that Unicode processing paths will be used on UTF-8 character sequences 
+// to determine identifier validity.  In addition, if this value is true, 
+// slightly different rules will be used to evaluate generic identifiers 
+// vs. prim names.  Setting this value to false will fallback to original
+// ASCII identifier rules.
+TF_DEFINE_ENV_SETTING(TF_UTF8_IDENTIFIERS, 
+    true, 
+    "Allow UTF8 strings as identifiers and prim names");
+
+namespace {
+    bool _IsASCIIValue(const char& c)
+    {
+        return (c & (1<<7)) == 0;
+    }
+
+    bool _IsInASCIIValueRange(const std::string& str)
+    {
+        for (const char &c : str)
+        {
+            if ((c & (1<<7)) != 0)
+            {
+                return false;
+            }
+        }
+
+        return true;
+    }
+}
+
 string
 TfVStringPrintf(const std::string& fmt, va_list ap)
 {
@@ -269,10 +300,6 @@ TfStringToUpper(const string &source)
 string
 TfStringCapitalize(const string& source)
 {
-    if (source.empty()) {
-        return source;
-    }
-
     string result(source);
     result[0] = toupper(result[0]);
 
@@ -294,6 +321,8 @@ TfStringGetCommonPrefix(string a, string b)
 string
 TfStringGetSuffix(const string& name, char delimiter)
 {
+    TF_DEV_AXIOM(_IsASCIIValue(delimiter));
+
     size_t i = name.rfind(delimiter);
     if (i == string::npos)
         return "";
@@ -304,6 +333,8 @@ TfStringGetSuffix(const string& name, char delimiter)
 string
 TfStringGetBeforeSuffix(const string& name, char delimiter)
 {
+    TF_DEV_AXIOM(_IsASCIIValue(delimiter));
+
     size_t i = name.rfind(delimiter);
     if (i == string::npos)
         return name;
@@ -484,6 +515,8 @@ TfStringSplit(string const &src, string const &separator)
 vector<string>
 TfStringTokenize(string const &src, const char* delimiters)
 {
+    TF_DEV_AXIOM(_IsInASCIIValueRange(delimiters));
+
     vector<pair<char const *, char const *> > segments;
     _TokenizeToSegments(src, delimiters, segments);
 
@@ -497,6 +530,8 @@ TfStringTokenize(string const &src, const char* delimiters)
 set<string>
 TfStringTokenizeToSet(string const &src, const char* delimiters)
 {
+    TF_DEV_AXIOM(_IsInASCIIValueRange(delimiters));
+
     vector<pair<char const *, char const *> > segments;
     _TokenizeToSegments(src, delimiters, segments);
 
@@ -523,7 +558,9 @@ _FindFirstOfNotEscaped(const string &source, const char *toFind, size_t offset)
 vector<string>
 TfQuotedStringTokenize(const string &source, const char *delimiters, 
                        string *errors)
-{    
+{   
+    TF_DEV_AXIOM(_IsInASCIIValueRange(delimiters));
+
     vector<string> resultVec;
     size_t j, quoteIndex, delimIndex;
     const char *quotes = "\"\'`";
@@ -801,7 +838,8 @@ TfDictionaryLessThan::_LessImpl(const string& lstr, const string& rstr) const
         }
         l = *lcur, r = *rcur;
         // If they are letters that differ disregarding case, we're done.
-        if (((l & ~0x20) != (r & ~0x20)) & bool(l & r & ~0x3f)) {
+        // but only if they are ASCII (i.e., the high bit is not set)
+        if (_IsASCIIValue(l) && _IsASCIIValue(r) && (((l & ~0x20) != (r & ~0x20)) & bool(l & r & ~0x3f))) {
             // Add 5 mod 32 makes '_' sort before all letters.
             return ((l + 5) & 31) < ((r + 5) & 31);
         }
@@ -883,7 +921,12 @@ TfDictionaryLessThan::_LessImpl(const string& lstr, const string& rstr) const
         }
         else if (!IsAlpha(l) || !IsAlpha(r)) {
             // At least one isn't a letter.
-            return l < r;
+            // this either means it's an ASCII symbol (digit was checked above)
+            // or a utf-8 encoded unicode character
+            // we want ASCII symbols to sort first, so we can't treat the
+            // utf-8 characters as unsigned (and all ASCII values remain the same
+            // after the cast)
+            return static_cast<unsigned char>(l) < static_cast<unsigned char>(r);
         }
         else {
             // Both letters, differ by case, continue.
@@ -1150,37 +1193,52 @@ TfStringCatPaths( const string &prefix, const string &suffix )
     return TfNormPath(prefix + "/" + suffix);
 }
 
+bool UseUTF8Identifiers()
+{
+    static bool useUtf8Identifiers = 
+        (TfGetEnvSetting(TF_UTF8_IDENTIFIERS) == true);
+
+    return useUtf8Identifiers;
+}
+
 std::string
 TfMakeValidIdentifier(const std::string &in)
 {
-    std::string result;
-
-    if (in.empty()) {
-        result.push_back('_');
-        return result;
+    if (UseUTF8Identifiers())
+    {
+        return TfUnicodeUtils::MakeValidUTF8Identifier(in);
     }
+    else
+    {
+        std::string result;
 
-    result.reserve(in.size());
-    char const *p = in.c_str();
-    if (!(('a' <= *p && *p <= 'z') || 
-          ('A' <= *p && *p <= 'Z') || 
-          *p == '_')) {
-        result.push_back('_');
-    } else {
-        result.push_back(*p);
-    }
+        if (in.empty()) {
+            result.push_back('_');
+            return result;
+        }
 
-    for (++p; *p; ++p) {
-        if (!(('a' <= *p && *p <= 'z') ||    
-              ('A' <= *p && *p <= 'Z') ||  
-              ('0' <= *p && *p <= '9') ||  
-              *p == '_')) {
+        result.reserve(in.size());
+        char const *p = in.c_str();
+        if (!(('a' <= *p && *p <= 'z') || 
+            ('A' <= *p && *p <= 'Z') || 
+            *p == '_')) {
             result.push_back('_');
         } else {
             result.push_back(*p);
         }
+
+        for (++p; *p; ++p) {
+            if (!(('a' <= *p && *p <= 'z') ||    
+                ('A' <= *p && *p <= 'Z') ||  
+                ('0' <= *p && *p <= '9') ||  
+                *p == '_')) {
+                result.push_back('_');
+            } else {
+                result.push_back(*p);
+            }
+        }
+        return result;
     }
-    return result;
 }
 
 std::string