Bug 101032 - assignments to nsSMState in nsCodingStateMachine result...

... in unspecified behavior.
When compiling with UBSan (-fsanitize=undefined), execution complains:
> runtime error: load of value 5, which is not a valid value for type 'nsSMState'
Since the machine states depend on every different charset's state
machine, it is not possible to simply extend the enum with more generic
values. Instead let's just make the state as an unsigned int value and
define the 3 generic states as constants.

src/nsBig5Prober.cpp

@@ -46,7 +46,7 @@ void  nsBig5Prober::Reset(void)
 
 nsProbingState nsBig5Prober::HandleData(const char* aBuf, PRUint32 aLen)
 {
-  nsSMState codingState;
+  PRUint32 codingState;
 
   for (PRUint32 i = 0; i < aLen; i++)
   {

src/nsCodingStateMachine.h

@@ -39,11 +39,12 @@
 
 #include "nsPkgInt.h"
 
-typedef enum {
-   eStart = 0,
-   eError = 1,
-   eItsMe = 2 
-} nsSMState;
+/* Apart from these 3 generic states, machine states are specific to
+ * each charset prober.
+ */
+#define eStart 0
+#define eError 1
+#define eItsMe 2
 
 #define GETCLASS(c) GETFROMPCK(((unsigned char)(c)), mModel->classTable)
 
@@ -60,7 +61,7 @@ typedef struct
 class nsCodingStateMachine {
 public:
   nsCodingStateMachine(const SMModel* sm) : mModel(sm) { mCurrentState = eStart; }
-  nsSMState NextState(char c){
+  PRUint32 NextState(char c){
     //for each byte we get its class , if it is first byte, we also get byte length
     PRUint32 byteCls = GETCLASS(c);
     if (mCurrentState == eStart)
@@ -69,8 +70,8 @@ public:
       mCurrentCharLen = mModel->charLenTable[byteCls];
     }
     //from byte's class and stateTable, we get its next state
-    mCurrentState=(nsSMState)GETFROMPCK(mCurrentState*(mModel->classFactor)+byteCls,
-                                       mModel->stateTable);
+    mCurrentState = GETFROMPCK(mCurrentState * mModel->classFactor + byteCls,
+                               mModel->stateTable);
     mCurrentBytePos++;
     return mCurrentState;
   }
@@ -79,7 +80,7 @@ public:
   const char * GetCodingStateMachine() {return mModel->name;}
 
 protected:
-  nsSMState mCurrentState;
+  PRUint32 mCurrentState;
   PRUint32 mCurrentCharLen;
   PRUint32 mCurrentBytePos;
 

src/nsEUCJPProber.cpp

@@ -52,7 +52,7 @@ void  nsEUCJPProber::Reset(void)
 
 nsProbingState nsEUCJPProber::HandleData(const char* aBuf, PRUint32 aLen)
 {
-  nsSMState codingState;
+  PRUint32 codingState;
 
   for (PRUint32 i = 0; i < aLen; i++)
   {

src/nsEUCKRProber.cpp

@@ -47,7 +47,7 @@ void  nsEUCKRProber::Reset(void)
 
 nsProbingState nsEUCKRProber::HandleData(const char* aBuf, PRUint32 aLen)
 {
-  nsSMState codingState;
+  PRUint32 codingState;
 
   for (PRUint32 i = 0; i < aLen; i++)
   {

src/nsEUCTWProber.cpp

@@ -47,7 +47,7 @@ void  nsEUCTWProber::Reset(void)
 
 nsProbingState nsEUCTWProber::HandleData(const char* aBuf, PRUint32 aLen)
 {
-  nsSMState codingState;
+  PRUint32 codingState;
 
   for (PRUint32 i = 0; i < aLen; i++)
   {

src/nsEscCharsetProber.cpp

@@ -75,7 +75,7 @@ void nsEscCharSetProber::Reset(void)
 
 nsProbingState nsEscCharSetProber::HandleData(const char* aBuf, PRUint32 aLen)
 {
-  nsSMState codingState;
+  PRUint32 codingState;
   PRInt32 j;
   PRUint32 i;
 

src/nsGB2312Prober.cpp

@@ -52,7 +52,7 @@ void  nsGB18030Prober::Reset(void)
 
 nsProbingState nsGB18030Prober::HandleData(const char* aBuf, PRUint32 aLen)
 {
-  nsSMState codingState;
+  PRUint32 codingState;
 
   for (PRUint32 i = 0; i < aLen; i++)
   {

src/nsSJISProber.cpp

@@ -52,7 +52,7 @@ void  nsSJISProber::Reset(void)
 
 nsProbingState nsSJISProber::HandleData(const char* aBuf, PRUint32 aLen)
 {
-  nsSMState codingState;
+  PRUint32 codingState;
 
   for (PRUint32 i = 0; i < aLen; i++)
   {

src/nsUTF8Prober.cpp

@@ -46,7 +46,7 @@ void  nsUTF8Prober::Reset(void)
 
 nsProbingState nsUTF8Prober::HandleData(const char* aBuf, PRUint32 aLen)
 {
-  nsSMState codingState;
+  PRUint32 codingState;
 
   for (PRUint32 i = 0; i < aLen; i++)
   {