Wrong ClassCastException with varargs of uninstantiable type.

If there is a vararg of an uninstatiable type Foo as in the following
example,

  m(Foo... foos) {
    ...
  }

  fooArr = m();

the compiler might emit the following code for the call

  fooArr = (null[]) /* inlined code for m() */;

Such call will fail at runtime as the compiler determines that Foo[]
can not be cast to null[].

Three options were considered.
1. Make a cast from Foo[] to  null[] succeed if Foo is not
   instantiable.
2. Make the JNewArray AST node by of type null[] if its base type
   was not instantiable.
3. Don't model at all null[], i.e. don't let an array be tightned
   to null[].

The drawbacks of 1 and 2 is loss of information by transforming casts
to different array classes to (null[]).

The drawback of 3 is potential increase in codesize which does not
seem to occurr in practice.

Bug: issue 8736.
Change-Id: I3348836b78ad59e11685a3a55650df81afa75269
diff --git a/dev/core/src/com/google/gwt/dev/jjs/ast/JProgram.java b/dev/core/src/com/google/gwt/dev/jjs/ast/JProgram.java
index af202f4..ab8c418 100644
--- a/dev/core/src/com/google/gwt/dev/jjs/ast/JProgram.java
+++ b/dev/core/src/com/google/gwt/dev/jjs/ast/JProgram.java
@@ -575,7 +575,13 @@
            */
           JReferenceType leafRefType1 = (JReferenceType) leafType1;
           JReferenceType leafRefType2 = (JReferenceType) leafType2;
-          JReferenceType leafGeneralization = generalizeTypes(leafRefType1, leafRefType2);
+
+          /**
+           * Never generalize arrays to arrays of {@link JNonNullType} as null array initialization
+           * is not accounted for in {@link TypeTightener}.
+           */
+          JReferenceType leafGeneralization =
+              generalizeTypes(leafRefType1, leafRefType2).getUnderlyingType();
           return getTypeArray(leafGeneralization, dims1);
 
         } else {
diff --git a/dev/core/src/com/google/gwt/dev/jjs/impl/ComputeCastabilityInformation.java b/dev/core/src/com/google/gwt/dev/jjs/impl/ComputeCastabilityInformation.java
index f77281a..1813d2d 100644
--- a/dev/core/src/com/google/gwt/dev/jjs/impl/ComputeCastabilityInformation.java
+++ b/dev/core/src/com/google/gwt/dev/jjs/impl/ComputeCastabilityInformation.java
@@ -167,20 +167,22 @@
       type = type.getUnderlyingType();
       qType = qType.getUnderlyingType();
 
+      if (program.typeOracle.canTriviallyCast(type, qType)) {
+        return true;
+      }
+
       if (type instanceof JArrayType && qType instanceof JArrayType) {
         JArrayType aType = (JArrayType) type;
         JArrayType aqType = (JArrayType) qType;
-        return program.typeOracle.canTriviallyCast(type, qType)
-            || (program.isJavaScriptObject(aType.getLeafType()) && program
+        return (program.isJavaScriptObject(aType.getLeafType()) && program
                 .isJavaScriptObject(aqType.getLeafType()));
       }
 
-      return program.typeOracle.canTriviallyCast(type, qType)
-          || (program.isJavaScriptObject(type) && program.isJavaScriptObject(qType));
+      return (program.isJavaScriptObject(type) && program.isJavaScriptObject(qType));
     }
 
     /**
-     * Create the mapping from a class to its query types.
+     * Create the mapping from a class to the types it can be cast to.
      */
     private void computeCastMap(JReferenceType type) {
       if (type == null || alreadyRan.contains(type)) {
diff --git a/dev/core/src/com/google/gwt/dev/jjs/impl/TypeTightener.java b/dev/core/src/com/google/gwt/dev/jjs/impl/TypeTightener.java
index bbaa2db..836dc17 100644
--- a/dev/core/src/com/google/gwt/dev/jjs/impl/TypeTightener.java
+++ b/dev/core/src/com/google/gwt/dev/jjs/impl/TypeTightener.java
@@ -19,7 +19,6 @@
 import com.google.gwt.dev.jjs.ast.CanBeAbstract;
 import com.google.gwt.dev.jjs.ast.Context;
 import com.google.gwt.dev.jjs.ast.JArrayRef;
-import com.google.gwt.dev.jjs.ast.JArrayType;
 import com.google.gwt.dev.jjs.ast.JBinaryOperation;
 import com.google.gwt.dev.jjs.ast.JBinaryOperator;
 import com.google.gwt.dev.jjs.ast.JCastOperation;
@@ -687,20 +686,6 @@
       return null;
     }
 
-    private JArrayType nullifyArrayType(JArrayType arrayType) {
-      JType elementType = arrayType.getElementType();
-      if (elementType instanceof JReferenceType) {
-        JReferenceType refType = (JReferenceType) elementType;
-        if (!program.typeOracle.isInstantiatedType(refType)) {
-          return program.getTypeArray(JNullType.INSTANCE);
-        } else if (elementType instanceof JArrayType) {
-          JArrayType newElementType = nullifyArrayType((JArrayType) elementType);
-          return program.getTypeArray(newElementType.getLeafType(), newElementType.getDims() + 1);
-        }
-      }
-      return arrayType;
-    }
-
     /**
      * Tighten based on assignment, and for parameters, callArgs as well.
      */
@@ -721,11 +706,6 @@
         return;
       }
 
-      if (refType instanceof JArrayType) {
-        JArrayType arrayType = (JArrayType) refType;
-        refType = nullifyArrayType(arrayType);
-      }
-
       // tighten based on leaf types
       JReferenceType leafType = getSingleConcreteType(refType);
       if (leafType != null) {
diff --git a/user/test/com/google/gwt/dev/jjs/test/VarargsTest.java b/user/test/com/google/gwt/dev/jjs/test/VarargsTest.java
index f780cae..23e0a03 100644
--- a/user/test/com/google/gwt/dev/jjs/test/VarargsTest.java
+++ b/user/test/com/google/gwt/dev/jjs/test/VarargsTest.java
@@ -24,6 +24,8 @@
  */
 public class VarargsTest extends GWTTestCase {
 
+  private static class Foo { }
+
   @Override
   public String getModuleName() {
     return "com.google.gwt.dev.jjs.CompilerSuite";
@@ -59,10 +61,29 @@
     assertTrue(Arrays.equals(expected, actual));
   }
 
+  /**
+   * Test for issue 8736.
+   */
+  public void testNullEmptyUninstantiable_Varargs() {
+    assertNotNull(fooVararg());
+  }
+
+  public void testNullEmptyUninstantiable_NoVarargs() {
+    assertNotNull(fooIdent(new Foo[]{}));
+  }
+
   private String[] vararg(String... args) {
     return args;
   }
 
+  private Foo[] fooVararg(Foo... args) {
+    return args;
+  }
+
+  private Foo[] fooIdent(Foo[] args) {
+    return args;
+  }
+
   private int[] varargUnboxed(int... args) {
     return args;
   }