Workaround for 64KB string limit for RPC responses in GWT Hosted Mode client due to Rhino library limitation. Strings literals will be split into 64KB chunks and concatenated with the '+' operator. This is not strict JSON format, but is valid Javascript and can be evaluated with negligable performance impact. Review by: jat@google.com git-svn-id: https://google-web-toolkit.googlecode.com/svn/trunk@10888 8db76d5a-ed1c-0410-87a9-c151d255dfc7
diff --git a/user/src/com/google/gwt/user/client/rpc/impl/ClientSerializationStreamReader.java b/user/src/com/google/gwt/user/client/rpc/impl/ClientSerializationStreamReader.java index ca4c183..1114c87 100644 --- a/user/src/com/google/gwt/user/client/rpc/impl/ClientSerializationStreamReader.java +++ b/user/src/com/google/gwt/user/client/rpc/impl/ClientSerializationStreamReader.java
@@ -18,6 +18,8 @@ import com.google.gwt.dev.jjs.SourceOrigin; import com.google.gwt.dev.js.JsParser; import com.google.gwt.dev.js.ast.JsArrayLiteral; +import com.google.gwt.dev.js.ast.JsBinaryOperation; +import com.google.gwt.dev.js.ast.JsBinaryOperator; import com.google.gwt.dev.js.ast.JsBooleanLiteral; import com.google.gwt.dev.js.ast.JsContext; import com.google.gwt.dev.js.ast.JsExpression; @@ -194,7 +196,7 @@ * This visitor reverses that transform by reducing all concat invocations into a single array * literal. */ - private static class ConcatEvaler extends JsModVisitor { + private static class ArrayConcatEvaler extends JsModVisitor { @Override public boolean visit(JsInvocation invoke, JsContext ctx) { @@ -202,7 +204,7 @@ if (!(expr instanceof JsNameRef)) { return super.visit(invoke, ctx); } - + JsNameRef name = (JsNameRef) expr; if (!name.getIdent().equals("concat")) { return super.visit(invoke, ctx); @@ -214,12 +216,93 @@ JsArrayLiteral arg = (JsArrayLiteral) ex; headElements.getExpressions().addAll(arg.getExpressions()); } - + ctx.replaceMe(headElements); return true; } } - + + /** + * The server splits up string literals into 64KB chunks using '+' operators. For example + * ['chunk1chunk2'] is broken up into ['chunk1' + 'chunk2']. + * <p> + * This visitor reverses that transform by reducing such strings into a single string literal. + */ + private static class StringConcatEvaler extends JsModVisitor { + + @Override public boolean visit(JsBinaryOperation x, JsContext ctx) { + if (x.getOperator() != JsBinaryOperator.ADD) { + return super.visit(x, ctx); + } + + // Do a first pass to get the total string length to avoid dynamically resizing the buffer. + int stringLength = getStringLength(x); + if (stringLength >= 0) { + StringBuilder builder = new StringBuilder(stringLength); + if (expressionToString(x, builder)) { + ctx.replaceMe(new JsStringLiteral(x.getSourceInfo(), builder.toString())); + } + } + + return true; + } + + /** + * Transforms an expression into a string. This will recurse into JsBinaryOperations of type + * JsBinaryOperator.ADD, which may have other ADD operations or JsStringLiterals as arguments. + * + * @param expression the expression to evaluate + * @param builder a builder that the string will be appended to + * @return true if the expression represents a valid string, or false otherwise + */ + private boolean expressionToString(JsExpression expression, StringBuilder builder) { + if (expression instanceof JsStringLiteral) { + builder.append(((JsStringLiteral) expression).getValue()); + return true; + } + + if (expression instanceof JsBinaryOperation) { + JsBinaryOperation operation = (JsBinaryOperation) expression; + if (operation.getOperator() != JsBinaryOperator.ADD) { + return false; + } + return expressionToString(operation.getArg1(), builder) + && expressionToString(operation.getArg2(), builder); + } + + return false; + } + + /** + * Gets the total string length of the given expression. This will recurse into + * JsBinaryOperations of type JsBinaryOperator.ADD, which may have other ADD operations or + * JsStringLiterals as arguments. + * + * @param expression the expression to evaluate + * @return the total string length, or -1 if the given expression does not represent a valid + * string + */ + private int getStringLength(JsExpression expression) { + if (expression instanceof JsStringLiteral) { + return ((JsStringLiteral) expression).getValue().length(); + } + + if (expression instanceof JsBinaryOperation) { + JsBinaryOperation operation = (JsBinaryOperation) expression; + if (operation.getOperator() != JsBinaryOperator.ADD) { + return -1; + } + + int arg1Length = getStringLength(operation.getArg1()); + int arg2Length = getStringLength(operation.getArg2()); + + return (arg1Length >= 0 && arg2Length >= 0) ? (arg1Length + arg2Length) : -1; + } + + return -1; + } + } + public ClientSerializationStreamReader(Serializer serializer) { this.serializer = serializer; } @@ -229,8 +312,10 @@ try { List<JsStatement> stmts = JsParser.parse(SourceOrigin.UNKNOWN, JsRootScope.INSTANCE, new StringReader(encoded)); - ConcatEvaler concatEvaler = new ConcatEvaler(); - concatEvaler.acceptList(stmts); + ArrayConcatEvaler arrayConcatEvaler = new ArrayConcatEvaler(); + arrayConcatEvaler.acceptList(stmts); + StringConcatEvaler stringConcatEvaler = new StringConcatEvaler(); + stringConcatEvaler.acceptList(stmts); decoder = new RpcDecoder(); decoder.acceptList(stmts); } catch (Exception e) {
diff --git a/user/src/com/google/gwt/user/server/rpc/impl/ServerSerializationStreamWriter.java b/user/src/com/google/gwt/user/server/rpc/impl/ServerSerializationStreamWriter.java index 1bdf41e..9ea59c1 100644 --- a/user/src/com/google/gwt/user/server/rpc/impl/ServerSerializationStreamWriter.java +++ b/user/src/com/google/gwt/user/server/rpc/impl/ServerSerializationStreamWriter.java
@@ -323,6 +323,14 @@ private static final char NON_BREAKING_HYPHEN = '\u2011'; + /** + * Maximum length of a string node in RPC responses, not including surrounding + * quote characters (2 ^ 16 - 1) = 65535. + * This exists to work around a Rhino parser bug in the hosted mode client + * that limits string node lengths to 64KB. + */ + private static final int MAX_STRING_NODE_LENGTH = 0xFFFF; + static { /* * NOTE: The JS VM in IE6 & IE7 do not interpret \v correctly. They convert @@ -370,18 +378,59 @@ * than 1.3 that supports unicode strings. */ public static String escapeString(String toEscape) { - // make output big enough to escape every character (plus the quotes) + return escapeString(toEscape, false); + } + + /** + * This method takes a string and outputs a JavaScript string literal. The + * data is surrounded with quotes, and any contained characters that need to + * be escaped are mapped onto their escape sequence. + * + * This splits strings into 64KB chunks to workaround an issue with the hosted mode client where + * the Rhino parser can't handle string nodes larger than 64KB, e.g. {@code "longstring"} is + * converted to {@code "long" + "string"}. + * + * Assumptions: We are targeting a version of JavaScript that that is later + * than 1.3 that supports unicode strings. + */ + public static String escapeStringSplitNodes(String toEscape) { + return escapeString(toEscape, true); + } + + private static String escapeString(String toEscape, boolean splitNodes) { char[] input = toEscape.toCharArray(); - CharVector charVector = new CharVector(input.length * 2 + 2, input.length); + + // Since escaped characters will increase the output size, allocate extra room to start. + int capacityIncrement = Math.max(input.length, 16); + CharVector charVector = new CharVector(capacityIncrement * 2, capacityIncrement); + + int i = 0; + int length = input.length; charVector.add(JS_QUOTE_CHAR); - for (int i = 0, n = input.length; i < n; ++i) { - char c = input[i]; - if (needsUnicodeEscape(c)) { - unicodeEscape(c, charVector); - } else { - charVector.add(c); + while (i < length) { + + // Add one segment at a time, up to maxNodeLength characters. Note this always leave room + // for at least 6 characters at the end (maximum unicode escaped character size). + int maxSegmentVectorSize = splitNodes + ? (charVector.getSize() + MAX_STRING_NODE_LENGTH - 5) + : Integer.MAX_VALUE; + + while (i < length && charVector.getSize() < maxSegmentVectorSize) { + char c = input[i++]; + if (needsUnicodeEscape(c)) { + unicodeEscape(c, charVector); + } else { + charVector.add(c); + } + } + + // If there's another segment left, insert a '+' operator. + if (splitNodes && i < length) { + charVector.add(JS_QUOTE_CHAR); + charVector.add('+'); + charVector.add(JS_QUOTE_CHAR); } } @@ -798,7 +847,7 @@ private void writeStringTable(LengthConstrainedArray stream) { LengthConstrainedArray tableStream = new LengthConstrainedArray(); for (String s : getStringTable()) { - tableStream.addToken(escapeString(s)); + tableStream.addToken(escapeStringSplitNodes(s)); } stream.addToken(tableStream.toString()); }
diff --git a/user/test/com/google/gwt/user/RPCSuite.java b/user/test/com/google/gwt/user/RPCSuite.java index dc3f5e6..c2459a2 100644 --- a/user/test/com/google/gwt/user/RPCSuite.java +++ b/user/test/com/google/gwt/user/RPCSuite.java
@@ -49,6 +49,7 @@ import com.google.gwt.user.client.rpc.ValueTypesTest; import com.google.gwt.user.client.rpc.ValueTypesTestWithTypeObfuscation; import com.google.gwt.user.client.rpc.XsrfProtectionTest; +import com.google.gwt.user.client.rpc.impl.ClientSerializationStreamReaderTest; import com.google.gwt.user.rebind.rpc.BlacklistTypeFilterTest; import com.google.gwt.user.rebind.rpc.SerializableTypeOracleBuilderTest; import com.google.gwt.user.rebind.rpc.TypeHierarchyUtilsTest; @@ -60,6 +61,7 @@ import com.google.gwt.user.server.rpc.RPCTest; import com.google.gwt.user.server.rpc.SerializationPolicyLoaderTest; import com.google.gwt.user.server.rpc.impl.LegacySerializationPolicyTest; +import com.google.gwt.user.server.rpc.impl.ServerSerializationStreamWriterTest; import com.google.gwt.user.server.rpc.impl.StandardSerializationPolicyTest; import junit.framework.Test; @@ -98,6 +100,8 @@ suite.addTestSuite(Base64Test.class); suite.addTestSuite(UtilTest.class); suite.addTestSuite(AbstractXsrfProtectedServiceServletTest.class); + suite.addTestSuite(ClientSerializationStreamReaderTest.class); + suite.addTestSuite(ServerSerializationStreamWriterTest.class); // GWTTestCases suite.addTestSuite(ValueTypesTest.class);
diff --git a/user/test/com/google/gwt/user/client/rpc/ValueTypesTest.java b/user/test/com/google/gwt/user/client/rpc/ValueTypesTest.java index d602443..ecafa12 100644 --- a/user/test/com/google/gwt/user/client/rpc/ValueTypesTest.java +++ b/user/test/com/google/gwt/user/client/rpc/ValueTypesTest.java
@@ -594,7 +594,36 @@ } }); } - + + public void testString() { + assertEcho("test"); + } + + public void testString_empty() { + assertEcho(""); + } + + public void testString_over64KB() { + // Test a string over 64KB of a-z characters repeated. + String testString = ""; + int totalChars = 0xFFFF + 0xFF; + for (int i = 0; i < totalChars; i++) { + testString += (char) ('a' + (i % 26)); + } + assertEcho(testString); + } + + public void testString_over64KBWithUnicode() { + // Test a string over64KB string that requires unicode escaping. + String testString = ""; + int totalChars = 0xFFFF + 0xFF; + for (int i = 0; i < totalChars; i += 2) { + testString += '\u2011'; + testString += (char) 0x08; + } + assertEcho(testString); + } + private void assertEcho(final BigDecimal value) { ValueTypesTestServiceAsync service = getServiceAsync(); delayTestFinishForRpc(); @@ -625,6 +654,21 @@ }); } + private void assertEcho(final String value) { + ValueTypesTestServiceAsync service = getServiceAsync(); + delayTestFinishForRpc(); + service.echo(value, new AsyncCallback<String>() { + public void onFailure(Throwable caught) { + TestSetValidator.rethrowException(caught); + } + + public void onSuccess(String result) { + assertEquals(value, result); + finishTest(); + } + }); + } + private ValueTypesTestServiceAsync getServiceAsync() { if (primitiveTypeTestService == null) { primitiveTypeTestService = (ValueTypesTestServiceAsync) GWT.create(ValueTypesTestService.class);
diff --git a/user/test/com/google/gwt/user/client/rpc/ValueTypesTestService.java b/user/test/com/google/gwt/user/client/rpc/ValueTypesTestService.java index cecbf1b..faf3d63 100644 --- a/user/test/com/google/gwt/user/client/rpc/ValueTypesTestService.java +++ b/user/test/com/google/gwt/user/client/rpc/ValueTypesTestService.java
@@ -43,6 +43,8 @@ SerializableGenericWrapperType<Void> echo( SerializableGenericWrapperType<Void> value); + String echo(String value); + boolean echo_FALSE(boolean value); byte echo_MAX_VALUE(byte value);
diff --git a/user/test/com/google/gwt/user/client/rpc/ValueTypesTestServiceAsync.java b/user/test/com/google/gwt/user/client/rpc/ValueTypesTestServiceAsync.java index 1a79fda..bf414bf 100644 --- a/user/test/com/google/gwt/user/client/rpc/ValueTypesTestServiceAsync.java +++ b/user/test/com/google/gwt/user/client/rpc/ValueTypesTestServiceAsync.java
@@ -44,6 +44,8 @@ void echo(SerializableGenericWrapperType<Void> value, AsyncCallback<SerializableGenericWrapperType<Void>> callback); + void echo(String value, AsyncCallback<String> callback); + void echo_FALSE(boolean value, AsyncCallback<Boolean> callback); void echo_MAX_VALUE(byte value, AsyncCallback<Byte> callback);
diff --git a/user/test/com/google/gwt/user/client/rpc/impl/ClientSerializationStreamReaderTest.java b/user/test/com/google/gwt/user/client/rpc/impl/ClientSerializationStreamReaderTest.java new file mode 100644 index 0000000..e757339 --- /dev/null +++ b/user/test/com/google/gwt/user/client/rpc/impl/ClientSerializationStreamReaderTest.java
@@ -0,0 +1,132 @@ +/* + * Copyright 2012 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.gwt.user.client.rpc.impl; + +import com.google.gwt.user.client.rpc.SerializationException; + +import junit.framework.TestCase; + +/** + * Tests {@link ClientSerializationStreamReader}. + */ +public class ClientSerializationStreamReaderTest extends TestCase { + + public void testRead() throws SerializationException { + ClientSerializationStreamReader reader = new ClientSerializationStreamReader(null); + + reader.prepareToRead("[" + + "3.5," // a double + + "1," // String table index: "one" + + "3," // String table index: "three" + + "2," // String table index: "two" + + "[\"one\",\"two\",\"three\"]," + + "0," // flags + + AbstractSerializationStream.SERIALIZATION_STREAM_VERSION // version + + "]"); + + assertEquals("two", reader.readString()); + assertEquals("three", reader.readString()); + assertEquals("one", reader.readString()); + assertEquals(3.5, reader.readDouble()); + } + + public void testRead_stringConcats() throws SerializationException { + ClientSerializationStreamReader reader = new ClientSerializationStreamReader(null); + + reader.prepareToRead("[" + + "1," // String table index: "onetwothree" + + "[\"one\"+\"two\"+\"three\"]," + + "0," // flags + + AbstractSerializationStream.SERIALIZATION_STREAM_VERSION // version + + "]"); + + assertEquals("onetwothree", reader.readString()); + } + + public void testRead_arrayConcats() throws SerializationException { + ClientSerializationStreamReader reader = new ClientSerializationStreamReader(null); + + reader.prepareToRead("[" + + "1," // String table index: "one" + + "2" // String table index: "two" + + "].concat([" + + "[\"one\"].concat([\"two\"])," + + "0," // flags + + AbstractSerializationStream.SERIALIZATION_STREAM_VERSION // version + + "])"); + + assertEquals("two", reader.readString()); + assertEquals("one", reader.readString()); + } + + /* + * Note: this test verifies a issue with the Rhino parser that limits the size of a single string + * node to 64KB. If this test starts failing, then the Rhino parser may have been fixed to support + * larger strings and the string concat workaround could be removed. + */ + public void testRead_stringOver64KB() { + ClientSerializationStreamReader reader = new ClientSerializationStreamReader(null); + + int stringLength = 0xFFFF; + StringBuilder builder = new StringBuilder(stringLength); + for (int i = 0; i < stringLength; i++) { + builder.append('y'); + } + + // Push the string size over 64KB. + builder.append('z'); + + try { + reader.prepareToRead("[" + + "1," // String table index + + "[\"" + builder.toString() + "\"]," + + "0," // flags + + AbstractSerializationStream.SERIALIZATION_STREAM_VERSION // version + + "]"); + fail("Expected SerializationException"); + } catch (SerializationException e) { + // Expected. + } + } + + public void testRead_stringOver64KB_concat() throws SerializationException { + ClientSerializationStreamReader reader = new ClientSerializationStreamReader(null); + + // First node is maximum allowed 64KB. + int node1Length = 0xFFFF; + StringBuilder node1Builder = new StringBuilder(node1Length); + for (int i = 0; i < node1Length; i++) { + node1Builder.append('y'); + } + + int node2Length = 0xFF; + StringBuilder node2Builder = new StringBuilder(0xFF); + for (int i = 0; i < node2Length; i++) { + node2Builder.append('z'); + } + + reader.prepareToRead("[" + + "1," // String table index + + "[\"" + node1Builder.toString() + "\"+\"" + node2Builder.toString() + "\"]," + + "0," // flags + + AbstractSerializationStream.SERIALIZATION_STREAM_VERSION // version + + "]"); + + assertEquals(node1Builder.toString() + node2Builder.toString(), reader.readString()); + } +} +
diff --git a/user/test/com/google/gwt/user/server/rpc/ValueTypesTestServiceImpl.java b/user/test/com/google/gwt/user/server/rpc/ValueTypesTestServiceImpl.java index 16604ac..00b8e8f 100644 --- a/user/test/com/google/gwt/user/server/rpc/ValueTypesTestServiceImpl.java +++ b/user/test/com/google/gwt/user/server/rpc/ValueTypesTestServiceImpl.java
@@ -71,6 +71,10 @@ return value; } + public String echo(String value) { + return value; + } + public boolean echo_FALSE(boolean value) { if (value != false) { throw new RuntimeException();
diff --git a/user/test/com/google/gwt/user/server/rpc/impl/ServerSerializationStreamWriterTest.java b/user/test/com/google/gwt/user/server/rpc/impl/ServerSerializationStreamWriterTest.java new file mode 100644 index 0000000..33d4156 --- /dev/null +++ b/user/test/com/google/gwt/user/server/rpc/impl/ServerSerializationStreamWriterTest.java
@@ -0,0 +1,117 @@ +/* + * Copyright 2012 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.gwt.user.server.rpc.impl; + +import junit.framework.TestCase; + +/** + * Tests {@link ServerSerializationStreamWriter}. + */ +public class ServerSerializationStreamWriterTest extends TestCase { + + public void testEscapeString() { + // Ensure that when using escapeString, a large string is not split into + // separate nodes like escapeStringSplitNodes does. + int nodeLength = 0xFFFF * 2; + StringBuilder firstNodeBuilder = new StringBuilder(nodeLength); + for (int i = 0; i < nodeLength; i++) { + firstNodeBuilder.append('1'); + } + + String escaped = ServerSerializationStreamWriter.escapeString( + firstNodeBuilder.toString()); + + assertEquals("\"" + firstNodeBuilder.toString() + "\"", escaped); + } + + public void testEscapeStringSplitNodes() { + String escaped = ServerSerializationStreamWriter.escapeStringSplitNodes("test"); + assertEquals("\"test\"", escaped); + } + + public void testEscapeStringSplitNodes_unicodeEscape() { + String escaped = ServerSerializationStreamWriter.escapeStringSplitNodes( + "测试" // Unicode characters + + "\"" // JS quote char + + "\\" // JS escape char + + '\u2011' // Unicode non-breaking hyphen char + + (char) 0x8); // Combining spacing mark + assertEquals( + "\"" + + "测试" // Unicode characters + + "\\\"" // JS quote char + + "\\\\" // JS escape char + + "\\u2011" // Unicode non-breaking hyphen char + + "\\b" // Combining spacing mark + + "\"", + escaped); + } + + public void testEscapeStringSplitNodes_over64KB() { + // String node length limit is 64KB. + int firstNodeLength = 0xFFFF; + StringBuilder firstNodeBuilder = new StringBuilder(firstNodeLength); + for (int i = 0; i < firstNodeLength - 5; i++) { + firstNodeBuilder.append('1'); + } + + int secondNodeLength = 0xFF; + StringBuilder secondNodeBuilder = new StringBuilder(secondNodeLength); + for (int i = 0; i < secondNodeLength; i++) { + secondNodeBuilder.append('2'); + } + + String escaped = ServerSerializationStreamWriter.escapeStringSplitNodes( + firstNodeBuilder.toString() + secondNodeBuilder.toString()); + + assertEquals( + "\"" + firstNodeBuilder.toString() + "\"+\"" + secondNodeBuilder.toString() + "\"", + escaped); + } + + public void testEscapeStringSplitNodes_over64KBEscaped() { + // Fill the entire 64KB string, but leave 6 characters for an escaped unicode character added + // below. + int firstNodeLength = 0xFFFF; + StringBuilder firstNodeBuilder = new StringBuilder(firstNodeLength); + for (int i = 0; i < firstNodeLength - 6; i++) { + firstNodeBuilder.append('y'); + } + String firstNodeNoUnicode = firstNodeBuilder.toString(); + + // Add a unicode character on the boundary, this should be the last character added to the first + // node. + firstNodeBuilder.append('\u2011'); + + int secondNodeLength = 0xFF; + StringBuilder secondNodeBuilder = new StringBuilder(secondNodeLength); + for (int i = 0; i < secondNodeLength; i++) { + secondNodeBuilder.append('z'); + } + String secondNode = secondNodeBuilder.toString(); + + String escaped = ServerSerializationStreamWriter.escapeStringSplitNodes( + firstNodeBuilder.toString() + secondNode); + + assertEquals( + "\"" + firstNodeNoUnicode + "\\u2011" // first node (including escaped unicode character) + + "\"+\"" + + secondNode + "\"", // second node + escaped); + } + +}