Skip to content

Commit 622c498

Browse files
slandelleslandelle
committed
Optimize UTF-8 HeapByteBuf(s) to String, close AsyncHttpClient#1245 
1 parent cbb2cc9 commit 622c498

File tree

7 files changed

+261
-201
lines changed

7 files changed

+261
-201
lines changed

‎client/src/main/java/org/asynchttpclient/netty/ws/NettyWebSocket.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
package org.asynchttpclient.netty.ws;
1515

1616
import static io.netty.buffer.Unpooled.wrappedBuffer;
17+
import static java.nio.charset.StandardCharsets.UTF_8;
1718
import static org.asynchttpclient.netty.util.ByteBufUtils.byteBuf2Bytes;
1819
import io.netty.channel.Channel;
1920
import io.netty.handler.codec.http.HttpHeaders;
@@ -220,7 +221,7 @@ public void onBinaryFrame(BinaryWebSocketFrame frame) {
220221
public void onTextFrame(TextWebSocketFrame frame) {
221222
if (interestedInTextMessages) {
222223
try {
223-
notifyTextListeners(ByteBufUtils.byteBuf2Utf8String(frame.content()));
224+
notifyTextListeners(ByteBufUtils.byteBuf2String(UTF_8, frame.content()));
224225
} catch (CharacterCodingException e) {
225226
throw new IllegalStateException(e);
226227
}

‎netty-utils/src/main/java/org/asynchttpclient/netty/util/ByteBufUtils.java

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,35 +13,44 @@
1313
*/
1414
package org.asynchttpclient.netty.util;
1515

16+
import static java.nio.charset.StandardCharsets.*;
1617
import io.netty.buffer.ByteBuf;
18+
import io.netty.buffer.CompositeByteBuf;
19+
import io.netty.buffer.Unpooled;
1720

18-
import java.io.UTFDataFormatException;
1921
import java.nio.charset.CharacterCodingException;
2022
import java.nio.charset.Charset;
21-
import java.nio.charset.StandardCharsets;
22-
import java.util.Collections;
2323

2424
public final class ByteBufUtils {
2525

2626
private ByteBufUtils() {
2727
}
2828

29-
public static String byteBuf2Utf8String(ByteBuf buf) throws CharacterCodingException {
30-
return Utf8ByteBufDecoder.pooled().decode(Collections.singleton(buf));
29+
public static String byteBuf2String(Charset charset, ByteBuf buf) throws CharacterCodingException {
30+
if (charset == UTF_8 || charset == US_ASCII) {
31+
return Utf8ByteBufCharsetDecoder.decodeUtf8(buf);
32+
} else {
33+
return buf.toString(charset);
34+
}
3135
}
3236

33-
public static String byteBuf2UsAsciiString(ByteBuf buf) throws CharacterCodingException {
34-
return UsAsciiByteBufDecoder.pooled().decode(Collections.singleton(buf));
35-
}
37+
public static String byteBuf2String(Charset charset, ByteBuf... bufs) throws CharacterCodingException {
38+
if (charset == UTF_8 || charset == US_ASCII) {
39+
return Utf8ByteBufCharsetDecoder.decodeUtf8(bufs);
40+
} else {
41+
CompositeByteBuf composite = Unpooled.compositeBuffer(bufs.length);
3642

37-
public static String byteBuf2String(ByteBuf buf, Charset charset) throws UTFDataFormatException, IndexOutOfBoundsException, CharacterCodingException {
43+
try {
44+
for (ByteBuf buf : bufs) {
45+
buf.retain();
46+
composite.addComponent(buf);
47+
}
3848

39-
if (charset.equals(StandardCharsets.US_ASCII)) {
40-
return byteBuf2UsAsciiString(buf);
41-
} else if (charset.equals(StandardCharsets.UTF_8)) {
42-
return byteBuf2Utf8String(buf);
43-
} else {
44-
return buf.toString(charset);
49+
return composite.toString(charset);
50+
51+
} finally {
52+
composite.release();
53+
}
4554
}
4655
}
4756

‎netty-utils/src/main/java/org/asynchttpclient/netty/util/UsAsciiByteBufDecoder.java

Lines changed: 0 additions & 48 deletions
This file was deleted.

‎netty-utils/src/main/java/org/asynchttpclient/netty/util/Utf8ByteBufCharsetDecoder.java

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
/*
2+
* Copyright (c) 2016 AsyncHttpClient Project. All rights reserved.
3+
*
4+
* This program is licensed to you under the Apache License Version 2.0,
5+
* and you may not use this file except in compliance with the Apache License Version 2.0.
6+
* You may obtain a copy of the Apache License Version 2.0 at
7+
* http://www.apache.org/licenses/LICENSE-2.0.
8+
*
9+
* Unless required by applicable law or agreed to in writing,
10+
* software distributed under the Apache License Version 2.0 is distributed on an
11+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
13+
*/
14+
package org.asynchttpclient.netty.util;
15+
16+
import static java.nio.charset.StandardCharsets.UTF_8;
17+
import io.netty.buffer.ByteBuf;
18+
import io.netty.buffer.Unpooled;
19+
20+
import java.nio.ByteBuffer;
21+
import java.nio.CharBuffer;
22+
import java.nio.charset.CharacterCodingException;
23+
import java.nio.charset.CharsetDecoder;
24+
import java.nio.charset.CoderResult;
25+
26+
public class Utf8ByteBufCharsetDecoder {
27+
28+
private static final ThreadLocal<Utf8ByteBufCharsetDecoder> POOL = new ThreadLocal<Utf8ByteBufCharsetDecoder>() {
29+
protected Utf8ByteBufCharsetDecoder initialValue() {
30+
return new Utf8ByteBufCharsetDecoder();
31+
}
32+
};
33+
34+
private static Utf8ByteBufCharsetDecoder pooledDecoder() {
35+
Utf8ByteBufCharsetDecoder decoder = POOL.get();
36+
decoder.reset();
37+
return decoder;
38+
}
39+
40+
public static String decodeUtf8(ByteBuf buf) throws CharacterCodingException {
41+
return pooledDecoder().decode(buf);
42+
}
43+
44+
public static String decodeUtf8(ByteBuf... bufs) throws CharacterCodingException {
45+
return pooledDecoder().decode(bufs);
46+
}
47+
48+
private final CharsetDecoder decoder = UTF_8.newDecoder();
49+
protected CharBuffer charBuffer = allocateCharBuffer(1024);
50+
private ByteBuffer splitCharBuffer;
51+
52+
protected void initSplitCharBuffer() {
53+
if (splitCharBuffer == null) {
54+
// UTF-8 chars are 4 bytes max
55+
splitCharBuffer = ByteBuffer.allocate(4);
56+
}
57+
}
58+
59+
protected CharBuffer allocateCharBuffer(int l) {
60+
return CharBuffer.allocate(l);
61+
}
62+
63+
private void ensureCapacity(int l) {
64+
if (charBuffer.position() == 0) {
65+
if (charBuffer.capacity() < l) {
66+
charBuffer = allocateCharBuffer(l);
67+
}
68+
} else if (charBuffer.remaining() < l) {
69+
CharBuffer newCharBuffer = allocateCharBuffer(charBuffer.position() + l);
70+
charBuffer.flip();
71+
newCharBuffer.put(charBuffer);
72+
charBuffer = newCharBuffer;
73+
}
74+
}
75+
76+
public void reset() {
77+
decoder.reset();
78+
charBuffer.position(0);
79+
}
80+
81+
private static int charSize(byte firstByte) throws CharacterCodingException {
82+
if ((firstByte >> 5) == -2 && (firstByte & 0x1e) != 0) {
83+
// 2 bytes, 11 bits: 110xxxxx 10xxxxxx
84+
return 2;
85+
86+
} else if ((firstByte >> 4) == -2) {
87+
// 3 bytes, 16 bits: 1110xxxx 10xxxxxx 10xxxxxx
88+
return 3;
89+
90+
} else if ((firstByte >> 3) == -2) {
91+
// 4 bytes, 21 bits: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
92+
return 4;
93+
94+
} else {
95+
// charSize isn't supposed to be called for regular bytes
96+
throw new CharacterCodingException();
97+
}
98+
}
99+
100+
private void handleSplitCharBuffer(ByteBuffer nioBuffer, boolean endOfInput) throws CharacterCodingException {
101+
// TODO we could save charSize
102+
int missingBytes = charSize(splitCharBuffer.get(0)) - splitCharBuffer.position();
103+
104+
if (nioBuffer.remaining() < missingBytes) {
105+
if (endOfInput) {
106+
throw new CharacterCodingException();
107+
}
108+
109+
// still not enough bytes
110+
splitCharBuffer.put(nioBuffer);
111+
112+
} else {
113+
// FIXME better way?
114+
for (int i = 0; i < missingBytes; i++) {
115+
splitCharBuffer.put(nioBuffer.get());
116+
}
117+
118+
splitCharBuffer.flip();
119+
CoderResult res = decoder.decode(splitCharBuffer, charBuffer, endOfInput && !nioBuffer.hasRemaining());
120+
if (res.isError()) {
121+
res.throwException();
122+
}
123+
124+
splitCharBuffer.position(0);
125+
}
126+
}
127+
128+
protected void decodePartial(ByteBuffer nioBuffer, boolean endOfInput) throws CharacterCodingException {
129+
// deal with pending splitCharBuffer
130+
if (splitCharBuffer != null && splitCharBuffer.position() > 0 && nioBuffer.hasRemaining()) {
131+
handleSplitCharBuffer(nioBuffer, endOfInput);
132+
}
133+
134+
// decode remaining buffer
135+
if (nioBuffer.hasRemaining()) {
136+
CoderResult res = decoder.decode(nioBuffer, charBuffer, endOfInput);
137+
if (res.isUnderflow()) {
138+
if (nioBuffer.remaining() > 0) {
139+
initSplitCharBuffer();
140+
splitCharBuffer.put(nioBuffer);
141+
}
142+
} else if (res.isError()) {
143+
res.throwException();
144+
}
145+
}
146+
}
147+
148+
private void decode(ByteBuffer[] nioBuffers, int length) throws CharacterCodingException {
149+
int count = nioBuffers.length;
150+
for (int i = 0; i < count; i++) {
151+
decodePartial(nioBuffers[i].duplicate(), i == count - 1);
152+
}
153+
}
154+
155+
private void decodeSingleNioBuffer(ByteBuffer nioBuffer, int length) throws CharacterCodingException {
156+
CoderResult res = decoder.decode(nioBuffer, charBuffer, true);
157+
if (res.isError()) {
158+
res.throwException();
159+
}
160+
}
161+
162+
public String decode(ByteBuf buf) throws CharacterCodingException {
163+
if (buf.isDirect()) {
164+
return buf.toString(UTF_8);
165+
}
166+
167+
int length = buf.readableBytes();
168+
ensureCapacity(length);
169+
170+
if (buf.nioBufferCount() == 1) {
171+
decodeSingleNioBuffer(buf.internalNioBuffer(buf.readerIndex(), length).duplicate(), length);
172+
} else {
173+
decode(buf.nioBuffers(), buf.readableBytes());
174+
}
175+
176+
return charBuffer.flip().toString();
177+
}
178+
179+
public String decode(ByteBuf... bufs) throws CharacterCodingException {
180+
181+
int totalSize = 0;
182+
int totalNioBuffers = 0;
183+
boolean direct = false;
184+
for (ByteBuf buf : bufs) {
185+
if (buf.isDirect()) {
186+
direct = true;
187+
break;
188+
}
189+
totalSize += buf.readableBytes();
190+
totalNioBuffers += buf.nioBufferCount();
191+
}
192+
193+
if (direct) {
194+
ByteBuf wrappedBuffer = Unpooled.wrappedBuffer(bufs);
195+
try {
196+
return wrappedBuffer.toString(UTF_8);
197+
} finally {
198+
wrappedBuffer.release();
199+
}
200+
201+
} else {
202+
ByteBuffer[] nioBuffers = new ByteBuffer[totalNioBuffers];
203+
int i = 0;
204+
for (ByteBuf buf : bufs) {
205+
for (ByteBuffer nioBuffer : buf.nioBuffers()) {
206+
nioBuffers[i++] = nioBuffer;
207+
}
208+
}
209+
210+
ensureCapacity(totalSize);
211+
decode(nioBuffers, totalSize);
212+
213+
return charBuffer.flip().toString();
214+
}
215+
}
216+
}

‎netty-utils/src/main/java/org/asynchttpclient/netty/util/Utf8ByteBufDecoder.java

Lines changed: 0 additions & 50 deletions
This file was deleted.

0 commit comments

Comments
 (0)