diff --git a/protocol/src/main/java/com/zfoo/protocol/buffer/ByteBufUtils.java b/protocol/src/main/java/com/zfoo/protocol/buffer/ByteBufUtils.java index eccc9818..1604bdea 100644 --- a/protocol/src/main/java/com/zfoo/protocol/buffer/ByteBufUtils.java +++ b/protocol/src/main/java/com/zfoo/protocol/buffer/ByteBufUtils.java @@ -22,12 +22,15 @@ import io.netty.util.ReferenceCountUtil; import io.netty.util.collection.IntObjectHashMap; import io.netty.util.collection.LongObjectHashMap; -import java.util.*; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; /** - * “可变长字节码算法”的压缩数据的算法，以达到压缩数据，减少磁盘IO。 + * zfoo的自定义私有协议格式，可以非常方便的对大部分数据定制序列化和反序列化实现，可以对大部分数据结构定制高性能实现方式 *

- * google的ProtocolBuffer和Facebook的thrift底层的通信协议都是由这个算法实现 + * “可变长字节码算法”的压缩数据的算法，压缩数据和减少磁盘IO。google的ProtocolBuf和Facebook的thrift底层的通信协议都是由这个算法实现 * * @author godotg * @version 3.0 diff --git a/protocol/src/main/java/com/zfoo/protocol/buffer/CustomByteBuf.java b/protocol/src/main/java/com/zfoo/protocol/buffer/CustomByteBuf.java new file mode 100644 index 00000000..f67f0399 --- /dev/null +++ b/protocol/src/main/java/com/zfoo/protocol/buffer/CustomByteBuf.java @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2020 The zfoo Authors + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + */ + +package com.zfoo.protocol.buffer; + +import com.zfoo.protocol.collection.CollectionUtils; +import io.netty.buffer.ByteBuf; + +import java.nio.ByteBuffer; + +/** + * 自定义私有协议格式，可以针对性的对存在性能瓶颈的数据结构做特定优化 + * + * @author godotg + * @version 3.0 + */ +public abstract class CustomByteBuf { + + // ------------------------------------------------------------------------------------------------------------- + // 针对于int数组提高性能的简单方式 + public static void writeIntArraySimple(ByteBuf byteBuf, int[] array) { + if (array == null) { + byteBuf.writeByte(0); + return; + } + var length = array.length; + ByteBufUtils.writeInt(byteBuf, length); + var writeIndex = byteBuf.writerIndex(); + byteBuf.ensureWritable(length * 4); + for (var value : array) { + byteBuf.setInt(writeIndex, value); + writeIndex += 4; + } + byteBuf.writerIndex(writeIndex); + } + + public static int[] readIntArraySimple(ByteBuf byteBuf) { + var length = ByteBufUtils.readInt(byteBuf); + var ints = new int[CollectionUtils.comfortableLength(length)]; + var readIndex = byteBuf.readerIndex(); + for (var i = 0; i < length; i++) { + ints[i] = byteBuf.getInt(readIndex); + readIndex += 4; + } + byteBuf.readerIndex(readIndex); + return ints; + } + + // 针对于int数组提高性能的复杂方式 + public static void writeIntArrayComplex(ByteBuf byteBuf, int[] array) { + if (array == null) { + byteBuf.writeByte(0); + return; + } + var length = array.length; + ByteBufUtils.writeInt(byteBuf, length); + + var byteBuffer = ByteBuffer.allocate(length * 4); + var intBuffer = byteBuffer.asIntBuffer(); + intBuffer.put(array); + byteBuf.writeBytes(byteBuffer); + } + + public static int[] readIntArrayComplex(ByteBuf byteBuf) { + var length = ByteBufUtils.readInt(byteBuf); + var byteBuffer = ByteBuffer.allocate(length * 4); + byteBuf.readBytes(byteBuffer); + var intBuffer = byteBuffer.asIntBuffer(); + return intBuffer.array(); + } +} diff --git a/protocol/src/test/java/com/zfoo/protocol/buffer/ByteBufUtilsTest.java b/protocol/src/test/java/com/zfoo/protocol/buffer/ByteBufUtilsTest.java index aa379a9e..b7f74911 100644 --- a/protocol/src/test/java/com/zfoo/protocol/buffer/ByteBufUtilsTest.java +++ b/protocol/src/test/java/com/zfoo/protocol/buffer/ByteBufUtilsTest.java @@ -13,6 +13,9 @@ package com.zfoo.protocol.buffer; +import com.zfoo.protocol.ProtocolManager; +import com.zfoo.protocol.buffer.model.BigDataPacket; +import com.zfoo.protocol.util.StringUtils; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; import io.netty.buffer.Unpooled; @@ -21,6 +24,9 @@ import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; +import java.util.Arrays; +import java.util.Set; + /** * @author godotg * @version 3.0 @@ -162,4 +168,24 @@ public class ByteBufUtilsTest { } } + @Ignore + @Test + public void bigDataTest() { + ProtocolManager.initProtocol(Set.of(BigDataPacket.class)); + + var bigDataPact = new BigDataPacket(); + Arrays.fill(bigDataPact.a, 99); + + var buffer = new UnpooledHeapByteBuf(ByteBufAllocator.DEFAULT, 100, 100_0000); + + long startTime = System.currentTimeMillis(); + for (int i = 0; i < 10_0000; i++) { + buffer.clear(); + ProtocolManager.write(buffer, bigDataPact); + var newPacket = ProtocolManager.read(buffer); + } + + System.out.println(StringUtils.format("[zfoo][size:{}] [time:{}]", buffer.writerIndex(), System.currentTimeMillis() - startTime)); + } + } diff --git a/protocol/src/test/java/com/zfoo/protocol/buffer/model/BigDataPacket.java b/protocol/src/test/java/com/zfoo/protocol/buffer/model/BigDataPacket.java new file mode 100644 index 00000000..5f150d37 --- /dev/null +++ b/protocol/src/test/java/com/zfoo/protocol/buffer/model/BigDataPacket.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2020 The zfoo Authors + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + */ + +package com.zfoo.protocol.buffer.model; + +import com.zfoo.protocol.IPacket; +import com.zfoo.protocol.registration.anno.Protocol; + +/** + * @author godotg + * @version 3.0 + */ +@Protocol(id = 1000) +public class BigDataPacket implements IPacket { + + public int[] a = new int[10_0000]; + +}