tiennm99/zfoo
1
0
Fork 0
mirror of https://github.com/tiennm99/zfoo.git synced 2026-08-20 06:24:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat[storage]: 新增excel转换成json格式数据

Browse Source
This commit is contained in:
meiwei
2022-07-07 15:20:20 +08:00
parent c5760e52b8
commit e8b1d8f079
10 changed files with 342 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
protocol/src/main/java/com/zfoo/protocol/util/JsonUtils.java
+23 -2
View File
@@ -14,9 +14,13 @@ package com.zfoo.protocol.util;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.PrettyPrinter;
import com.fasterxml.jackson.core.util.DefaultIndenter;
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationConfig;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.module.afterburner.AfterburnerModule;
import com.zfoo.protocol.exception.RunException;
@@ -53,6 +57,13 @@ public abstract class JsonUtils {
//反序列化
//当反序列化有未知属性则抛异常，true打开这个设置
MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true);
//美化输出
SerializationConfig config = MAPPER.getSerializationConfig();
PrettyPrinter prettyPrinter = config.getDefaultPrettyPrinter();
DefaultPrettyPrinter defpp = (DefaultPrettyPrinter) prettyPrinter;
DefaultPrettyPrinter.Indenter indenter = new DefaultIndenter(" ", DefaultIndenter.SYS_LF);
defpp.indentArraysWith(indenter);
defpp.indentObjectsWith(indenter);
MAPPER_TURBO.setSerializationInclusion(JsonInclude.Include.NON_NULL);
MAPPER_TURBO.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
@@ -68,6 +79,7 @@ public abstract class JsonUtils {
}
}
//普通输出
public static String object2String(Object object) {
try {
return MAPPER.writeValueAsString(object);
@@ -76,6 +88,15 @@ public abstract class JsonUtils {
}
}
//格式化/美化/优雅的输出
public static String object2StringPrettyPrinter(Object object) {
try {
return MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(object);
} catch (Exception e) {
throw new RunException(e, "将对象[object:{}]转换为json字符串时异常", object);
}
}
/**
* 涡轮增压极大提升json转换性能，字节码增强
*/
@@ -193,9 +214,9 @@ public abstract class JsonUtils {
// 循环遍历子节点下的信息
while (iterator.hasNext()) {
var node = iterator.next();
var field = node.getKey();
var filed = node.getKey();
var value = node.getValue().asText();
jsonMap.put(field, value);
jsonMap.put(filed, value);
}
}
return jsonMap;
storage/src/main/java/com/zfoo/storage/interpreter/JsonResource.java
+20 -1
View File
@@ -44,7 +44,18 @@ public class JsonResource {
private String name;
//类型
private String type;
//列
private int index;
public ColumnMeta() {
}
public ColumnMeta(String name, String type, int index) {
this.name = name;
this.type = type;
this.index = index;
}
public String getName() {
return name;
}
@@ -60,5 +71,13 @@ public class JsonResource {
public void setType(String type) {
this.type = type;
}
public int getIndex() {
return index;
}
public void setIndex(int index) {
this.index = index;
}
}
}
storage/src/main/java/com/zfoo/storage/util/ExcelToJsonUtils.java
+162
View File
@@ -0,0 +1,162 @@
/*
* Copyright (C) 2020 The zfoo Authors
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and limitations under the License.
*/
package com.zfoo.storage.util;
import com.zfoo.protocol.exception.RunException;
import com.zfoo.protocol.util.JsonUtils;
import com.zfoo.protocol.util.StringUtils;
import com.zfoo.storage.interpreter.JsonResource;
import com.zfoo.storage.interpreter.JsonResource.ColumnMeta;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Objects;
import org.apache.commons.io.FileUtils;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
/**
* @author jaysunxiao
* @version 3.0
*/
public class ExcelToJsonUtils {
public static void main(String[] args) throws Exception{
File inputDir = new File("E:\\workspace\\zfoo\\storage\\src\\test\\resources\\excel");
String outputDir = "E:\\workspace\\zfoo\\storage\\src\\test\\resources\\excel";
var listFiles = FileUtils.listFiles(inputDir, new String[] { "xls", "xlsx"}, true);
for (var file : listFiles) {
var fileName = getFileName(file);
var inputStream = FileUtils.openInputStream(file);
var jsonStr = read(inputStream, fileName);
writeJsonFile(outputDir, jsonStr, fileName);
}
}
public static void writeJsonFile(String outDir, String jsonStr, String name) {
System.out.println("resource: " + name + ".txt");
PrintWriter pw = null;
try {
File outFile = new File(outDir, name + ".txt");
if (!outFile.exists())
outFile.createNewFile();
pw = new PrintWriter(outFile, "utf-8");
pw.write(jsonStr);
} catch (IOException e) {
System.err.println("error resource:" + name);
} finally {
if (pw != null)
pw.close();
}
}
private static String getFileName(File file) {
String name = file.getName();
int index = name.lastIndexOf(".");
if (index <= 0) {
return "";
}
return name.substring(0, index);
}
public static String read(InputStream inputStream, String fileName) {
var wb = createWorkbook(inputStream, fileName);
var resource = new JsonResource();
resource.setName(fileName);
// 默认取到第一个sheet页
var sheet = wb.getSheetAt(0);
//设置所有列
var columns = getColumnMetas(sheet, fileName);
resource.setColumns(columns);
// 行数定位到有效数据行，默认是第四行为有效数据行
var iterator = sheet.iterator();
iterator.next();
iterator.next();
iterator.next();
// 从ROW_SERVER这行开始读取数据
List<List<String>> data = new ArrayList<>();
while (iterator.hasNext()) {
var row = iterator.next();
List<String> rowData = new ArrayList<>();
for (var column : columns) {
var cell = row.getCell(column.getIndex());
var content = CellUtils.getCellStringValue(cell);
rowData.add(content);
}
data.add(rowData);
}
resource.setData(data);
return JsonUtils.object2StringPrettyPrinter(resource);
}
// 只读取代码里写的字段
private static List<ColumnMeta> getColumnMetas(Sheet sheet, String fileName) {
var iterator = sheet.iterator();
// 获取配置表的有效列名称，默认第一行就是字段名称
var fieldRow = iterator.next();
if (fieldRow == null) {
throw new RunException("无法获取资源[class:{}]的Excel文件的属性控制列", fileName);
}
//默认第二行字段类型
var typeRow = iterator.next();
if (typeRow == null) {
throw new RunException("无法获取资源[class:{}]的Excel文件的类型控制列", fileName);
}
var columnMetaList = new ArrayList<ColumnMeta>();
var cellFieldMap = new HashMap<String, Integer>();
for (var i = 0; i < fieldRow.getLastCellNum(); i++) {
var fieldCell = fieldRow.getCell(i);
if (Objects.isNull(fieldCell)) {
continue;
}
var typeCell = typeRow.getCell(i);
if (Objects.isNull(typeCell)) {
continue;
}
var fieldName = CellUtils.getCellStringValue(fieldCell);
if (StringUtils.isEmpty(fieldName)) {
continue;
}
var typeName = CellUtils.getCellStringValue(typeCell);
if (StringUtils.isEmpty(typeName)) {
continue;
}
var previousValue = cellFieldMap.put(fieldName, i);
if (Objects.nonNull(previousValue)) {
throw new RunException("资源[class:{}]的Excel文件出现重复的属性控制列[field:{}]", fileName, fieldName);
}
columnMetaList.add(new ColumnMeta(fieldName, typeName, i));
}
return columnMetaList;
}
private static Workbook createWorkbook(InputStream input, String fileName) {
try {
return WorkbookFactory.create(input);
} catch (IOException e) {
throw new RunException("静态资源[{}]异常，无法读取文件", fileName);
}
}
}
storage/src/test/java/com/zfoo/storage/ApplicationTest.java
+35 -3
View File
@@ -71,18 +71,50 @@ public class ApplicationTest {
// 配置文件中resource，需要映射Excel的文件所在位置
var context = new ClassPathXmlApplicationContext("application.xml");
// Excel的映射内容需要在被Spring管理的bean的方法上加上@ResInjection注解，即可自动注入Excel对应的对象
// 参考StudentManager中的标准用法
var studentManager = context.getBean(StudentManager.class);
var studentResources = studentManager.studentResources;
// 类名称和Excel名称必须完全一致，Excel的列名称必须对应对象的属性名称
// 类名称和Excel名称必须完全一致，Excel的列名称必须对应对象的属性名称
for (StudentResource resource : studentResources.getAll()) {
logger.info(JsonUtils.object2String(resource));
}
System.out.println(StringUtils.MULTIPLE_HYPHENS);
// 通过id找到对应的行
var id = 1002;
var valueById = studentResources.get(id);
logger.info(JsonUtils.object2String(valueById));
System.out.println(StringUtils.MULTIPLE_HYPHENS);
// 通过索引找对应的行
var valuesByIndex = studentResources.getIndex("name", "james0");
logger.info(JsonUtils.object2String(valuesByIndex));
}
// storage教程
@Test
public void startStorageTestTxt2() {
// 加载配置文件，配置文件中必须引入storage
// 配置文件中scan，需要映射Excel的类所在位置，会自动搜索文件夹下的Excel文件，Excel文件可以放在指定文件夹的任意目录
// 配置文件中resource，需要映射Excel的文件所在位置
var context = new ClassPathXmlApplicationContext("application.xml");
// Excel的映射内容需要在被Spring管理的bean的方法上加上@ResInjection注解，即可自动注入Excel对应的对象
var testManager = context.getBean(TestManager.class);
var testResources = testManager.testResources;
for (com.zfoo.storage.resource.TestResource resource : testResources.getAll()) {
Map<Integer, String> map = resource.getType9();
logger.info(map.get(1));
// logger.info(JsonUtils.object2String(resource));
System.err.println(map.get(1));
logger.info(JsonUtils.object2String(resource));
}
// 通过id找到对应的行
var id = 2;
var valueById = testResources.get(id);
logger.info(JsonUtils.object2String(valueById));
logger.info(StringUtils.MULTIPLE_HYPHENS);
System.out.println(StringUtils.MULTIPLE_HYPHENS);
}
}
storage/src/test/java/com/zfoo/storage/StudentManager.java
+1 -1
View File
@@ -22,7 +22,7 @@ import org.springframework.stereotype.Component;
* @author jaysunxiao
* @version 3.0
*/
//@Component
@Component
public class StudentManager {
@ResInjection
storage/src/test/java/com/zfoo/storage/TestManager.java
+1 -1
View File
@@ -24,7 +24,7 @@ import org.springframework.stereotype.Component;
* @author jaysunxiao
* @version 3.0
*/
@Component
//@Component
public class TestManager {
@ResInjection
storage/src/test/java/com/zfoo/storage/resource/StudentResource.java
+1 -1
View File
@@ -21,7 +21,7 @@ import com.zfoo.storage.model.anno.Resource;
* @author jaysunxiao
* @version 3.0
*/
//@Resource
@Resource
public class StudentResource {
@Id
storage/src/test/java/com/zfoo/storage/resource/TestResource.java
+1 -1
View File
@@ -24,7 +24,7 @@ import com.zfoo.storage.model.anno.Resource;
* @author jaysunxiao
* @version 3.0
*/
@Resource
//@Resource
public class TestResource {
@Id
storage/src/test/resources/excel/StudentResource.txt
+78
View File
@@ -0,0 +1,78 @@
{
"name" : "StudentResource",
"columns" : [
{
"name" : "id",
"type" : "string",
"index" : 0
},
{
"name" : "name",
"type" : "string",
"index" : 1
},
{
"name" : "age",
"type" : "int",
"index" : 2
},
{
"name" : "score",
"type" : "float",
"index" : 3
},
{
"name" : "courses",
"type" : "array",
"index" : 4
},
{
"name" : "users",
"type" : "array",
"index" : 5
},
{
"name" : "user",
"type" : "string",
"index" : 6
}
],
"data" : [
[
"1000",
"james0",
"10",
"60.8",
"[\"History\",\"Chiness\"]",
"[{\"id\":\"1000\",\"name\":\"SunInsanity\",\"sex\":\"boy\",\"age\":22},{\"id\":\"1000\",\"name\":\"SunInsanity\",\"sex\":\"boy\",\"age\":22}]",
"{\"id\":\"1000\",\"name\":\"SunInsanity\",\"sex\":\"boy\",\"age\":22}"
],
[
"1001",
"james1",
"10",
"70.1",
"[\"History\",\"Chiness\"]",
"[{\"id\":\"1000\",\"name\":\"SunInsanity\",\"sex\":\"boy\",\"age\":22},{\"id\":\"1000\",\"name\":\"SunInsanity\",\"sex\":\"boy\",\"age\":23}]",
"{\"id\":\"1000\",\"name\":\"SunInsanity\",\"sex\":\"boy\",\"age\":22}"
],
[
"1002",
"james2",
"10",
"80.2",
"[\"History\",\"Chiness\"]",
"[{\"id\":\"1000\",\"name\":\"SunInsanity\",\"sex\":\"boy\",\"age\":22},{\"id\":\"1000\",\"name\":\"SunInsanity\",\"sex\":\"boy\",\"age\":24}]",
"{\"id\":\"1000\",\"name\":\"SunInsanity\",\"sex\":\"boy\",\"age\":22}"
],
[
"1003",
"james3",
"10",
"59",
"[\"History\",\"Chiness\"]",
"[{\"id\":\"1000\",\"name\":\"SunInsanity\",\"sex\":\"boy\",\"age\":22},{\"id\":\"1000\",\"name\":\"SunInsanity\",\"sex\":\"boy\",\"age\":25}]",
"{\"id\":\"1000\",\"name\":\"SunInsanity\",\"sex\":\"boy\",\"age\":22}"
]
]
}
storage/src/test/resources/excel/TestResource.txt
+20 -10
View File
@@ -3,43 +3,53 @@
"columns": [
{
"name": "Id",
"type": "int"
"type": "int",
"index" : 0
},
{
"name": "Type0",
"type": "long"
"type": "long",
"index" : 0
},
{
"name": "Type1",
"type": "string"
"type": "string",
"index" : 0
},
{
"name": "Type2",
"type": "list\u003cstring\u003e"
"type": "list\u003cstring\u003e",
"index" : 0
},
{
"name": "Type3",
"type": "list\u003cint\u003e"
"type": "list\u003cint\u003e",
"index" : 0
},
{
"name": "Type4",
"type": "list\u003clong\u003e"
"type": "list\u003clong\u003e",
"index" : 0
},
{
"name": "Type5",
"type": "array\u003cattr,id,name\u003e"
"type": "array\u003cattr,id,name\u003e",
"index" : 0
},
{
"name": "Type8",
"type": "map\u003cint,int\u003e"
"type": "map\u003cint,int\u003e",
"index" : 0
},
{
"name": "Type9",
"type": "map\u003cint,string\u003e"
"type": "map\u003cint,string\u003e",
"index" : 0
},
{
"name": "Type10",
"type": "map\u003cstring,string\u003e"
"type": "map\u003cstring,string\u003e",
"index" : 0
}
],
"data": [