diff --git a/examples/cs_bin_meta/big_data.toml b/examples/cs_bin_meta/big_data.toml new file mode 100644 index 0000000..c03c71f --- /dev/null +++ b/examples/cs_bin_meta/big_data.toml @@ -0,0 +1,15 @@ +target = "big_data" +mode = "" +source_type = "excel" + +sources = [ + { file_name = "big_data.xlsx", sheet_name = "Sheet1" }, +] + +fields = [ + { active = true, sname = "id" , tname = "id" , type = "int" , key = 1, desc = "id" }, + { active = true, sname = "name" , tname = "name" , type = "string" , key = 0, desc = "名字" }, + { active = true, sname = "age" , tname = "age" , type = "int" , key = 0, desc = "年龄" }, + { active = true, sname = "course" , tname = "course" , type = "[]int" , key = 0, desc = "学科id" }, + { active = true, sname = "score" , tname = "score" , type = "map[int]int" , key = 0, desc = "成绩组" }, +] diff --git a/examples/cs_bin_meta/csv_test.toml b/examples/cs_bin_meta/csv_test.toml new file mode 100644 index 0000000..8459988 --- /dev/null +++ b/examples/cs_bin_meta/csv_test.toml @@ -0,0 +1,13 @@ +target = "csv_test" +mode = "" +source_type = "csv" + +sources = [ + { file_name = "csv_test.csv", sheet_name = "" }, +] + +fields = [ + { active = true, sname = "id" , tname = "id" , type = "int" , key = 1, desc = "id" }, + { active = true, sname = "name" , tname = "name" , type = "string" , key = 0, desc = "名字" }, + { active = true, sname = "age" , tname = "age" , type = "int" , key = 0, desc = "年龄" }, +] diff --git a/examples/excels/big_data.xlsx b/examples/excels/big_data.xlsx new file mode 100644 index 0000000..b577ee2 Binary files /dev/null and b/examples/excels/big_data.xlsx differ diff --git a/examples/projects/proj_cs_bin/src/gen/Tblbig_data.cs b/examples/projects/proj_cs_bin/src/gen/Tblbig_data.cs new file mode 100644 index 0000000..92d4eee --- /dev/null +++ b/examples/projects/proj_cs_bin/src/gen/Tblbig_data.cs @@ -0,0 +1,122 @@ +using Serialization; + +using System.Collections.Generic; + +namespace CfgTable +{ + + public partial class Tblbig_data + { + + private readonly Dictionary _dataMap; + private readonly List _dataList; + + + public Tblbig_data(ByteBuf _buf) + { + //first read common data + _TbCommonbig_data _commonData = null; + var commonSize = _buf.ReadSize(); + if( commonSize > 0) + { + _commonData = new _TbCommonbig_data(_buf); + } + + var size = _buf.ReadSize(); + _dataMap = new Dictionary(16); + _dataList = new List(size); + + for (int i = 0; i < size; i++) + { + Cfgbig_data _v; + _v = Cfgbig_data.DeserializeCfgbig_data(_buf, _commonData); + _dataList.Add(_v); + _dataMap[_v.id] = _v; + } + + PostInit(); + } + + public int DataCount => _dataList.Count; + public Dictionary DataMap => _dataMap; + public List DataList => _dataList; + + public Cfgbig_data GetDataById(int __k0) { if (_dataMap.TryGetValue(__k0, out var __tmpv0)) { return __tmpv0; } return null; } + + /// + /// post process table + /// + partial void PostInit(); + + } + + public partial class Cfgbig_data + { + private Cfgbig_data(ByteBuf _buf, _TbCommonbig_data _commonData) + { + + id = _buf.ReadInt(); + { int dataIndex = _buf.ReadInt() - 1; name = _commonData._field0[dataIndex]; } + age = _buf.ReadInt(); + { int dataIndex = _buf.ReadInt() - 1; course = _commonData._field1[dataIndex]; } + { int dataIndex = _buf.ReadInt() - 1; score = _commonData._field2[dataIndex]; } + PostInit(); + } + + internal static Cfgbig_data DeserializeCfgbig_data(ByteBuf _buf, _TbCommonbig_data _commonData) + { + return new Cfgbig_data(_buf, _commonData); + } + + /// + /// id + /// + public int id { get; private set; } + + /// + /// 名字 + /// + public string name { get; private set; } + + /// + /// 年龄 + /// + public int age { get; private set; } + + /// + /// 学科id + /// + public int[] course { get; private set; } + + /// + /// 成绩组 + /// + public Dictionary score { get; private set; } + + + /// + /// post process table + /// + partial void PostInit(); + } + + /// + /// internal common data.Optimize memory + /// + internal class _TbCommonbig_data + { + + internal string[] _field0 { get; private set; } + internal int[][] _field1 { get; private set; } + internal Dictionary[] _field2 { get; private set; } + internal _TbCommonbig_data(ByteBuf _buf) + { + + {int __n0 = _buf.ReadSize(); _field0 = new string[__n0];for(var __i0 = 0 ; __i0 < __n0 ; __i0++ ){string __v0; __v0 = _buf.ReadString(); _field0[__i0] = __v0;} } + {int __n0 = _buf.ReadSize(); _field1 = new int[__n0][];for(var __i0 = 0 ; __i0 < __n0 ; __i0++ ){int[] __v0; {int __n1 = _buf.ReadSize(); __v0 = new int[__n1];for(var __i1 = 0 ; __i1 < __n1 ; __i1++ ){int __v1; __v1 = _buf.ReadInt(); __v0[__i1] = __v1;} } _field1[__i0] = __v0;} } + {int __n0 = _buf.ReadSize(); _field2 = new Dictionary[__n0];for(var __i0 = 0 ; __i0 < __n0 ; __i0++ ){Dictionary __v0; { int __n1 = _buf.ReadSize(); var __f1 = new Dictionary (__n1 * 3 / 2) ; __v0 = __f1; for(var __i1 = 0 ; __i1 < __n1 ; __i1++ ) {int __k1; __k1 = _buf.ReadInt(); int __v1; __v1 = _buf.ReadInt(); __f1.Add(__k1, __v1); } } _field2[__i0] = __v0;} } + } + + } + +} diff --git a/examples/projects/proj_cs_bin/src/gen/Tblcsv_test.cs b/examples/projects/proj_cs_bin/src/gen/Tblcsv_test.cs new file mode 100644 index 0000000..ef96135 --- /dev/null +++ b/examples/projects/proj_cs_bin/src/gen/Tblcsv_test.cs @@ -0,0 +1,104 @@ +using Serialization; + +using System.Collections.Generic; + +namespace CfgTable +{ + + public partial class Tblcsv_test + { + + private readonly Dictionary _dataMap; + private readonly List _dataList; + + + public Tblcsv_test(ByteBuf _buf) + { + //first read common data + _TbCommoncsv_test _commonData = null; + var commonSize = _buf.ReadSize(); + if( commonSize > 0) + { + _commonData = new _TbCommoncsv_test(_buf); + } + + var size = _buf.ReadSize(); + _dataMap = new Dictionary(16); + _dataList = new List(size); + + for (int i = 0; i < size; i++) + { + Cfgcsv_test _v; + _v = Cfgcsv_test.DeserializeCfgcsv_test(_buf, _commonData); + _dataList.Add(_v); + _dataMap[_v.id] = _v; + } + + PostInit(); + } + + public int DataCount => _dataList.Count; + public Dictionary DataMap => _dataMap; + public List DataList => _dataList; + + public Cfgcsv_test GetDataById(int __k0) { if (_dataMap.TryGetValue(__k0, out var __tmpv0)) { return __tmpv0; } return null; } + + /// + /// post process table + /// + partial void PostInit(); + + } + + public partial class Cfgcsv_test + { + private Cfgcsv_test(ByteBuf _buf, _TbCommoncsv_test _commonData) + { + + id = _buf.ReadInt(); + name = _buf.ReadString(); + age = _buf.ReadInt(); + PostInit(); + } + + internal static Cfgcsv_test DeserializeCfgcsv_test(ByteBuf _buf, _TbCommoncsv_test _commonData) + { + return new Cfgcsv_test(_buf, _commonData); + } + + /// + /// id + /// + public int id { get; private set; } + + /// + /// 名字 + /// + public string name { get; private set; } + + /// + /// 年龄 + /// + public int age { get; private set; } + + + /// + /// post process table + /// + partial void PostInit(); + } + + /// + /// internal common data.Optimize memory + /// + internal class _TbCommoncsv_test + { + + internal _TbCommoncsv_test(ByteBuf _buf) + { + + } + + } + +} diff --git a/examples/projects/proj_cs_bin/table_bytes/big_data.bytes b/examples/projects/proj_cs_bin/table_bytes/big_data.bytes new file mode 100644 index 0000000..53829bf Binary files /dev/null and b/examples/projects/proj_cs_bin/table_bytes/big_data.bytes differ diff --git a/examples/projects/proj_cs_bin/table_bytes/csv_test.bytes b/examples/projects/proj_cs_bin/table_bytes/csv_test.bytes new file mode 100644 index 0000000..04c3d42 Binary files /dev/null and b/examples/projects/proj_cs_bin/table_bytes/csv_test.bytes differ diff --git a/meta/gen_meta.go b/meta/gen_meta.go index 7f66336..4898342 100644 --- a/meta/gen_meta.go +++ b/meta/gen_meta.go @@ -28,8 +28,10 @@ func (g *GenMeta) Run() { sourceSlice := strings.Split(g.genSource, ",") inputOk := false + isCsv := false if len(sourceSlice) == 2 && strings.HasSuffix(sourceSlice[1], ".csv") { inputOk = true + isCsv = true } if len(sourceSlice) == 3 && !strings.HasSuffix(sourceSlice[1], ".csv") { inputOk = true @@ -55,7 +57,7 @@ func (g *GenMeta) Run() { sheetName := "" var rows [][]string var err error - if len(sourceSlice) == 3 { + if !isCsv { sheetName = sourceSlice[2] rows, err = readExcelFile(filePath, sheetName) } else { @@ -73,7 +75,12 @@ func (g *GenMeta) Run() { rtm := NewRawTableMeta() rtm.Target = targetName rtm.Mode = "" - rtm.SourceType = "excel" + if isCsv { + rtm.SourceType = "csv" + } else { + rtm.SourceType = "excel" + } + rtm.Sources = []*RawTableSource{ &RawTableSource{ Table: srcFileName, @@ -127,5 +134,14 @@ func readCsvFile(filePath string) ([][]string, error) { } csvReader := csv.NewReader(csvFile) //创建一个新的写入文件流 csvReader.LazyQuotes = true - return csvReader.ReadAll() + rowData, err := csvReader.ReadAll() + if err != nil { + return nil, err + } + //去除utf-8的bom + if len(rowData) > 0 && len(rowData[0]) > 0 { + firstData := rowData[0][0] + rowData[0][0] = strings.TrimPrefix(firstData, "\uFEFF") + } + return rowData, nil }