qdy
/
go-svc-agent
Watch 1
Star 0
Fork 0
Code Issues 0 Pull Requests 0 Releases 5 Wiki Activity
Browse Source

初始提交

qdy 3 months ago
commit
3e93dd0df2
12 changed files with 1528 additions and 0 deletions
Split View Show Diff Stats
  1. 39
    0
      functions/query_csv.go
  2. 23
    0
      functions/query_json.go
  3. 82
    0
      gct.sh
  4. 55
    0
      go.mod
  5. 143
    0
      go.sum
  6. 120
    0
      main.go
  7. 302
    0
      test/my0_test.go
  8. 112
    0
      test/my1_test.go
  9. 92
    0
      test/my2_test.go
  10. 394
    0
      test/my_ora_clothingToDoris_test.go
  11. 69
    0
      test/mycsv_test.go
  12. 97
    0
      test/mysql_test.go

+ 39
- 0
functions/query_csv.go View File 

@@ -0,0 +1,39 @@
1 
+package functions
2 
+
3 
+import (
4 
+	"log"
5 
+
6 
+	"git.x2erp.com/qdy/go-base/types"
7 
+	"git.x2erp.com/qdy/go-db/factory/database"
8 
+)
9 
+
10 
+// 执行查询，返回CSV数据格式。无参数查询
11 
+func QueryToCSV(dbFactory *database.DBFactory, req types.QueryRequest) []byte {
12 
+	csvData, err := dbFactory.QueryToCSV(req.SQL, req.WriterHeader)
13 
+	if err != nil {
14 
+		log.Fatalf("QueryToCSV error: %v", err)
15 
+
16 
+	}
17 
+	return csvData
18 
+}
19 
+
20 
+// 执行查询，返回CSV数据格式。带参数名称进行查询
21 
+func QueryParamNameToCSV(dbFactory *database.DBFactory, req types.QueryRequest) []byte {
22 
+
23 
+	csvData, err := dbFactory.QueryParamsNameToCSV(req.SQL, req.WriterHeader, req.Params)
24 
+	if err != nil {
25 
+		log.Fatalf("QueryParamNameToCSV Error: %v", err)
26 
+
27 
+	}
28 
+	return csvData
29 
+}
30 
+
31 
+// 执行查询，返回CSV数据格式。带占位参数进行查询
32 
+func QueryPositionalToCSV(dbFactory *database.DBFactory, req types.QueryRequest) []byte {
33 
+	csvData, err := dbFactory.QueryPositionalToCSV(req.SQL, req.WriterHeader, req.PositionalParams)
34 
+	if err != nil {
35 
+		log.Fatalf("QueryParamNameToCSV Error: %v", err)
36 
+
37 
+	}
38 
+	return csvData
39 
+}

+ 23
- 0
functions/query_json.go View File 

@@ -0,0 +1,23 @@
1 
+package functions
2 
+
3 
+import (
4 
+	"git.x2erp.com/qdy/go-base/types"
5 
+	"git.x2erp.com/qdy/go-db/factory/database"
6 
+)
7 
+
8 
+// 执行查询，返回CSV数据格式。无参数查询
9 
+func QueryToJSON(dbFactory *database.DBFactory, req types.QueryRequest) *types.QueryResult {
10 
+	return dbFactory.QueryToJSON(req.SQL)
11 
+
12 
+}
13 
+
14 
+// 执行查询，返回CSV数据格式。带参数名称进行查询
15 
+func QueryParamNameToJSON(dbFactory *database.DBFactory, req types.QueryRequest) *types.QueryResult {
16 
+	return dbFactory.QueryParamsNameToJSON(req.SQL, req.Params)
17 
+}
18 
+
19 
+// 执行查询，返回JSON数据格式。带占位参数进行查询
20 
+func QueryPositionalToJSON(dbFactory *database.DBFactory, req types.QueryRequest) *types.QueryResult {
21 
+	return dbFactory.QueryPositionalToJSON(req.SQL, req.PositionalParams)
22 
+
23 
+}

+ 82
- 0
gct.sh View File 

@@ -0,0 +1,82 @@
1 
+#!/bin/bash
2 
+
3 
+# 脚本用法：./git-commit-and-tag.sh "你的提交描述" "版本号"
4 
+
5 
+# 检查参数数量
6 
+if [ $# -ne 2 ]; then
7 
+    echo "错误: 脚本需要2个参数。"
8 
+    echo "用法: $0 \"提交描述\" \"版本号\""
9 
+    echo "示例: $0 \"修复了登录问题\" \"v1.2.3\""
10 
+    exit 1
11 
+fi
12 
+
13 
+# 分配参数
14 
+COMMIT_MESSAGE="$1"
15 
+VERSION_TAG="$2"
16 
+
17 
+# 检查当前目录是否为Git仓库
18 
+if ! git rev-parse --git-dir > /dev/null 2>&1; then
19 
+    echo "错误: 当前目录不是一个Git仓库。"
20 
+    exit 1
21 
+fi
22 
+
23 
+echo "开始处理提交和版本标签..."
24 
+echo "提交描述: $COMMIT_MESSAGE"
25 
+echo "版本标签: $VERSION_TAG"
26 
+
27 
+# 检查是否有未提交的更改
28 
+if [ -n "$(git status --porcelain)" ]; then
29 
+    echo "检测到未提交的更改，正在提交..."
30 
+
31 
+    # 添加所有更改到暂存区
32 
+    git add .
33 
+
34 
+    # 进行提交
35 
+    git commit -m "$COMMIT_MESSAGE"
36 
+    if [ $? -ne 0 ]; then
37 
+        echo "错误: 提交失败。"
38 
+        exit 1
39 
+    fi
40 
+    echo "✅ 更改已提交"
41 
+else
42 
+    echo "提示: 没有未提交的更改，跳过提交步骤"
43 
+
44 
+    # 检查是否有未提交的commit但未推送
45 
+    LOCAL_COMMITS=$(git log @{u}..HEAD --oneline 2>/dev/null | wc -l)
46 
+    if [ $LOCAL_COMMITS -eq 0 ]; then
47 
+        echo "错误: 没有需要推送的提交。"
48 
+        exit 1
49 
+    else
50 
+        echo "检测到 $LOCAL_COMMITS 个本地提交等待推送"
51 
+    fi
52 
+fi
53 
+
54 
+# 检查标签是否已存在
55 
+if git rev-parse "$VERSION_TAG" >/dev/null 2>&1; then
56 
+    echo "错误: 标签 '$VERSION_TAG' 已经存在。"
57 
+    exit 1
58 
+fi
59 
+
60 
+# 创建标签
61 
+git tag "$VERSION_TAG"
62 
+if [ $? -ne 0 ]; then
63 
+    echo "错误: 创建标签失败。"
64 
+    exit 1
65 
+fi
66 
+echo "✅ 标签 '$VERSION_TAG' 已创建"
67 
+
68 
+# 推送到远程仓库并推送标签
69 
+echo "正在推送到远程仓库..."
70 
+git push
71 
+if [ $? -ne 0 ]; then
72 
+    echo "错误: 推送提交失败。"
73 
+    exit 1
74 
+fi
75 
+
76 
+git push origin "$VERSION_TAG"
77 
+if [ $? -ne 0 ]; then
78 
+    echo "错误: 推送标签失败。"
79 
+    exit 1
80 
+fi
81 
+
82 
+echo "✅ 完成！提交已推送，版本标签 $VERSION_TAG 已创建并推送。"

+ 55
- 0
go.mod View File 

@@ -0,0 +1,55 @@
1 
+module git.x2erp.com/qdy/go-svc-agent
2 
+
3 
+go 1.25.4
4 
+
5 
+require (
6 
+	git.x2erp.com/qdy/go-base v0.1.70
7 
+	git.x2erp.com/qdy/go-db v0.1.70
8 
+)
9 
+
10 
+require (
11 
+	filippo.io/edwards25519 v1.1.0 // indirect
12 
+	github.com/bytedance/sonic v1.14.0 // indirect
13 
+	github.com/bytedance/sonic/loader v0.3.0 // indirect
14 
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
15 
+	github.com/cloudwego/base64x v0.1.6 // indirect
16 
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
17 
+	github.com/gabriel-vasile/mimetype v1.4.8 // indirect
18 
+	github.com/gin-contrib/sse v1.1.0 // indirect
19 
+	github.com/go-playground/locales v0.14.1 // indirect
20 
+	github.com/go-playground/universal-translator v0.18.1 // indirect
21 
+	github.com/go-playground/validator/v10 v10.27.0 // indirect
22 
+	github.com/go-redis/redis/v8 v8.11.5 // indirect
23 
+	github.com/go-resty/resty/v2 v2.17.0 // indirect
24 
+	github.com/go-sql-driver/mysql v1.9.3 // indirect
25 
+	github.com/goccy/go-json v0.10.2 // indirect
26 
+	github.com/goccy/go-yaml v1.18.0 // indirect
27 
+	github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
28 
+	github.com/golang-sql/sqlexp v0.1.0 // indirect
29 
+	github.com/google/uuid v1.6.0 // indirect
30 
+	github.com/jmoiron/sqlx v1.4.0 // indirect
31 
+	github.com/json-iterator/go v1.1.12 // indirect
32 
+	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
33 
+	github.com/leodido/go-urn v1.4.0 // indirect
34 
+	github.com/lib/pq v1.10.9 // indirect
35 
+	github.com/mattn/go-isatty v0.0.20 // indirect
36 
+	github.com/microsoft/go-mssqldb v1.9.4 // indirect
37 
+	github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
38 
+	github.com/modern-go/reflect2 v1.0.2 // indirect
39 
+	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
40 
+	github.com/quic-go/qpack v0.5.1 // indirect
41 
+	github.com/quic-go/quic-go v0.54.0 // indirect
42 
+	github.com/sijms/go-ora/v2 v2.9.0 // indirect
43 
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
44 
+	github.com/ugorji/go/codec v1.3.0 // indirect
45 
+	go.uber.org/mock v0.5.0 // indirect
46 
+	golang.org/x/arch v0.20.0 // indirect
47 
+	golang.org/x/crypto v0.45.0 // indirect
48 
+	golang.org/x/mod v0.29.0 // indirect
49 
+	golang.org/x/net v0.47.0 // indirect
50 
+	golang.org/x/sync v0.18.0 // indirect
51 
+	golang.org/x/sys v0.38.0 // indirect
52 
+	golang.org/x/text v0.31.0 // indirect
53 
+	golang.org/x/tools v0.38.0 // indirect
54 
+	gopkg.in/yaml.v2 v2.4.0 // indirect
55 
+)

+ 143
- 0
go.sum View File 

@@ -0,0 +1,143 @@
1 
+dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
2 
+dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
3 
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
4 
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
5 
+git.x2erp.com/qdy/go-base v0.1.45 h1:kPus8pBAwXPGL1WKFwxbzMwuVWH91mabWnpUuuW8vLM=
6 
+git.x2erp.com/qdy/go-base v0.1.45/go.mod h1:Q+YLwpCoU8CVSnzATLdz2LAzVMlz/CEGzo8DePf7cug=
7 
+github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
8 
+github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
9 
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
10 
+github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw=
11 
+github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE=
12 
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
13 
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
14 
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
15 
+github.com/bitly/go-simplejson v0.5.1 h1:xgwPbetQScXt1gh9BmoJ6j9JMr3TElvuIyjR8pgdoow=
16 
+github.com/bitly/go-simplejson v0.5.1/go.mod h1:YOPVLzCfwK14b4Sff3oP1AmGhI9T9Vsg84etUnlyp+Q=
17 
+github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0=
18 
+github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
19 
+github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
20 
+github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
21 
+github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE=
22 
+github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc=
23 
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
24 
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
25 
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
26 
+github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o=
27 
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
28 
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
29 
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
30 
+github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
31 
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
32 
+github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
33 
+github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
34 
+github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
35 
+github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM=
36 
+github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU=
37 
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
38 
+github.com/go-git/go-git/v5 v5.16.4 h1:7ajIEZHZJULcyJebDLo99bGgS0jRrOxzZG4uCk2Yb2Y=
39 
+github.com/go-git/go-git/v5 v5.16.4/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8=
40 
+github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
41 
+github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
42 
+github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
43 
+github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
44 
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
45 
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
46 
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
47 
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
48 
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
49 
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
50 
+github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
51 
+github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
52 
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
53 
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
54 
+github.com/kevinburke/ssh_config v1.4.0 h1:6xxtP5bZ2E4NF5tuQulISpTO2z8XbtH8cg1PWkxoFkQ=
55 
+github.com/kevinburke/ssh_config v1.4.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M=
56 
+github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
57 
+github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
58 
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
59 
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
60 
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
61 
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
62 
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
63 
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
64 
+github.com/miekg/dns v1.1.68 h1:jsSRkNozw7G/mnmXULynzMNIsgY2dHC8LO6U6Ij2JEA=
65 
+github.com/miekg/dns v1.1.68/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps=
66 
+github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
67 
+github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
68 
+github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
69 
+github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw=
70 
+github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0=
71 
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
72 
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
73 
+github.com/pjbgf/sha1cd v0.5.0 h1:a+UkboSi1znleCDUNT3M5YxjOnN1fz2FhN48FlwCxs0=
74 
+github.com/pjbgf/sha1cd v0.5.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM=
75 
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
76 
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
77 
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
78 
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
79 
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
80 
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
81 
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
82 
+github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
83 
+github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
84 
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
85 
+github.com/skeema/knownhosts v1.3.2 h1:EDL9mgf4NzwMXCTfaxSD/o/a5fxDw/xL9nkU28JjdBg=
86 
+github.com/skeema/knownhosts v1.3.2/go.mod h1:bEg3iQAuw+jyiw+484wwFJoKSLwcfd7fqRy+N0QTiow=
87 
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
88 
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
89 
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
90 
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
91 
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
92 
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
93 
+github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
94 
+github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
95 
+github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
96 
+github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
97 
+github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg=
98 
+github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
99 
+go-micro.dev/v4 v4.11.0 h1:DZ2xcr0pnZJDlp6MJiCLhw4tXRxLw9xrJlPT91kubr0=
100 
+go-micro.dev/v4 v4.11.0/go.mod h1:eE/tD53n3KbVrzrWxKLxdkGw45Fg1qaNLWjpJMvIUF4=
101 
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
102 
+golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
103 
+golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
104 
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
105 
+golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
106 
+golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
107 
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
108 
+golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
109 
+golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
110 
+golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
111 
+golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
112 
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
113 
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
114 
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
115 
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
116 
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
117 
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
118 
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
119 
+golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
120 
+golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
121 
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
122 
+golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
123 
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
124 
+golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
125 
+golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
126 
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
127 
+golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
128 
+golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
129 
+google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
130 
+google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
131 
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
132 
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
133 
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
134 
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
135 
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
136 
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
137 
+gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
138 
+gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
139 
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
140 
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
141 
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
142 
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
143 
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

+ 120
- 0
main.go View File 

@@ -0,0 +1,120 @@
1 
+package main
2 
+
3 
+import (
4 
+	"log"
5 
+	"net/http"
6 
+	"os"
7 
+	"os/signal"
8 
+	"syscall"
9 
+
10 
+	"git.x2erp.com/qdy/go-base/config"
11 
+	"git.x2erp.com/qdy/go-base/middleware"
12 
+	"git.x2erp.com/qdy/go-base/myservice"
13 
+	"git.x2erp.com/qdy/go-db/factory/database"
14 
+	"git.x2erp.com/qdy/go-db/myhandle"
15 
+	"git.x2erp.com/qdy/go-svc-agent/functions"
16 
+	"go-micro.dev/v4/web"
17 
+)
18 
+
19 
+func main() {
20 
+	cfg := config.GetConfig()
21 
+	serviceConfig := cfg.GetService()
22 
+
23 
+	log.Printf("Service Port: %d", serviceConfig.Port)
24 
+	log.Printf("Service IdleTimeout: %d", serviceConfig.IdleTimeout)
25 
+	log.Printf("Service ReadTimeout: %d", serviceConfig.ReadTimeout)
26 
+	log.Printf("Service WriteTimeout: %d", serviceConfig.WriteTimeout)
27 
+	log.Printf("Service TrustedProxies: %s", serviceConfig.TrustedProxies)
28 
+
29 
+	log.Printf("Using database type: %s", cfg.GetDatabase().Type)
30 
+	log.Printf("Database host: %s:%d", cfg.GetDatabase().Host, cfg.GetDatabase().Port)
31 
+	log.Printf("Database name: %s", cfg.GetDatabase().Database)
32 
+	log.Println("Database connection test passed!")
33 
+
34 
+	// 启动微服务
35 
+	startMicroService()
36 
+}
37 
+
38 
+// 启动微服务
39 
+func startMicroService() {
40 
+	cfg := config.GetConfig()
41 
+	serviceConfig := cfg.GetService()
42 
+
43 
+	// 初始化数据库
44 
+	dbFactory, err := database.GetDBFactory()
45 
+	if err != nil {
46 
+		log.Fatalf("Failed to create DB factory: %v", err)
47 
+	}
48 
+	defer func() {
49 
+		if err := dbFactory.Close(); err != nil {
50 
+			log.Printf("Database close error: %v", err)
51 
+		}
52 
+	}()
53 
+
54 
+	// 设置优雅关闭
55 
+	setupGracefulShutdown(dbFactory)
56 
+
57 
+	webService := myservice.StartStandalone(cfg)
58 
+
59 
+	// 注册HTTP路由到webService
60 
+	registerRoutes(webService, dbFactory)
61 
+
62 
+	// 等待服务运行
63 
+	log.Printf("Service started successfully")
64 
+	log.Printf("   • Service: %s", serviceConfig.ServiceName)
65 
+	log.Printf("   • Port: %d", serviceConfig.Port)
66 
+	log.Printf("   • Mode: %s", getServiceMode(cfg))
67 
+
68 
+	// 保持主程序运行
69 
+	select {}
70 
+}
71 
+
72 
+// 判断是否使用注册中心
73 
+func shouldUseRegistry(cfg config.IConfig) bool {
74 
+	microConfig := cfg.GetMicro()
75 
+	// 如果有配置注册中心地址且不为空，则使用注册中心
76 
+	return microConfig.RegistryAddress != ""
77 
+}
78 
+
79 
+// 获取服务模式描述
80 
+func getServiceMode(cfg config.IConfig) string {
81 
+	if shouldUseRegistry(cfg) {
82 
+		return "With Service Discovery"
83 
+	}
84 
+	return "Standalone"
85 
+}
86 
+
87 
+// 注册所有路由
88 
+func registerRoutes(webService web.Service, dbFactory *database.DBFactory) {
89 
+
90 
+	// 查询接口 - JSON
91 
+	webService.Handle("/api/query/json", middleware.JWTAuthMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
92 
+		myhandle.QueryHandlerJson(w, r, dbFactory, functions.QueryToJSON)
93 
+	})))
94 
+
95 
+	// 查询接口 - CSV
96 
+	webService.Handle("/api/query/csv", middleware.JWTAuthMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
97 
+		myhandle.QueryHandlerBytes(w, r, dbFactory, functions.QueryToCSV)
98 
+	})))
99 
+
100 
+	// 查询接口 - CSV with positional params
101 
+	webService.Handle("/api/query/csv/param", middleware.JWTAuthMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
102 
+		myhandle.QueryHandlerBytes(w, r, dbFactory, functions.QueryPositionalToCSV)
103 
+	})))
104 
+}
105 
+
106 
+// 设置优雅关闭
107 
+func setupGracefulShutdown(dbFactory *database.DBFactory) {
108 
+	signalCh := make(chan os.Signal, 1)
109 
+	signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM)
110 
+
111 
+	go func() {
112 
+		<-signalCh
113 
+		log.Println("\nReceived shutdown signal, closing database connection...")
114 
+		if err := dbFactory.Close(); err != nil {
115 
+			log.Printf("Error closing database: %v", err)
116 
+		}
117 
+		log.Println("Database connection closed gracefully")
118 
+		os.Exit(0)
119 
+	}()
120 
+}

+ 302
- 0
test/my0_test.go View File 

@@ -0,0 +1,302 @@
1 
+package main
2 
+
3 
+import (
4 
+	"fmt"
5 
+	"log"
6 
+	"sync"
7 
+	"testing"
8 
+	"time"
9 
+
10 
+	"git.x2erp.com/qdy/go-base/config"
11 
+	"git.x2erp.com/qdy/go-base/types"
12 
+	"git.x2erp.com/qdy/go-db/factory/doris"
13 
+	"git.x2erp.com/qdy/go-db/factory/http"
14 
+)
15 
+
16 
+// // QueryRequest 查询请求结构体
17 
+// type QueryRequest struct {
18 
+// 	SQL              string                 `json:"sql" binding:"required"`
19 
+// 	Params           map[string]interface{} `json:"params,omitempty"`           // 名称参数
20 
+// 	PositionalParams []interface{}          `json:"positionalParams,omitempty"` // 位置参数
21 
+// }
22 
+
23 
+// 分页配置
24 
+type PageConfig1 struct {
25 
+	TotalRows  int // 总共需要查询多少行
26 
+	PageSize   int // 每次返回多少行
27 
+	MaxWorkers int // 几条线程同时工作
28 
+	StartRow   int //从哪行开始查询
29 
+}
30 
+
31 
+// 查询任务
32 
+type QueryTask1 struct {
33 
+	Page        int
34 
+	StartRow    int
35 
+	EndRow      int
36 
+	QuerySQL    string
37 
+	QueryParams []interface{}
38 
+	CSVData     string
39 
+	Error       error
40 
+	QueryTime   time.Duration
41 
+	SaveTime    time.Duration
42 
+}
43 
+
44 
+func TestQueryAndInsertToDoris1(t *testing.T) {
45 
+	// 记录总开始时间
46 
+	totalStartTime := time.Now()
47 
+
48 
+	// 配置分页参数
49 
+	pageConfig := PageConfig{
50 
+		TotalRows:  100, // 总共需要查询多少行
51 
+		PageSize:   50,  // 每次查询2000条
52 
+		MaxWorkers: 2,   // 几条线程同时工作
53 
+		StartRow:   0,   //从哪行开始查询
54 
+	}
55 
+
56 
+	fmt.Printf("开始执行分页查询，总共%d行，每页%d条，%d线程工作\n", pageConfig.TotalRows, pageConfig.PageSize, pageConfig.MaxWorkers)
57 
+
58 
+	// 1. 获取HTTP工厂实例
59 
+	httpFactory, err := http.GetHTTPFactory()
60 
+	if err != nil {
61 
+		t.Fatalf("Failed to get HTTP factory: %v", err)
62 
+	}
63 
+	fmt.Println("HTTP factory created successfully")
64 
+
65 
+	// 7. 获取Doris工厂实例
66 
+	dorisFactory, err := doris.GetDorisFactory(httpFactory)
67 
+	if err != nil {
68 
+		t.Fatalf("Failed to get Doris factory: %v", err)
69 
+	}
70 
+	fmt.Println("Doris factory created successfully")
71 
+
72 
+	// 检查Doris表结构
73 
+	fmt.Println("Checking Doris table structure...")
74 
+
75 
+	// 获取Doris配置
76 
+	cfg := config.GetConfig()
77 
+	database := "X6_STOCK_DEV"
78 
+	table := "A3_CLOTHING_LOG"
79 
+	skipHeader := false // 改为true，跳过CSV头行
80 
+	url := fmt.Sprintf("http://%s:%d/api/%s/%s/_stream_load", cfg.GetDoris().FEHost, cfg.GetDoris().FEPort, database, table)
81 
+	fmt.Printf("Doris stream load URL: %s\n", url)
82 
+
83 
+	// 计算需要的页数
84 
+	totalPages := (pageConfig.TotalRows + pageConfig.PageSize - 1) / pageConfig.PageSize
85 
+	fmt.Printf("预计总共需要查询 %d 页\n", totalPages)
86 
+
87 
+	// 创建任务通道和结果通道
88 
+	taskChan := make(chan QueryTask, totalPages)
89 
+	resultChan := make(chan QueryTask, totalPages)
90 
+	doneChan := make(chan bool)
91 
+
92 
+	var wg sync.WaitGroup
93 
+	var mu sync.Mutex
94 
+
95 
+	// 执行统计变量
96 
+	totalQueryTime := time.Duration(0)
97 
+	totalSaveTime := time.Duration(0)
98 
+	totalRowsInserted := 0
99 
+	completedTasks := 0
100 
+
101 
+	// 启动工作线程
102 
+	for i := 0; i < pageConfig.MaxWorkers; i++ {
103 
+		wg.Add(1)
104 
+		go func(workerID int) {
105 
+			defer wg.Done()
106 
+
107 
+			// 每个worker创建自己的HTTP客户端
108 
+			httpClient := httpFactory.CreateClient()
109 
+
110 
+			for task := range taskChan {
111 
+				fmt.Printf("Worker %d 处理第 %d 页 (行 %d-%d, CLOTHING_ID > %s)...\n",
112 
+					workerID, task.Page, task.StartRow, task.EndRow)
113 
+
114 
+				// 记录查询开始时间
115 
+				queryStartTime := time.Now()
116 
+
117 
+				// 准备查询请求
118 
+				queryRequest := types.QueryRequest{
119 
+					SQL:              task.QuerySQL,
120 
+					PositionalParams: task.QueryParams,
121 
+					WriterHeader:     false,
122 
+				}
123 
+
124 
+				// 发送POST请求到 /api/query/csv 获取CSV格式数据
125 
+				resp, err := httpClient.PostWithAuth(
126 
+					"http://localhost:8080/api/query/csv/param",
127 
+					queryRequest,
128 
+					"123", // Bearer Token
129 
+					nil,
130 
+				)
131 
+
132 
+				if err != nil {
133 
+					task.Error = fmt.Errorf("第%d页查询失败: %v", task.Page, err)
134 
+					resultChan <- task
135 
+					continue
136 
+				}
137 
+
138 
+				if resp.StatusCode() != 200 {
139 
+					task.Error = fmt.Errorf("第%d页查询请求失败, 状态码: %d", task.Page, resp.StatusCode())
140 
+					resultChan <- task
141 
+					continue
142 
+				}
143 
+
144 
+				// 获取CSV数据
145 
+				csvData := string(resp.Body())
146 
+				if len(csvData) == 0 {
147 
+					task.Error = fmt.Errorf("第%d页没有数据", task.Page)
148 
+					resultChan <- task
149 
+					continue
150 
+				}
151 
+				log.Printf("csvData:\n%s", csvData)
152 
+				// 记录查询结束时间
153 
+				queryEndTime := time.Now()
154 
+				queryDuration := queryEndTime.Sub(queryStartTime)
155 
+				task.QueryTime = queryDuration
156 
+				task.CSVData = csvData
157 
+
158 
+				// 插入数据到Doris
159 
+				saveStartTime := time.Now()
160 
+				err = dorisFactory.InsertCSV(database, table, csvData, skipHeader)
161 
+				if err != nil {
162 
+					task.Error = fmt.Errorf("第%d页数据插入Doris失败: %v", task.Page, err)
163 
+					resultChan <- task
164 
+					continue
165 
+				}
166 
+
167 
+				// 记录保存结束时间
168 
+				saveEndTime := time.Now()
169 
+				saveDuration := saveEndTime.Sub(saveStartTime)
170 
+				task.SaveTime = saveDuration
171 
+
172 
+				resultChan <- task
173 
+			}
174 
+		}(i + 1)
175 
+	}
176 
+
177 
+	// 启动结果处理协程
178 
+	go func() {
179 
+		for task := range resultChan {
180 
+			mu.Lock()
181 
+
182 
+			if task.Error != nil {
183 
+				fmt.Printf("❌ 第 %d 页处理失败: %v\n", task.Page, task.Error)
184 
+			} else {
185 
+				// 计算本页数据行数
186 
+				estimatedRows := task.EndRow - task.StartRow + 1
187 
+				// 如果数据不足，按实际估算
188 
+				if len(task.CSVData) < estimatedRows*50 { // 保守估计每行至少50字符
189 
+					estimatedRows = len(task.CSVData) / 50
190 
+				}
191 
+
192 
+				totalQueryTime += task.QueryTime
193 
+				totalSaveTime += task.SaveTime
194 
+				totalRowsInserted += estimatedRows
195 
+				completedTasks++
196 
+
197 
+				fmt.Printf("✅ Worker 完成第 %d 页 (行 %d-%d, CLOTHING_ID > %s)\n",
198 
+					task.Page, task.StartRow, task.EndRow)
199 
+				fmt.Printf("   查询耗时: %v, 保存耗时: %v\n", task.QueryTime, task.SaveTime)
200 
+				fmt.Printf("   估算数据行数: %d\n", estimatedRows)
201 
+
202 
+			}
203 
+
204 
+			mu.Unlock()
205 
+
206 
+			// 如果所有任务都完成了，发送完成信号
207 
+			if completedTasks >= totalPages {
208 
+				doneChan <- true
209 
+				break
210 
+			}
211 
+		}
212 
+	}()
213 
+
214 
+	// 生成任务并发送到任务通道
215 
+	fmt.Println("\n📋 开始生成查询任务...")
216 
+
217 
+	for page := 1; page <= totalPages; page++ {
218 
+		startRow := pageConfig.StartRow + (page-1)*pageConfig.PageSize + 1
219 
+		endRow := pageConfig.StartRow + page*pageConfig.PageSize
220 
+
221 
+		// 检查不超过要查询的总行数
222 
+		maxEndRow := pageConfig.StartRow + pageConfig.TotalRows
223 
+		if endRow > maxEndRow {
224 
+			endRow = maxEndRow
225 
+		}
226 
+
227 
+		// 生成查询SQL（使用参数模式）
228 
+		querySQL, queryParams := getSQLWithPagination(startRow, endRow)
229 
+
230 
+		task := QueryTask{
231 
+			Page:        page,
232 
+			StartRow:    startRow,
233 
+			EndRow:      endRow,
234 
+			QuerySQL:    querySQL,
235 
+			QueryParams: queryParams,
236 
+		}
237 
+
238 
+		fmt.Printf("生成第 %d 页任务 (行 %d-%d), 使用CLOTHING_ID > '%s'\n", page, startRow, endRow)
239 
+		taskChan <- task
240 
+
241 
+	}
242 
+
243 
+	// 关闭任务通道，通知worker没有更多任务
244 
+	close(taskChan)
245 
+
246 
+	// 等待所有worker完成
247 
+	wg.Wait()
248 
+
249 
+	// 关闭结果通道
250 
+	close(resultChan)
251 
+
252 
+	// 等待结果处理完成
253 
+	<-doneChan
254 
+
255 
+	// 记录总结束时间
256 
+	totalEndTime := time.Now()
257 
+	totalDuration := totalEndTime.Sub(totalStartTime)
258 
+
259 
+	// 打印性能统计
260 
+	fmt.Println("\n📊 性能统计:")
261 
+	fmt.Printf("   完成页数: %d/%d\n", completedTasks, totalPages)
262 
+	fmt.Printf("   总查询耗时: %v\n", totalQueryTime)
263 
+	fmt.Printf("   总保存耗时: %v\n", totalSaveTime)
264 
+	fmt.Printf("   总耗时: %v\n", totalDuration)
265 
+	fmt.Printf("   估算插入总行数: %d\n", totalRowsInserted)
266 
+	if completedTasks > 0 {
267 
+		fmt.Printf("   平均每页查询耗时: %v\n", totalQueryTime/time.Duration(completedTasks))
268 
+		fmt.Printf("   平均每页保存耗时: %v\n", totalSaveTime/time.Duration(completedTasks))
269 
+		fmt.Printf("   平均每秒处理行数: %.2f\n", float64(totalRowsInserted)/totalDuration.Seconds())
270 
+	}
271 
+
272 
+	fmt.Println("✅ 所有数据成功插入到 Doris!")
273 
+}
274 
+
275 
+// getSQLWithPagination 生成带分页的SQL语句（参数模式）
276 
+// 返回SQL语句和参数映射
277 
+func getSQLWithPagination1(startRow, endRow int) (string, []interface{}) {
278 
+	sql := `SELECT
279 
+    CLOTHING_ID,
280 
+    CLOTHING_YEAR,
281 
+    CLOTHING_NAME
282 
+
283 
+   FROM (
284 
+    SELECT a.*, ROWNUM as rn
285 
+    FROM (
286 
+        SELECT *
287 
+        FROM X6_STOCK_DEV.A3_CLOTHING
288 
+        WHERE CLOTHING_ID > :1
289 
+        ORDER BY CLOTHING_ID
290 
+    ) a
291 
+    WHERE ROWNUM <= :2
292 
+)
293 
+WHERE rn > :3`
294 
+
295 
+	// 创建参数映射
296 
+	params := []interface{}{
297 
+		endRow,
298 
+		startRow - 1, // WHERE rn > :start_row 所以是startRow-1
299 
+	}
300 
+
301 
+	return sql, params
302 
+}

+ 112
- 0
test/my1_test.go View File 

@@ -0,0 +1,112 @@
1 
+package main
2 
+
3 
+import (
4 
+	"encoding/json"
5 
+	"fmt"
6 
+	"io"
7 
+	"net/http"
8 
+	"strings"
9 
+	"testing"
10 
+	"time"
11 
+)
12 
+
13 
+// Doris Stream Load响应结构
14 
+type DorisResponse struct {
15 
+	TxnID            int    `json:"TxnId"`
16 
+	Label            string `json:"Label"`
17 
+	Status           string `json:"Status"`
18 
+	Message          string `json:"Message"`
19 
+	NumberTotalRows  int    `json:"NumberTotalRows"`
20 
+	NumberLoadedRows int    `json:"NumberLoadedRows"`
21 
+	LoadBytes        int    `json:"LoadBytes"`
22 
+	ErrorURL         string `json:"ErrorURL"`
23 
+}
24 
+
25 
+func insertToDorisDirect(csvData string) error {
26 
+	url := "http://161.189.89.196:8040/api/X6_STOCK_DEV/A3_CLOTHING_LOG/_stream_load"
27 
+	username := "root"
28 
+	password := "mos8555"
29 
+
30 
+	fmt.Printf("=== Doris Stream Load测试 ===\n")
31 
+	fmt.Printf("URL: %s\n", url)
32 
+	fmt.Printf("数据: %s\n", csvData)
33 
+
34 
+	// 创建PUT请求
35 
+	req, err := http.NewRequest("PUT", url, strings.NewReader(csvData))
36 
+	if err != nil {
37 
+		return fmt.Errorf("创建请求失败: %v", err)
38 
+	}
39 
+
40 
+	// 设置Basic Auth
41 
+	req.SetBasicAuth(username, password)
42 
+
43 
+	// 设置headers - 与curl命令保持一致
44 
+	req.Header.Set("Expect", "100-continue")
45 
+	req.Header.Set("column_separator", ",")
46 
+	req.Header.Set("enclose", "\"")
47 
+
48 
+	// 设置唯一的label用于追踪
49 
+	label := fmt.Sprintf("test_%d", time.Now().UnixNano())
50 
+	req.Header.Set("label", label)
51 
+
52 
+	// 创建HTTP客户端，允许自动重定向
53 
+	client := &http.Client{
54 
+		Timeout: 30 * time.Second,
55 
+	}
56 
+
57 
+	// 发送请求
58 
+	fmt.Println("发送请求到Doris...")
59 
+	resp, err := client.Do(req)
60 
+	if err != nil {
61 
+		return fmt.Errorf("请求失败: %v", err)
62 
+	}
63 
+	defer resp.Body.Close()
64 
+
65 
+	// 处理响应
66 
+	body, err := io.ReadAll(resp.Body)
67 
+	if err != nil {
68 
+		return fmt.Errorf("读取响应失败: %v", err)
69 
+	}
70 
+
71 
+	responseBody := string(body)
72 
+	fmt.Printf("HTTP状态码: %d\n", resp.StatusCode)
73 
+	fmt.Printf("响应内容: %s\n", responseBody)
74 
+
75 
+	// 解析JSON响应
76 
+	var dorisResp DorisResponse
77 
+	if err := json.Unmarshal(body, &dorisResp); err != nil {
78 
+		return fmt.Errorf("解析响应失败: %v, 原始响应: %s", err, responseBody)
79 
+	}
80 
+
81 
+	// 检查处理状态 - 修正状态检查逻辑
82 
+	if dorisResp.Status == "FAIL" {
83 
+		return fmt.Errorf("Stream Load失败: %s (TxnID: %d)", dorisResp.Message, dorisResp.TxnID)
84 
+	}
85 
+
86 
+	// 修正：Doris返回的状态可能是 "Success" 而不是 "SUCCESS"
87 
+	if strings.ToUpper(dorisResp.Status) != "SUCCESS" {
88 
+		return fmt.Errorf("未知状态: %s, 消息: %s", dorisResp.Status, dorisResp.Message)
89 
+	}
90 
+
91 
+	fmt.Printf("✅ Stream Load成功! \n")
92 
+	fmt.Printf("   TxnID: %d\n", dorisResp.TxnID)
93 
+	fmt.Printf("   Label: %s\n", dorisResp.Label)
94 
+	fmt.Printf("   加载行数: %d/%d\n", dorisResp.NumberLoadedRows, dorisResp.NumberTotalRows)
95 
+	fmt.Printf("   数据大小: %d bytes\n", dorisResp.LoadBytes)
96 
+
97 
+	return nil
98 
+}
99 
+
100 
+// 测试函数
101 
+func TestDorisStreamLoad(t *testing.T) {
102 
+	fmt.Println("=== 测试Doris Stream Load ===")
103 
+
104 
+	csvData := "EWE322Y20600491-03,2021,H-大部分"
105 
+
106 
+	err := insertToDorisDirect(csvData)
107 
+	if err != nil {
108 
+		t.Fatalf("Stream Load失败: %v", err)
109 
+	}
110 
+
111 
+	fmt.Println("✅ 数据成功插入Doris!")
112 
+}

+ 92
- 0
test/my2_test.go View File 

@@ -0,0 +1,92 @@
1 
+package main
2 
+
3 
+import (
4 
+	"database/sql"
5 
+	"fmt"
6 
+	"testing"
7 
+
8 
+	_ "github.com/go-sql-driver/mysql"
9 
+)
10 
+
11 
+// 使用MySQL协议插入数据
12 
+func insertViaMySQL() error {
13 
+	// Doris MySQL连接信息
14 
+	dsn := "root:mos8555@tcp(161.189.89.196:9030)/X6_STOCK_DEV"
15 
+
16 
+	db, err := sql.Open("mysql", dsn)
17 
+	if err != nil {
18 
+		return fmt.Errorf("连接数据库失败: %v", err)
19 
+	}
20 
+	defer db.Close()
21 
+
22 
+	// 测试连接
23 
+	err = db.Ping()
24 
+	if err != nil {
25 
+		return fmt.Errorf("数据库连接测试失败: %v", err)
26 
+	}
27 
+
28 
+	fmt.Println("✅ 成功连接到Doris MySQL")
29 
+
30 
+	// 插入数据
31 
+	query := "INSERT INTO A3_CLOTHING (col1, col2, col3) VALUES (?, ?, ?)"
32 
+	result, err := db.Exec(query, "EWE322Y2060049145", 2021, "Y-连衣裙")
33 
+	if err != nil {
34 
+		return fmt.Errorf("插入数据失败: %v", err)
35 
+	}
36 
+
37 
+	rowsAffected, err := result.RowsAffected()
38 
+	if err != nil {
39 
+		return fmt.Errorf("获取影响行数失败: %v", err)
40 
+	}
41 
+
42 
+	fmt.Printf("✅ 数据插入成功，影响行数: %d\n", rowsAffected)
43 
+	return nil
44 
+}
45 
+
46 
+// 先检查表结构
47 
+func checkTableSchema() error {
48 
+	dsn := "root:mos8555@tcp(161.189.89.196:9030)/X6_STOCK_DEV"
49 
+	db, err := sql.Open("mysql", dsn)
50 
+	if err != nil {
51 
+		return err
52 
+	}
53 
+	defer db.Close()
54 
+
55 
+	// 查询表结构
56 
+	rows, err := db.Query("DESC A3_CLOTHING")
57 
+	if err != nil {
58 
+		return fmt.Errorf("查询表结构失败: %v", err)
59 
+	}
60 
+	defer rows.Close()
61 
+
62 
+	fmt.Println("=== A3_CLOTHING 表结构 ===")
63 
+	for rows.Next() {
64 
+		var field, fieldType, isNull, key, defaultValue, extra string
65 
+		err := rows.Scan(&field, &fieldType, &isNull, &key, &defaultValue, &extra)
66 
+		if err != nil {
67 
+			return err
68 
+		}
69 
+		fmt.Printf("字段: %s, 类型: %s, 允许空: %s\n", field, fieldType, isNull)
70 
+	}
71 
+
72 
+	return nil
73 
+}
74 
+
75 
+// 测试函数
76 
+func TestMySQLInsert(t *testing.T) {
77 
+	fmt.Println("=== 使用MySQL协议插入数据 ===")
78 
+
79 
+	// 先检查表结构
80 
+	err := checkTableSchema()
81 
+	if err != nil {
82 
+		t.Fatalf("检查表结构失败: %v", err)
83 
+	}
84 
+
85 
+	// 插入数据
86 
+	err = insertViaMySQL()
87 
+	if err != nil {
88 
+		t.Fatalf("MySQL插入失败: %v", err)
89 
+	}
90 
+
91 
+	fmt.Println("✅ 通过MySQL协议插入数据成功!")
92 
+}

+ 394
- 0
test/my_ora_clothingToDoris_test.go View File 

@@ -0,0 +1,394 @@
1 
+package main
2 
+
3 
+import (
4 
+	"fmt"
5 
+	"log"
6 
+	"sync"
7 
+	"testing"
8 
+	"time"
9 
+
10 
+	"git.x2erp.com/qdy/go-base/config"
11 
+	"git.x2erp.com/qdy/go-base/types"
12 
+	"git.x2erp.com/qdy/go-db/factory/doris"
13 
+	"git.x2erp.com/qdy/go-db/factory/http"
14 
+)
15 
+
16 
+// 一定要执行环境变量
17 
+// export DB_CONFIG_PATH=/Users/kenqdy/Documents/v-bdx-workspace/db_doris.yaml
18 
+
19 
+// // QueryRequest 查询请求结构体
20 
+// type QueryRequest struct {
21 
+// 	SQL              string                 `json:"sql" binding:"required"`
22 
+// 	Params           map[string]interface{} `json:"params,omitempty"`           // 名称参数
23 
+// 	PositionalParams []interface{}          `json:"positionalParams,omitempty"` // 位置参数
24 
+// }
25 
+
26 
+// 分页配置
27 
+type PageConfig struct {
28 
+	TotalRows  int // 总共需要查询多少行
29 
+	PageSize   int // 每次返回多少行
30 
+	MaxWorkers int // 几条线程同时工作
31 
+	StartRow   int //从哪行开始查询
32 
+}
33 
+
34 
+// 查询任务
35 
+type QueryTask struct {
36 
+	Page        int
37 
+	StartRow    int
38 
+	EndRow      int
39 
+	QuerySQL    string
40 
+	QueryParams []interface{}
41 
+	CSVData     string
42 
+	Error       error
43 
+	QueryTime   time.Duration
44 
+	SaveTime    time.Duration
45 
+}
46 
+
47 
+func TestQueryAndInsertToDoris(t *testing.T) {
48 
+	// 记录总开始时间
49 
+	totalStartTime := time.Now()
50 
+
51 
+	// 配置分页参数
52 
+	pageConfig := PageConfig{
53 
+		TotalRows:  400000, // 总共需要查询多少行
54 
+		PageSize:   3000,   // 每次查询2000条
55 
+		MaxWorkers: 10,     // 几条线程同时工作
56 
+		StartRow:   0,      //从哪行开始查询
57 
+	}
58 
+
59 
+	fmt.Printf("开始执行分页查询，总共%d行，每页%d条，%d线程工作\n", pageConfig.TotalRows, pageConfig.PageSize, pageConfig.MaxWorkers)
60 
+
61 
+	// 1. 获取HTTP工厂实例
62 
+	httpFactory, err := http.GetHTTPFactory()
63 
+	if err != nil {
64 
+		t.Fatalf("Failed to get HTTP factory: %v", err)
65 
+	}
66 
+	fmt.Println("HTTP factory created successfully")
67 
+
68 
+	// 7. 获取Doris工厂实例
69 
+	dorisFactory, err := doris.GetDorisFactory(httpFactory)
70 
+	if err != nil {
71 
+		t.Fatalf("Failed to get Doris factory: %v", err)
72 
+	}
73 
+	fmt.Println("Doris factory created successfully")
74 
+
75 
+	// 检查Doris表结构
76 
+	fmt.Println("Checking Doris table structure...")
77 
+
78 
+	// 获取Doris配置
79 
+	cfg := config.GetConfig()
80 
+	database := "X6_STOCK_DEV"
81 
+	table := "A3_CLOTHING"
82 
+	skipHeader := false // 改为true，跳过CSV头行
83 
+	url := fmt.Sprintf("http://%s:%d/api/%s/%s/_stream_load", cfg.GetDoris().FEHost, cfg.GetDoris().FEPort, database, table)
84 
+	fmt.Printf("Doris stream load URL: %s\n", url)
85 
+
86 
+	// 计算需要的页数
87 
+	totalPages := (pageConfig.TotalRows + pageConfig.PageSize - 1) / pageConfig.PageSize
88 
+	fmt.Printf("预计总共需要查询 %d 页\n", totalPages)
89 
+
90 
+	// 创建任务通道和结果通道
91 
+	taskChan := make(chan QueryTask, totalPages)
92 
+	resultChan := make(chan QueryTask, totalPages)
93 
+	doneChan := make(chan bool)
94 
+
95 
+	var wg sync.WaitGroup
96 
+	var mu sync.Mutex
97 
+
98 
+	// 执行统计变量
99 
+	totalQueryTime := time.Duration(0)
100 
+	totalSaveTime := time.Duration(0)
101 
+	totalRowsInserted := 0
102 
+	completedTasks := 0
103 
+
104 
+	// 启动工作线程
105 
+	for i := 0; i < pageConfig.MaxWorkers; i++ {
106 
+		wg.Add(1)
107 
+		go func(workerID int) {
108 
+			defer wg.Done()
109 
+
110 
+			// 每个worker创建自己的HTTP客户端
111 
+			httpClient := httpFactory.CreateClient()
112 
+
113 
+			for task := range taskChan {
114 
+				fmt.Printf("Worker %d 处理第 %d 页 (行 %d-%d, CLOTHING_ID > %s)...\n",
115 
+					workerID, task.Page, task.StartRow, task.EndRow)
116 
+
117 
+				// 记录查询开始时间
118 
+				queryStartTime := time.Now()
119 
+
120 
+				// 准备查询请求
121 
+				queryRequest := types.QueryRequest{
122 
+					SQL:              task.QuerySQL,
123 
+					PositionalParams: task.QueryParams,
124 
+					WriterHeader:     false,
125 
+				}
126 
+
127 
+				// 发送POST请求到 /api/query/csv 获取CSV格式数据
128 
+				resp, err := httpClient.PostWithAuth(
129 
+					"http://localhost:8080/api/query/csv/param",
130 
+					queryRequest,
131 
+					"123", // Bearer Token
132 
+					nil,
133 
+				)
134 
+
135 
+				if err != nil {
136 
+					task.Error = fmt.Errorf("第%d页查询失败: %v", task.Page, err)
137 
+					resultChan <- task
138 
+					continue
139 
+				}
140 
+
141 
+				if resp.StatusCode() != 200 {
142 
+					task.Error = fmt.Errorf("第%d页查询请求失败, 状态码: %d", task.Page, resp.StatusCode())
143 
+					resultChan <- task
144 
+					continue
145 
+				}
146 
+
147 
+				// 获取CSV数据
148 
+				csvData := string(resp.Body())
149 
+				if len(csvData) == 0 {
150 
+					task.Error = fmt.Errorf("第%d页没有数据", task.Page)
151 
+					resultChan <- task
152 
+					continue
153 
+				}
154 
+				log.Printf("csvData:\n%s", csvData)
155 
+				// 记录查询结束时间
156 
+				queryEndTime := time.Now()
157 
+				queryDuration := queryEndTime.Sub(queryStartTime)
158 
+				task.QueryTime = queryDuration
159 
+				task.CSVData = csvData
160 
+
161 
+				// 插入数据到Doris
162 
+				saveStartTime := time.Now()
163 
+				err = dorisFactory.InsertCSV(database, table, csvData, skipHeader)
164 
+				if err != nil {
165 
+					task.Error = fmt.Errorf("第%d页数据插入Doris失败: %v", task.Page, err)
166 
+					resultChan <- task
167 
+					continue
168 
+				}
169 
+
170 
+				// 记录保存结束时间
171 
+				saveEndTime := time.Now()
172 
+				saveDuration := saveEndTime.Sub(saveStartTime)
173 
+				task.SaveTime = saveDuration
174 
+
175 
+				resultChan <- task
176 
+			}
177 
+		}(i + 1)
178 
+	}
179 
+
180 
+	// 启动结果处理协程
181 
+	go func() {
182 
+		for task := range resultChan {
183 
+			mu.Lock()
184 
+
185 
+			if task.Error != nil {
186 
+				fmt.Printf("❌ 第 %d 页处理失败: %v\n", task.Page, task.Error)
187 
+			} else {
188 
+				// 计算本页数据行数
189 
+				estimatedRows := task.EndRow - task.StartRow + 1
190 
+				// 如果数据不足，按实际估算
191 
+				if len(task.CSVData) < estimatedRows*50 { // 保守估计每行至少50字符
192 
+					estimatedRows = len(task.CSVData) / 50
193 
+				}
194 
+
195 
+				totalQueryTime += task.QueryTime
196 
+				totalSaveTime += task.SaveTime
197 
+				totalRowsInserted += estimatedRows
198 
+				completedTasks++
199 
+
200 
+				fmt.Printf("✅ Worker 完成第 %d 页 (行 %d-%d, CLOTHING_ID > %s)\n",
201 
+					task.Page, task.StartRow, task.EndRow)
202 
+				fmt.Printf("   查询耗时: %v, 保存耗时: %v\n", task.QueryTime, task.SaveTime)
203 
+				fmt.Printf("   估算数据行数: %d\n", estimatedRows)
204 
+
205 
+			}
206 
+
207 
+			mu.Unlock()
208 
+
209 
+			// 如果所有任务都完成了，发送完成信号
210 
+			if completedTasks >= totalPages {
211 
+				doneChan <- true
212 
+				break
213 
+			}
214 
+		}
215 
+	}()
216 
+
217 
+	// 生成任务并发送到任务通道
218 
+	fmt.Println("\n📋 开始生成查询任务...")
219 
+
220 
+	for page := 1; page <= totalPages; page++ {
221 
+		startRow := pageConfig.StartRow + (page-1)*pageConfig.PageSize + 1
222 
+		endRow := pageConfig.StartRow + page*pageConfig.PageSize
223 
+
224 
+		// 检查不超过要查询的总行数
225 
+		maxEndRow := pageConfig.StartRow + pageConfig.TotalRows
226 
+		if endRow > maxEndRow {
227 
+			endRow = maxEndRow
228 
+		}
229 
+
230 
+		// 生成查询SQL（使用参数模式）
231 
+		querySQL, queryParams := getSQLWithPagination(startRow, endRow)
232 
+
233 
+		task := QueryTask{
234 
+			Page:        page,
235 
+			StartRow:    startRow,
236 
+			EndRow:      endRow,
237 
+			QuerySQL:    querySQL,
238 
+			QueryParams: queryParams,
239 
+		}
240 
+
241 
+		fmt.Printf("生成第 %d 页任务 (行 %d-%d), 使用CLOTHING_ID > '%s'\n", page, startRow, endRow)
242 
+
243 
+		taskChan <- task
244 
+
245 
+	}
246 
+
247 
+	// 关闭任务通道，通知worker没有更多任务
248 
+	close(taskChan)
249 
+
250 
+	// 等待所有worker完成
251 
+	wg.Wait()
252 
+
253 
+	// 关闭结果通道
254 
+	close(resultChan)
255 
+
256 
+	// 等待结果处理完成
257 
+	<-doneChan
258 
+
259 
+	// 记录总结束时间
260 
+	totalEndTime := time.Now()
261 
+	totalDuration := totalEndTime.Sub(totalStartTime)
262 
+
263 
+	// 打印性能统计
264 
+	fmt.Println("\n📊 性能统计:")
265 
+	fmt.Printf("   完成页数: %d/%d\n", completedTasks, totalPages)
266 
+	fmt.Printf("   总查询耗时: %v\n", totalQueryTime)
267 
+	fmt.Printf("   总保存耗时: %v\n", totalSaveTime)
268 
+	fmt.Printf("   总耗时: %v\n", totalDuration)
269 
+	fmt.Printf("   估算插入总行数: %d\n", totalRowsInserted)
270 
+	if completedTasks > 0 {
271 
+		fmt.Printf("   平均每页查询耗时: %v\n", totalQueryTime/time.Duration(completedTasks))
272 
+		fmt.Printf("   平均每页保存耗时: %v\n", totalSaveTime/time.Duration(completedTasks))
273 
+		fmt.Printf("   平均每秒处理行数: %.2f\n", float64(totalRowsInserted)/totalDuration.Seconds())
274 
+	}
275 
+
276 
+	fmt.Println("✅ 所有数据成功插入到 Doris!")
277 
+}
278 
+
279 
+// getSQLWithPagination 生成带分页的SQL语句（参数模式）
280 
+// 返回SQL语句和参数映射
281 
+func getSQLWithPagination(startRow, endRow int) (string, []interface{}) {
282 
+	sql := `SELECT
283 
+    CLOTHING_ID,
284 
+    CLOTHING_YEAR,
285 
+    CLOTHING_NAME,
286 
+    STYLECOLOR_ID,
287 
+    STYLE_ID,
288 
+    COLOR_ID,
289 
+    SIZE_ID,
290 
+    CREATE_DATE,
291 
+    STYLE_GROUP,
292 
+    J_PRICE,
293 
+    X_PRICE,
294 
+    V_PRICE,
295 
+    CLERK_ROYALTYRATE,
296 
+    CLERK_ROYALTYPRICE,
297 
+    BRAND_CODE,
298 
+    STYLEVER_ID,
299 
+    J_COST,
300 
+    CLOTHING_IMG,
301 
+    STYLE_UNIT_CODE,
302 
+    STYLE_SEX_CODE,
303 
+    STYLE_KIND_CODE,
304 
+    STYLE_CLASS_CODE,
305 
+    STYLE_SUBCLASS_CODE,
306 
+    STYLE_DESIGNER_CODE,
307 
+    STYLE_PLATER_CODE,
308 
+    STYLE_STYLES_CODE,
309 
+    STYLE_LOCATE_CODE,
310 
+    STYLE_SALETYPE_CODE,
311 
+    STYLE_COLORSYSTEM_CODE,
312 
+    STYLE_THEME_CODE,
313 
+    STYLE_INDENTTYPE_CODE,
314 
+    STYLE_PRICEBAND_CODE,
315 
+    STYLE_MONTH_CODE,
316 
+    STYLE_COMPOSITION_CODE,
317 
+    STYLE_SUPPLIER_CODE,
318 
+    STYLE_SPARE1_CODE,
319 
+    STYLE_SPARE2_CODE,
320 
+    STYLE_SPARE4_CODE,
321 
+    STYLE_SPARE5_CODE,
322 
+    CATEGORY_CODE,
323 
+    BRAND_ID,
324 
+    STYCOLVER_ID,
325 
+    STYLE_SAME,
326 
+    CLOTHING_BARCODE,
327 
+    CLOTHING_HELPID,
328 
+    CLOTHING_GBCODE,
329 
+    CLOTHING_RFID,
330 
+    STYLE_SUBJECT_ID,
331 
+    SIZEGRP_ID,
332 
+    STYLE_HELPID,
333 
+    CLOTHING_GBCODE1,
334 
+    COLOR_NAME,
335 
+    STYLEVER_NAME,
336 
+    SIZE_NAME,
337 
+    STYLE_UNIT,
338 
+    STYLE_SEX,
339 
+    STYLE_KIND,
340 
+    STYLE_CLASS,
341 
+    STYLE_SUBCLASS,
342 
+    STYLE_DESIGNER,
343 
+    STYLE_PLATER,
344 
+    STYLE_BAND,
345 
+    STYLE_STYLES,
346 
+    STYLE_LOCATE,
347 
+    STYLE_SALETYPE,
348 
+    STYLE_COLORSYSTEM,
349 
+    STYLE_THEME,
350 
+    STYLE_INDENTTYPE,
351 
+    STYLE_PRICEBAND,
352 
+    STYLE_MONTH,
353 
+    STYLE_COMPOSITION,
354 
+    STYLE_SUPPLIER,
355 
+    STYLE_SPARE1,
356 
+    STYLE_SPARE2,
357 
+    STYLE_SPARE3,
358 
+    STYLE_SPARE4,
359 
+    STYLE_SPARE5,
360 
+    CATEGORY_NAME,
361 
+    BRAND_NAME,
362 
+    STYLE_YEAR_NAME,
363 
+    STYLE_SEARCH_KEY,
364 
+    STYLE_SUBJECT_NAME,
365 
+    CLOTHING_REMARK,
366 
+    STYLE_SPARE3_CODE,
367 
+    COST,
368 
+    BRAND_GROUPCODE,
369 
+    CLASS_GROUPCODE,
370 
+    MONTH_GROUPCODE,
371 
+    RETURNSUBJECT_ID,
372 
+    PRODUCT_SORT,
373 
+    CLOTHING_PARTITION
374 
+   FROM (
375 
+    SELECT a.*, ROWNUM as rn
376 
+    FROM (
377 
+        SELECT *
378 
+        FROM X6_STOCK_DEV.A3_CLOTHING
379 
+
380 
+        ORDER BY CLOTHING_ID
381 
+    ) a
382 
+    WHERE ROWNUM <= :1
383 
+)
384 
+WHERE rn > :2`
385 
+
386 
+	// 创建参数映射
387 
+	params := []interface{}{
388 
+		//lastClothingID,
389 
+		endRow,
390 
+		startRow - 1, // WHERE rn > :start_row 所以是startRow-1
391 
+	}
392 
+
393 
+	return sql, params
394 
+}

+ 69
- 0
test/mycsv_test.go View File 

@@ -0,0 +1,69 @@
1 
+package main
2 
+
3 
+import (
4 
+	"encoding/json"
5 
+	"log"
6 
+	"testing"
7 
+
8 
+	"git.x2erp.com/qdy/go-db/factory/database"
9 
+)
10 
+
11 
+func TestNamedParamsQueryCSV(t *testing.T) {
12 
+	factory, err := database.GetDBFactory()
13 
+	if err != nil {
14 
+		t.Fatalf("Failed to get DB factory: %v", err)
15 
+	}
16 
+	defer factory.Close()
17 
+
18 
+	// 简化的SQL，只测试3个参数
19 
+	sql := `
20 
+		SELECT * FROM (
21 
+			SELECT a.*, ROWNUM rn FROM (
22 
+				SELECT CLOTHING_ID, CLOTHING_NAME
23 
+				FROM X6_STOCK_DEV.A3_CLOTHING
24 
+				where clothing_id>:1
25 
+				ORDER BY CLOTHING_ID
26 
+			) a WHERE ROWNUM <= :2
27 
+		) WHERE rn > :3
28 
+	`
29 
+
30 
+	// // 3个参数
31 
+	params := []interface{}{
32 
+		"A",
33 
+		10,
34 
+		0,
35 
+	}
36 
+
37 
+	// // 3个参数
38 
+	// params := map[string]interface{}{
39 
+
40 
+	// 	"end_row":   10,
41 
+	// 	"start_row": 0,
42 
+	// }
43 
+	// // 3个参数
44 
+	// params := map[string]interface{}{
45 
+	// 	"clothing_id": "`0`",
46 
+	// 	"end_row":     10,
47 
+	// 	"start_row":   0,
48 
+	// }
49 
+
50 
+	// 执行查询
51 
+	result := factory.QueryPositionalToJSON(sql, params)
52 
+
53 
+	// 检查结果 - 根据你的错误信息，result.Error 是 string 类型
54 
+	if err != nil { // 改成检查空字符串
55 
+		t.Errorf("Named parameters query failed: %s", err)
56 
+	} else {
57 
+		// 将请求参数输出为格式化的 JSON 日志
58 
+		reqJSON, err := json.MarshalIndent(result, "", "  ")
59 
+		if err != nil {
60 
+			log.Printf("无法序列化请求参数: %v", err)
61 
+		} else {
62 
+			log.Printf("QueryRequest 参数:\n%s", string(reqJSON))
63 
+		}
64 
+
65 
+		// 或者使用 fmt 输出
66 
+		//fmt.Printf("=== QueryRequest ===\n%s\n====================\n", string(reqJSON))
67 
+	}
68 
+
69 
+}

+ 97
- 0
test/mysql_test.go View File 

@@ -0,0 +1,97 @@
1 
+package main
2 
+
3 
+import (
4 
+	"encoding/json"
5 
+	"log"
6 
+	"testing"
7 
+
8 
+	"git.x2erp.com/qdy/go-db/factory/database"
9 
+)
10 
+
11 
+func TestNamedParamsQuery(t *testing.T) {
12 
+	factory, err := database.GetDBFactory()
13 
+	if err != nil {
14 
+		t.Fatalf("Failed to get DB factory: %v", err)
15 
+	}
16 
+	defer factory.Close()
17 
+
18 
+	// 简化的SQL，只测试3个参数
19 
+	sql := `
20 
+		SELECT * FROM (
21 
+			SELECT a.*, ROWNUM rn FROM (
22 
+				SELECT CLOTHING_ID, CLOTHING_NAME
23 
+				FROM X6_STOCK_DEV.A3_CLOTHING
24 
+
25 
+				ORDER BY CLOTHING_ID
26 
+			) a WHERE ROWNUM <= :1
27 
+		) WHERE rn > :2
28 
+	`
29 
+
30 
+	// // 3个参数
31 
+	params := []interface{}{
32 
+		10,
33 
+		0,
34 
+	}
35 
+
36 
+	db := factory.GetDB()
37 
+	rows, err := db.Query(sql, params...) // 获取行结果集
38 
+
39 
+	if err != nil {
40 
+		log.Fatalf("查询失败: %v", err)
41 
+	}
42 
+	defer rows.Close()
43 
+
44 
+	// 1. 获取列信息
45 
+	columns, err := rows.Columns()
46 
+	if err != nil {
47 
+		log.Fatalf("获取列失败: %v", err)
48 
+	}
49 
+
50 
+	// 2. 准备存储结果的切片
51 
+	var results []map[string]interface{}
52 
+
53 
+	// 3. 遍历每一行
54 
+	for rows.Next() {
55 
+		// 创建值的切片（每个列一个值）
56 
+		values := make([]interface{}, len(columns))
57 
+		valuePtrs := make([]interface{}, len(columns))
58 
+		for i := range values {
59 
+			valuePtrs[i] = &values[i]
60 
+		}
61 
+
62 
+		// 扫描行数据
63 
+		err := rows.Scan(valuePtrs...)
64 
+		if err != nil {
65 
+			log.Fatalf("扫描行失败: %v", err)
66 
+		}
67 
+
68 
+		// 将当前行转换为 map
69 
+		rowMap := make(map[string]interface{})
70 
+		for i, col := range columns {
71 
+			val := values[i]
72 
+
73 
+			// 处理特殊类型（如 []byte 转换为 string）
74 
+			if b, ok := val.([]byte); ok {
75 
+				rowMap[col] = string(b)
76 
+			} else {
77 
+				rowMap[col] = val
78 
+			}
79 
+		}
80 
+
81 
+		results = append(results, rowMap)
82 
+	}
83 
+
84 
+	// 检查遍历过程中的错误
85 
+	if err = rows.Err(); err != nil {
86 
+		log.Fatalf("遍历行错误: %v", err)
87 
+	}
88 
+
89 
+	// 4. 现在可以序列化为 JSON
90 
+	reqJSON1, err := json.MarshalIndent(results, "", "  ")
91 
+	if err != nil {
92 
+		log.Fatalf("JSON序列化失败: %v", err)
93 
+	}
94 
+
95 
+	log.Printf("查询结果:\n%s", string(reqJSON1))
96 
+	log.Printf("共 %d 行数据", len(results))
97 
+}

Write Preview
Loading…
Cancel
Save