package main import ( "fmt" "testing" "time" "git.x2erp.com/qdy/go-base/config" "git.x2erp.com/qdy/go-db/factory" ) // QueryRequest 查询请求结构体 type QueryRequest struct { SQL string `json:"sql"` } func TestQueryAndInsertToDoris(t *testing.T) { // 记录总开始时间 totalStartTime := time.Now() // 1. 获取HTTP工厂实例 httpFactory, err := factory.GetHTTPFactory() if err != nil { t.Fatalf("Failed to get HTTP factory: %v", err) } fmt.Println("HTTP factory created successfully") // 2. 创建HTTP客户端 httpClient := httpFactory.CreateClient() // 3. 准备查询SQL - 限制数据量用于测试 querySQL := getSQL() // 4. 准备查询请求 queryRequest := QueryRequest{ SQL: querySQL, } // 记录查询开始时间 queryStartTime := time.Now() // 5. 发送POST请求到 /api/query/csv 获取CSV格式数据 fmt.Println("Sending query request to localhost:8080/api/query/csv...") resp, err := httpClient.PostWithAuth( "http://localhost:8080/api/query/csv", queryRequest, "123", // Bearer Token nil, ) if err != nil { t.Fatalf("Failed to send query request: %v", err) } if resp.StatusCode() != 200 { t.Fatalf("Query request failed with status: %d, body: %s", resp.StatusCode(), string(resp.Body())) } // 6. 获取CSV数据 csvData := string(resp.Body()) if len(csvData) == 0 { t.Fatalf("No CSV data received") } // 记录查询结束时间 queryEndTime := time.Now() queryDuration := queryEndTime.Sub(queryStartTime) fmt.Printf("✅ Query completed in %v\n", queryDuration) fmt.Printf("Successfully retrieved CSV data, length: %d bytes\n", len(csvData)) //fmt.Printf("CSV data content:\n%s\n", csvData) // 7. 获取Doris工厂实例 dorisFactory, err := factory.GetDorisFactory(httpFactory) if err != nil { t.Fatalf("Failed to get Doris factory: %v", err) } fmt.Println("Doris factory created successfully") // 8. 先检查Doris表结构 fmt.Println("Checking Doris table structure...") // 9. 插入数据到Doris database := "X6_STOCK_DEV" table := "A3_CLOTHING" skipHeader := false // 改为true,跳过CSV头行 fmt.Printf("Inserting data to Doris database: %s, table: %s\n", database, table) //fmt.Printf("CSV data to insert:\n%q\n", csvData) cfg := config.GetConfig() url := fmt.Sprintf("http://%s:%d/api/%s/%s/_stream_load", cfg.GetDoris().FEHost, cfg.GetDoris().FEPort, database, table) fmt.Print(url + ".\n") // 记录保存开始时间 saveStartTime := time.Now() err = dorisFactory.InsertCSV(database, table, csvData, skipHeader) if err != nil { t.Fatalf("Failed to insert data to Doris: %v", err) } // 记录保存结束时间 saveEndTime := time.Now() saveDuration := saveEndTime.Sub(saveStartTime) fmt.Printf("✅ Data saved to Doris in %v\n", saveDuration) // 记录总结束时间 totalEndTime := time.Now() totalDuration := totalEndTime.Sub(totalStartTime) // 打印性能统计 fmt.Println("\n📊 Performance Statistics:") fmt.Printf(" Query Time: %v\n", queryDuration) fmt.Printf(" Save Time: %v\n", saveDuration) fmt.Printf(" Total Time: %v\n", totalDuration) fmt.Printf(" Data Size: %d bytes\n", len(csvData)) fmt.Println("Data successfully inserted to Doris!") } func getSQLm() string { return `SELECT CLOTHING_ID, CLOTHING_YEAR, CLOTHING_NAME FROM X6_STOCK_DEV.A3_CLOTHING WHERE rownum <= 100000` } func getSQL() string { // 3. 准备查询SQL return `SELECT CLOTHING_ID, CLOTHING_YEAR, CLOTHING_NAME, STYLECOLOR_ID, STYLE_ID, COLOR_ID, SIZE_ID, CREATE_DATE, STYLE_GROUP, J_PRICE, X_PRICE, V_PRICE, CLERK_ROYALTYRATE, CLERK_ROYALTYPRICE, BRAND_CODE, STYLEVER_ID, J_COST, CLOTHING_IMG, STYLE_UNIT_CODE, STYLE_SEX_CODE, STYLE_KIND_CODE, STYLE_CLASS_CODE, STYLE_SUBCLASS_CODE, STYLE_DESIGNER_CODE, STYLE_PLATER_CODE, STYLE_STYLES_CODE, STYLE_LOCATE_CODE, STYLE_SALETYPE_CODE, STYLE_COLORSYSTEM_CODE, STYLE_THEME_CODE, STYLE_INDENTTYPE_CODE, STYLE_PRICEBAND_CODE, STYLE_MONTH_CODE, STYLE_COMPOSITION_CODE, STYLE_SUPPLIER_CODE, STYLE_SPARE1_CODE, STYLE_SPARE2_CODE, STYLE_SPARE4_CODE, STYLE_SPARE5_CODE, CATEGORY_CODE, BRAND_ID, STYCOLVER_ID, STYLE_SAME, CLOTHING_BARCODE, CLOTHING_HELPID, CLOTHING_GBCODE, CLOTHING_RFID, STYLE_SUBJECT_ID, SIZEGRP_ID, STYLE_HELPID, CLOTHING_GBCODE1, COLOR_NAME, STYLEVER_NAME, SIZE_NAME, STYLE_UNIT, STYLE_SEX, STYLE_KIND, STYLE_CLASS, STYLE_SUBCLASS, STYLE_DESIGNER, STYLE_PLATER, STYLE_BAND, STYLE_STYLES, STYLE_LOCATE, STYLE_SALETYPE, STYLE_COLORSYSTEM, STYLE_THEME, STYLE_INDENTTYPE, STYLE_PRICEBAND, STYLE_MONTH, STYLE_COMPOSITION, STYLE_SUPPLIER, STYLE_SPARE1, STYLE_SPARE2, STYLE_SPARE3, STYLE_SPARE4, STYLE_SPARE5, CATEGORY_NAME, BRAND_NAME, STYLE_YEAR_NAME, STYLE_SEARCH_KEY, STYLE_SUBJECT_NAME, CLOTHING_REMARK, STYLE_SPARE3_CODE, COST, BRAND_GROUPCODE, CLASS_GROUPCODE, MONTH_GROUPCODE, RETURNSUBJECT_ID, PRODUCT_SORT, CLOTHING_PARTITION FROM ( SELECT a.*, ROWNUM as rn FROM ( SELECT * FROM X6_STOCK_DEV.A3_CLOTHING ORDER BY CLOTHING_ID ) a WHERE ROWNUM <= 41000 ) WHERE rn > 39000` // FROM X6_STOCK_DEV.A3_CLOTHING WHERE rownum > 100 and rownum <= 20000` }