go-service-agent/test/my_test.go

package main

import (
	"fmt"
	"testing"
	"time"

	"git.x2erp.com/qdy/go-base/config"
	"git.x2erp.com/qdy/go-db/factory"
)

// QueryRequest 查询请求结构体
type QueryRequest struct {
	SQL string `json:"sql"`
}

func TestQueryAndInsertToDoris(t *testing.T) {
	// 记录总开始时间
	totalStartTime := time.Now()
	// 1. 获取HTTP工厂实例
	httpFactory, err := factory.GetHTTPFactory()
	if err != nil {
		t.Fatalf("Failed to get HTTP factory: %v", err)
	}
	fmt.Println("HTTP factory created successfully")

	// 2. 创建HTTP客户端
	httpClient := httpFactory.CreateClient()

	// 3. 准备查询SQL - 限制数据量用于测试
	querySQL := getSQL()

	// 4. 准备查询请求
	queryRequest := QueryRequest{
		SQL: querySQL,
	}

	// 记录查询开始时间
	queryStartTime := time.Now()
	// 5. 发送POST请求到 /api/query/csv 获取CSV格式数据
	fmt.Println("Sending query request to localhost:8080/api/query/csv...")

	resp, err := httpClient.PostWithAuth(
		"http://localhost:8080/api/query/csv",
		queryRequest,
		"123", // Bearer Token
		nil,
	)

	if err != nil {
		t.Fatalf("Failed to send query request: %v", err)
	}

	if resp.StatusCode() != 200 {
		t.Fatalf("Query request failed with status: %d, body: %s", resp.StatusCode(), string(resp.Body()))
	}

	// 6. 获取CSV数据
	csvData := string(resp.Body())
	if len(csvData) == 0 {
		t.Fatalf("No CSV data received")
	}

	// 记录查询结束时间
	queryEndTime := time.Now()
	queryDuration := queryEndTime.Sub(queryStartTime)

	fmt.Printf("✅ Query completed in %v\n", queryDuration)
	fmt.Printf("Successfully retrieved CSV data, length: %d bytes\n", len(csvData))
	//fmt.Printf("CSV data content:\n%s\n", csvData)

	// 7. 获取Doris工厂实例
	dorisFactory, err := factory.GetDorisFactory(httpFactory)
	if err != nil {
		t.Fatalf("Failed to get Doris factory: %v", err)
	}
	fmt.Println("Doris factory created successfully")

	// 8. 先检查Doris表结构
	fmt.Println("Checking Doris table structure...")

	// 9. 插入数据到Doris
	database := "X6_STOCK_DEV"
	table := "A3_CLOTHING"
	skipHeader := false // 改为true，跳过CSV头行

	fmt.Printf("Inserting data to Doris database: %s, table: %s\n", database, table)
	//fmt.Printf("CSV data to insert:\n%q\n", csvData)

	cfg := config.GetConfig()
	url := fmt.Sprintf("http://%s:%d/api/%s/%s/_stream_load", cfg.GetDoris().FEHost, cfg.GetDoris().FEPort, database, table)

	fmt.Print(url + ".\n")

	// 记录保存开始时间
	saveStartTime := time.Now()
	err = dorisFactory.InsertCSV(database, table, csvData, skipHeader)
	if err != nil {
		t.Fatalf("Failed to insert data to Doris: %v", err)
	}

	// 记录保存结束时间
	saveEndTime := time.Now()
	saveDuration := saveEndTime.Sub(saveStartTime)

	fmt.Printf("✅ Data saved to Doris in %v\n", saveDuration)

	// 记录总结束时间
	totalEndTime := time.Now()
	totalDuration := totalEndTime.Sub(totalStartTime)

	// 打印性能统计
	fmt.Println("\n📊 Performance Statistics:")
	fmt.Printf("   Query Time: %v\n", queryDuration)
	fmt.Printf("   Save Time:  %v\n", saveDuration)
	fmt.Printf("   Total Time: %v\n", totalDuration)
	fmt.Printf("   Data Size:  %d bytes\n", len(csvData))

	fmt.Println("Data successfully inserted to Doris!")
}

func getSQLm() string {
	return `SELECT
    CLOTHING_ID,
    CLOTHING_YEAR,
    CLOTHING_NAME
FROM X6_STOCK_DEV.A3_CLOTHING WHERE rownum <= 100000`
}

func getSQL() string {
	// 3. 准备查询SQL
	return `SELECT
    CLOTHING_ID,
    CLOTHING_YEAR,
    CLOTHING_NAME,
    STYLECOLOR_ID,
    STYLE_ID,
    COLOR_ID,
    SIZE_ID,
    CREATE_DATE,
    STYLE_GROUP,
    J_PRICE,
    X_PRICE,
    V_PRICE,
    CLERK_ROYALTYRATE,
    CLERK_ROYALTYPRICE,
    BRAND_CODE,
    STYLEVER_ID,
    J_COST,
    CLOTHING_IMG,
    STYLE_UNIT_CODE,
    STYLE_SEX_CODE,
    STYLE_KIND_CODE,
    STYLE_CLASS_CODE,
    STYLE_SUBCLASS_CODE,
    STYLE_DESIGNER_CODE,
    STYLE_PLATER_CODE,
    STYLE_STYLES_CODE,
    STYLE_LOCATE_CODE,
    STYLE_SALETYPE_CODE,
    STYLE_COLORSYSTEM_CODE,
    STYLE_THEME_CODE,
    STYLE_INDENTTYPE_CODE,
    STYLE_PRICEBAND_CODE,
    STYLE_MONTH_CODE,
    STYLE_COMPOSITION_CODE,
    STYLE_SUPPLIER_CODE,
    STYLE_SPARE1_CODE,
    STYLE_SPARE2_CODE,
    STYLE_SPARE4_CODE,
    STYLE_SPARE5_CODE,
    CATEGORY_CODE,
    BRAND_ID,
    STYCOLVER_ID,
    STYLE_SAME,
    CLOTHING_BARCODE,
    CLOTHING_HELPID,
    CLOTHING_GBCODE,
    CLOTHING_RFID,
    STYLE_SUBJECT_ID,
    SIZEGRP_ID,
    STYLE_HELPID,
    CLOTHING_GBCODE1,
    COLOR_NAME,
    STYLEVER_NAME,
    SIZE_NAME,
    STYLE_UNIT,
    STYLE_SEX,
    STYLE_KIND,
    STYLE_CLASS,
    STYLE_SUBCLASS,
    STYLE_DESIGNER,
    STYLE_PLATER,
    STYLE_BAND,
    STYLE_STYLES,
    STYLE_LOCATE,
    STYLE_SALETYPE,
    STYLE_COLORSYSTEM,
    STYLE_THEME,
    STYLE_INDENTTYPE,
    STYLE_PRICEBAND,
    STYLE_MONTH,
    STYLE_COMPOSITION,
    STYLE_SUPPLIER,
    STYLE_SPARE1,
    STYLE_SPARE2,
    STYLE_SPARE3,
    STYLE_SPARE4,
    STYLE_SPARE5,
    CATEGORY_NAME,
    BRAND_NAME,
    STYLE_YEAR_NAME,
    STYLE_SEARCH_KEY,
    STYLE_SUBJECT_NAME,
    CLOTHING_REMARK,
    STYLE_SPARE3_CODE,
    COST,
    BRAND_GROUPCODE,
    CLASS_GROUPCODE,
    MONTH_GROUPCODE,
    RETURNSUBJECT_ID,
    PRODUCT_SORT,
    CLOTHING_PARTITION
   FROM (
    SELECT a.*, ROWNUM as rn
    FROM (
        SELECT *
        FROM X6_STOCK_DEV.A3_CLOTHING 
        ORDER BY CLOTHING_ID
    ) a
    WHERE ROWNUM <= 41000 
)
WHERE rn > 39000`

	// FROM X6_STOCK_DEV.A3_CLOTHING WHERE rownum > 100 and rownum <= 20000`
}