From 38300ba1e37d3a4249884f048d6e7638b4a1af8d Mon Sep 17 00:00:00 2001 From: dlapko Date: Thu, 27 Feb 2025 20:12:53 +0200 Subject: [PATCH 1/5] Add null values check for binary types in BulkCopy test --- bulkcopy_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bulkcopy_test.go b/bulkcopy_test.go index 180b8c96..b8702881 100644 --- a/bulkcopy_test.go +++ b/bulkcopy_test.go @@ -187,6 +187,11 @@ func testBulkcopy(t *testing.T, guidConversion bool) { {"test_varbinary_max", bin, nil}, {"test_binary", []byte("1"), nil}, {"test_binary_16", bin, nil}, + {"test_varbinaryn", nil, nil}, + {"test_varbinary_16n", nil, nil}, + {"test_varbinary_maxn", nil, nil}, + {"test_binaryn", nil, nil}, + {"test_binary_16n", nil, nil}, {"test_intvarchar", 1234, "1234"}, {"test_int64nvarchar", int64(123456), "123456"}, {"test_int32nvarchar", int32(12345), "12345"}, @@ -421,6 +426,11 @@ func setupTable(ctx context.Context, t *testing.T, conn *sql.Conn, tableName str [test_varbinary_max] VARBINARY(max) NOT NULL, [test_binary] BINARY NOT NULL, [test_binary_16] BINARY(16) NOT NULL, + [test_varbinaryn] VARBINARY NULL, + [test_varbinary_16n] VARBINARY(16) NULL, + [test_varbinary_maxn] VARBINARY(max) NULL, + [test_binaryn] BINARY NULL, + [test_binary_16n] BINARY(16) NULL, [test_intvarchar] [varchar](4) NULL, [test_int64nvarchar] [varchar](6) NULL, [test_int32nvarchar] [varchar](5) NULL, From 14406365d730ed72cef865cfeac585119cf400bd Mon Sep 17 00:00:00 2001 From: dlapko Date: Thu, 27 Feb 2025 20:56:23 +0200 Subject: [PATCH 2/5] Fix: handle nulls in writeLongLenType function --- bulkcopy_test.go | 8 ++++++-- types.go | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/bulkcopy_test.go b/bulkcopy_test.go index b8702881..476331d9 100644 --- a/bulkcopy_test.go +++ b/bulkcopy_test.go @@ -136,6 +136,8 @@ func testBulkcopy(t *testing.T, guidConversion bool) { {"test_nchar", "abcdefg ", nil}, {"test_text", "abcdefg", nil}, {"test_ntext", "abcdefg", nil}, + {"test_textn", nil, nil}, + {"test_ntextn", nil, nil}, {"test_float", 1234.56, nil}, {"test_floatn", 1234.56, nil}, {"test_real", 1234.56, nil}, @@ -378,8 +380,10 @@ func setupTable(ctx context.Context, t *testing.T, conn *sql.Conn, tableName str [test_varchar_max_nil] [varchar](max) NULL, [test_char] [char](10) NULL, [test_nchar] [nchar](10) NULL, - [test_text] [text] NULL, - [test_ntext] [ntext] NULL, + [test_text] [text] NOT NULL, + [test_ntext] [ntext] NOT NULL, + [test_textn] [text] NULL, + [test_ntextn] [ntext] NULL, [test_float] [float] NOT NULL, [test_floatn] [float] NULL, [test_real] [real] NULL, diff --git a/types.go b/types.go index 94b202f9..ca7dc0ab 100644 --- a/types.go +++ b/types.go @@ -578,6 +578,21 @@ func readLongLenType(ti *typeInfo, r *tdsBuffer, c *cryptoMetadata) interface{} panic("shoulnd't get here") } func writeLongLenType(w io.Writer, ti typeInfo, buf []byte) (err error) { + if buf == nil { + // According to the documentation, we MUST NOT specify the text pointer and timestamp when the value is NULL. + // + // https://learn.microsoft.com/openspecs/windows_protocols/ms-tds/3840ef93-3b10-4aca-9fd1-a210b8bb6d0c + // + // However, this approach fails with the error: + // "Expected the text length in data stream for bulk copy of text, ntext, or image data." + // + // But we can insert NULL successfully by setting the text pointer length to zero + // (without writing any additional bytes). + // Since there's no clear way to follow the documentation exactly, let's use this solution. + err = binary.Write(w, binary.LittleEndian, byte(0x00)) + return + } + //textptr err = binary.Write(w, binary.LittleEndian, byte(0x10)) if err != nil { From 75b88cd46b96ec1ab7f3ffdee3460d7d34106d5a Mon Sep 17 00:00:00 2001 From: dlapko Date: Thu, 27 Feb 2025 01:38:06 +0200 Subject: [PATCH 3/5] Fix: support image type for bulkcopy --- bulkcopy.go | 2 +- bulkcopy_test.go | 4 ++++ types.go | 14 ++++++++++++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/bulkcopy.go b/bulkcopy.go index be8398b1..8818165e 100644 --- a/bulkcopy.go +++ b/bulkcopy.go @@ -599,7 +599,7 @@ func (b *Bulk) makeParam(val DataValue, col columnStruct) (res param, err error) buf[i] = ub[j] } res.buffer = buf - case typeBigVarBin, typeBigBinary: + case typeBigVarBin, typeBigBinary, typeImage: switch val := val.(type) { case []byte: res.ti.Size = len(val) diff --git a/bulkcopy_test.go b/bulkcopy_test.go index 476331d9..6942c4dc 100644 --- a/bulkcopy_test.go +++ b/bulkcopy_test.go @@ -200,6 +200,8 @@ func testBulkcopy(t *testing.T, guidConversion bool) { {"test_int16nvarchar", int16(1234), "1234"}, {"test_int8nvarchar", int8(12), "12"}, {"test_intnvarchar", 1234, "1234"}, + {"test_image", []byte("1"), nil}, + {"test_imagen", nil, nil}, } columns := make([]string, len(testValues)) @@ -449,6 +451,8 @@ func setupTable(ctx context.Context, t *testing.T, conn *sql.Conn, tableName str [test_nullint32] [int] NULL, [test_nullint16] [smallint] NULL, [test_nulltime] [datetime] NULL, + [test_image] [image] NOT NULL, + [test_imagen] [image] NULL, CONSTRAINT [PK_` + tableName + `_id] PRIMARY KEY CLUSTERED ( [id] ASC diff --git a/types.go b/types.go index ca7dc0ab..e409deac 100644 --- a/types.go +++ b/types.go @@ -254,8 +254,16 @@ func writeVarLen(w io.Writer, ti *typeInfo, out bool, encoding msdsn.EncodeParam if err = binary.Write(w, binary.LittleEndian, uint32(ti.Size)); err != nil { return } - if err = writeCollation(w, ti.Collation); err != nil { - return + + // COLLATION occurs only if the type is BIGCHARTYPE, BIGVARCHARTYPE, TEXTTYPE, NTEXTTYPE, + // NCHARTYPE, or NVARCHARTYPE. + // + // https://learn.microsoft.com/openspecs/windows_protocols/ms-tds/cbe9c510-eae6-4b1f-9893-a098944d430a + switch ti.TypeId { + case typeText, typeNText: + if err = writeCollation(w, ti.Collation); err != nil { + return + } } ti.Writer = writeLongLenType default: @@ -1333,6 +1341,8 @@ func makeDecl(ti typeInfo) string { return "ntext" case typeUdt: return ti.UdtInfo.TypeName + case typeImage: + return "image" case typeGuid: return "uniqueidentifier" case typeTvp: From a91360d6b732fab3eff497b01489f38ba106ac55 Mon Sep 17 00:00:00 2001 From: dlapko Date: Thu, 27 Feb 2025 20:12:53 +0200 Subject: [PATCH 4/5] Fix: support xml type for bulkcopy --- bulkcopy.go | 8 +++++++- bulkcopy_test.go | 5 ++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/bulkcopy.go b/bulkcopy.go index 8818165e..db5583c0 100644 --- a/bulkcopy.go +++ b/bulkcopy.go @@ -82,6 +82,13 @@ func (b *Bulk) sendBulkCommand(ctx context.Context) (err error) { } } if bulkCol != nil { + // Note that for INSERT BULK operations, XMLTYPE is to be sent as NVARCHAR(N) or NVARCHAR(MAX) data type. + // An error is produced if XMLTYPE is specified. + // + // https://learn.microsoft.com/openspecs/windows_protocols/ms-tds/ab4a7d62-cd1f-4db1-b67d-ecae58f493e3 + if bulkCol.ti.TypeId == typeXml { + bulkCol.ti.TypeId = typeNVarChar + } if bulkCol.ti.TypeId == typeUdt { //send udt as binary @@ -617,7 +624,6 @@ func (b *Bulk) makeParam(val DataValue, col columnStruct) (res param, err error) err = fmt.Errorf("mssql: invalid type for Guid column: %T %s", val, val) return } - default: err = fmt.Errorf("mssql: type %x not implemented", col.ti.TypeId) } diff --git a/bulkcopy_test.go b/bulkcopy_test.go index 6942c4dc..c76bdfc1 100644 --- a/bulkcopy_test.go +++ b/bulkcopy_test.go @@ -202,6 +202,8 @@ func testBulkcopy(t *testing.T, guidConversion bool) { {"test_intnvarchar", 1234, "1234"}, {"test_image", []byte("1"), nil}, {"test_imagen", nil, nil}, + {"test_xml", "value", nil}, + {"test_xmln", nil, nil}, } columns := make([]string, len(testValues)) @@ -417,7 +419,8 @@ func setupTable(ctx context.Context, t *testing.T, conn *sql.Conn, tableName str [test_intf32] [int] NULL, [test_geom] [geometry] NULL, [test_geog] [geography] NULL, - [text_xml] [xml] NULL, + [test_xml] [xml] NOT NULL, + [test_xmln] [xml] NULL, [test_uniqueidentifier] [uniqueidentifier] NULL, [test_nulluniqueidentifier] [uniqueidentifier] NULL, [test_decimal_18_0] [decimal](18, 0) NULL, From 6babea6b6b08f371799787b6fe683aa25173d39a Mon Sep 17 00:00:00 2001 From: dlapko Date: Thu, 27 Feb 2025 20:31:27 +0200 Subject: [PATCH 5/5] Fix: support money/smallmoney types for bulkcopy --- bulkcopy.go | 69 +++++++++++++++++++++++++++++++++++++++++++++++- bulkcopy_test.go | 12 ++++++--- 2 files changed, 76 insertions(+), 5 deletions(-) diff --git a/bulkcopy.go b/bulkcopy.go index db5583c0..dfac8c34 100644 --- a/bulkcopy.go +++ b/bulkcopy.go @@ -542,7 +542,74 @@ func (b *Bulk) makeParam(val DataValue, col columnStruct) (res param, err error) err = fmt.Errorf("mssql: invalid type for time column: %T %s", val, val) return } - // case typeMoney, typeMoney4, typeMoneyN: + case typeMoney, typeMoney4, typeMoneyN: + var intvalue int64 + + string2Int64 := func(str string) (int64, error) { + // Split on decimal point + parts := strings.Split(str, ".") + if len(parts) > 2 { + return 0, fmt.Errorf("invalid money format") + } + + // Handle the decimal places + if len(parts) == 2 { + // Pad or truncate decimal places to exactly 4 digits + decimal := parts[1] + if len(decimal) > 4 { + decimal = decimal[:4] // truncate to 4 decimal places + } else { + decimal = decimal + strings.Repeat("0", 4-len(decimal)) // pad with zeros + } + str = parts[0] + decimal + } else { + // No decimal point, append 4 zeros + str = str + "0000" + } + + return strconv.ParseInt(str, 10, 64) + } + + switch val := val.(type) { + case int: + intvalue = int64(val) + case int64: + intvalue = val + case []byte: + intvalue, err = string2Int64(string(val)) + if err != nil { + return res, fmt.Errorf("mssql: invalid money string format: %s", string(val)) + } + case string: + intvalue, err = string2Int64(val) + if err != nil { + return res, fmt.Errorf("mssql: invalid money string format: %s", val) + } + default: + err = fmt.Errorf("mssql: invalid type for money column: %T %s", val, val) + return + } + + res.buffer = make([]byte, res.ti.Size) + + // smallmoney is a 4-byte integer stored as value * 10^4. + // money is an 8-byte integer stored as value * 10^4. + // + // https://learn.microsoft.com/openspecs/windows_protocols/ms-tds/1266679d-cd6e-492a-b2b2-3a9ba004196d + switch col.ti.Size { + case 4: + binary.LittleEndian.PutUint32(res.buffer, uint32(intvalue)) + case 8: + // The 8-byte signed integer is represented in the following sequence: + // - One 4-byte integer that represents the more significant half. + // - One 4-byte integer that represents the less significant half. + // + // https://learn.microsoft.com/openspecs/windows_protocols/ms-tds/1266679d-cd6e-492a-b2b2-3a9ba004196d + binary.LittleEndian.PutUint32(res.buffer[0:4], uint32(intvalue>>32)) + binary.LittleEndian.PutUint32(res.buffer[4:8], uint32(intvalue&0xFFFFFFFF)) + default: + err = fmt.Errorf("mssql: invalid size of column %d", col.ti.Size) + } case typeDecimal, typeDecimalN, typeNumeric, typeNumericN: prec := col.ti.Prec scale := col.ti.Scale diff --git a/bulkcopy_test.go b/bulkcopy_test.go index c76bdfc1..29913796 100644 --- a/bulkcopy_test.go +++ b/bulkcopy_test.go @@ -176,8 +176,10 @@ func testBulkcopy(t *testing.T, guidConversion bool) { {"test_nullint32", sql.NullInt32{2147483647, true}, 2147483647}, {"test_nullint16", sql.NullInt16{32767, true}, 32767}, {"test_nulltime", sql.NullTime{time.Date(2010, 11, 12, 13, 14, 15, 120000000, time.UTC), true}, time.Date(2010, 11, 12, 13, 14, 15, 120000000, time.UTC)}, - // {"test_smallmoney", 1234.56, nil}, - // {"test_money", 1234.56, nil}, + {"test_smallmoney", []byte("1234.5600"), nil}, + {"test_smallmoneyn", nil, nil}, + {"test_money", []byte("1234.5600"), nil}, + {"test_moneyn", nil, nil}, {"test_decimal_18_0", 1234.0001, "1234"}, {"test_decimal_9_2", -1234.560001, "-1234.56"}, {"test_decimal_20_0", 1234, "1234"}, @@ -407,8 +409,10 @@ func setupTable(ctx context.Context, t *testing.T, conn *sql.Conn, tableName str [test_date_2] [date] NULL, [test_time] [time](7) NULL, [test_time_2] [time](7) NULL, - [test_smallmoney] [smallmoney] NULL, - [test_money] [money] NULL, + [test_smallmoney] [smallmoney] NOT NULL, + [test_smallmoneyn] [smallmoney] NULL, + [test_money] [money] NOT NULL, + [test_moneyn] [money] NULL, [test_tinyint] [tinyint] NULL, [test_smallint] [smallint] NOT NULL, [test_smallintn] [smallint] NULL,