Skip to content

impl(bq_driver): Picosecond support #1549

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Khushikathuria008 wants to merge 5 commits into
base: main
Choose a base branch
from
Draft

impl(bq_driver): Picosecond support #1549

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8112278
impl(bq_driver): Picosecond support
Khushikathuria008 Mar 16, 2026
544e2cd
test
Khushikathuria008 Jun 5, 2026
ed02476
test
Khushikathuria008 Jun 5, 2026
1e982e9
test
Khushikathuria008 Jun 5, 2026
afdda39
test
Khushikathuria008 Jun 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
236 changes: 157 additions & 79 deletions google/cloud/odbc/bq_driver/internal/data_translation.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -935,26 +935,110 @@ odbc_internal::StatusRecord ConvertFromTimeDSValue(DSValue const& src_dsval,
return status_record;
}

odbc_internal::StatusRecord ConvertTimestampStringToChar(
const std::string& timestamp_src_str,
void* dest_buf,
SQLLEN buffer_length,
SQLLEN* res_len) {

SQLLEN timestamp_src_len =
static_cast<SQLLEN>(timestamp_src_str.size());
auto* dest = reinterpret_cast<char*>(dest_buf);
StatusRecord status_record;

if (buffer_length > timestamp_src_len) {
if (res_len) *res_len = timestamp_src_len;
std::strncpy(dest, timestamp_src_str.c_str(), timestamp_src_len);
dest[timestamp_src_len] = '\0';
} else if (20 <= buffer_length && buffer_length <= timestamp_src_len) {
if (res_len) *res_len = buffer_length;
std::strncpy(dest, timestamp_src_str.c_str(), buffer_length - 1);
dest[buffer_length - 1] = '\0';
status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"};
} else {
status_record =
StatusRecord{SQLStates::k_22003(), "Buffer length is insufficient"};
}
return status_record;
}

odbc_internal::StatusRecord ConvertTimestampStringToWChar(
const std::string& timestamp_src_str,
void* dest_buf,
SQLLEN buffer_length,
SQLLEN* res_len) {

auto wstr_or = Utf8ToUtf16(timestamp_src_str);
if (!wstr_or) {
return StatusRecord{
SQLStates::k_HY000(),
"DSValueToWchar Conversion Failed"};
}

std::vector<SQLWCHAR> wstr_data(
wstr_or->begin(), wstr_or->end());
wstr_data.emplace_back(L'\0');

auto* dest = reinterpret_cast<SQLWCHAR*>(dest_buf);

SQLLEN timestamp_src_len =
static_cast<SQLLEN>(wstr_or->size());

SQLLEN wchar_capacity =
buffer_length / sizeof(SQLWCHAR);

StatusRecord status_record = StatusRecord::Ok();

if (wchar_capacity > timestamp_src_len) {
if (res_len) {
*res_len = timestamp_src_len * sizeof(SQLWCHAR);
}

std::memcpy(dest, wstr_data.data(),
timestamp_src_len * sizeof(SQLWCHAR));

dest[timestamp_src_len] = L'\0';

} else if (20 <= wchar_capacity &&
wchar_capacity <= timestamp_src_len) {

if (res_len) {
*res_len = wchar_capacity * sizeof(SQLWCHAR);
}

std::memcpy(dest, wstr_data.data(),
wchar_capacity * sizeof(SQLWCHAR));

dest[wchar_capacity - 1] = L'\0';

status_record =
StatusRecord{SQLStates::k_01004(),
"Data truncated"};

} else {

status_record =
StatusRecord{SQLStates::k_22003(),
"Buffer length is insufficient"};
}

return status_record;
}

odbc_internal::StatusRecord ConvertFromTimestampDSValue(
DSValue const& src_dsval, DataBuffer& dest_data) {
using odbc_internal::SQLStates;
using odbc_internal::StatusRecord;
using odbc_internal::StatusRecordOr;

SQL_TIMESTAMP_STRUCT timestamp_src_struct;
DSValueToTimestamp(src_dsval, timestamp_src_struct);
using odbc_internal::SQLStates;
using odbc_internal::StatusRecord;

std::string timestamp_src_str;
timestamp_src_str = FormatTimestampToString(timestamp_src_struct);

SQLSMALLINT dest_type = dest_data.type;
SQLPOINTER dest_buf = dest_data.buf;
SQLLEN buffer_length = dest_data.buflen;
SQLLEN* res_len = dest_data.result_len;
std::string str_val;
DSValueToString(src_dsval, str_val);
std::string timestamp_src_str;

// Define length variables
int k_timestamp_src_len = timestamp_src_str.length();
constexpr int kTimestampBinaryLength = sizeof(SQL_TIMESTAMP_STRUCT);
SQLSMALLINT dest_type = dest_data.type;
SQLPOINTER dest_buf = dest_data.buf;
SQLLEN buffer_length = dest_data.buflen;
SQLLEN* res_len = dest_data.result_len;

if (!dest_buf) {
return StatusRecord::Ok();
Expand All @@ -965,82 +1049,76 @@ odbc_internal::StatusRecord ConvertFromTimestampDSValue(
return StatusRecord{SQLStates::k_HY090(), "Invalid Buffer length"};
}

StatusRecord status_record = StatusRecord::Ok();
StatusRecord status_record = StatusRecord::Ok();

switch (dest_type) {
case SQL_C_CHAR: {
auto* dest = reinterpret_cast<char*>(dest_buf);
if (buffer_length > k_timestamp_src_len) {
if (res_len) {
*res_len = k_timestamp_src_len;
// --- Check for ISO string with 'T' and long fraction ---
auto t_pos = str_val.find('T');
if (t_pos != std::string::npos) {
auto dot_pos = str_val.find('.', t_pos);
bool long_fraction = false;

if (dot_pos != std::string::npos) {
std::size_t fraction_length = str_val.size() - dot_pos - 1;
if (!str_val.empty() && str_val.back() == 'Z') --fraction_length;
if (fraction_length > 9) long_fraction = true;
}
std::strncpy(dest, timestamp_src_str.c_str(), k_timestamp_src_len);
dest[k_timestamp_src_len] = '\0';
} else if (20 <= buffer_length && buffer_length <= k_timestamp_src_len) {
if (res_len) {
*res_len = buffer_length;

if (long_fraction) {
// Trim 'Z' and replace 'T' with space
if (!str_val.empty() && str_val.back() == 'Z') str_val.pop_back();
str_val[t_pos] = ' ';
timestamp_src_str = str_val;

switch (dest_type) {
case SQL_C_CHAR:
return ConvertTimestampStringToChar(timestamp_src_str, dest_buf, buffer_length, res_len);
case SQL_C_WCHAR:
return ConvertTimestampStringToWChar(timestamp_src_str, dest_buf, buffer_length, res_len);
default:
LOG(ERROR) << "ConvertFromTimestampDSValue:: Conversion unsupported for picosecond for C-type: " << dest_type;
return StatusRecord{SQLStates::k_HY000(), "Conversion unsupported for picosecond"};
}
}
std::strncpy(dest, timestamp_src_str.c_str(), buffer_length - 1);
dest[buffer_length - 1] = '\0';
LOG(WARNING)
<< "ConvertFromTimestampDSValue:: Data truncated for SQL_C_CHAR.";
status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"};
} else {
LOG(ERROR) << "ConvertFromTimestampDSValue:: Buffer length is "
"insufficient for SQL_C_CHAR.";
status_record =
StatusRecord{SQLStates::k_22003(), "Buffer length is insufficient"};
}
break;
}
//Conversion for unix epoch time
bool looks_like_float_epoch = false;

case SQL_C_WCHAR: {
StatusRecordOr<std::wstring> wstr = Utf8ToUtf16(timestamp_src_str);
if (!wstr) {
LOG(ERROR)
<< "ConvertFromTimestampDSValue:: DSValueToWchar Conversion Failed";
status_record = StatusRecord{SQLStates::k_HY000(),
"DSValueToWchar Conversion Failed"};
break;
}
std::vector<SQLWCHAR> wstr_data(wstr->begin(), wstr->end());
wstr_data.emplace_back(L'\0');
try {
size_t idx = 0;
std::stod(str_val, &idx);
if (idx == str_val.length()) {
looks_like_float_epoch = true;
}
} catch (...) {
looks_like_float_epoch = false;
}
SQL_TIMESTAMP_STRUCT timestamp_src_struct ;
if (looks_like_float_epoch) {
timestamp_src_str = FloatTimestampToString(str_val);
timestamp_src_struct= ConvertStrToTimestampStruct(timestamp_src_str);
} else {
DSValueToTimestamp(src_dsval, timestamp_src_struct);
timestamp_src_str =
FormatTimestampToString(timestamp_src_struct);
}
int k_timestamp_src_len = static_cast<int>(timestamp_src_str.length());
constexpr int kTimestampBinaryLength = sizeof(SQL_TIMESTAMP_STRUCT);

auto* dest = reinterpret_cast<SQLWCHAR*>(dest_buf);
SQLLEN wchar_capacity = buffer_length / sizeof(SQLWCHAR);
if (wchar_capacity > k_timestamp_src_len) {
if (res_len) {
*res_len = k_timestamp_src_len * sizeof(SQLWCHAR);
}
std::memcpy(dest, wstr_data.data(),
(k_timestamp_src_len) * sizeof(SQLWCHAR));
dest[k_timestamp_src_len] = L'\0';
} else if (20 <= wchar_capacity &&
wchar_capacity <= k_timestamp_src_len) {
if (res_len) {
*res_len = wchar_capacity * sizeof(SQLWCHAR);
}
std::memcpy(dest, wstr_data.data(),
(wchar_capacity) * sizeof(SQLWCHAR));
dest[wchar_capacity - 1] = L'\0';
LOG(WARNING)
<< "ConvertFromTimestampDSValue:: Data truncated for SQL_C_WCHAR.";
status_record = StatusRecord{SQLStates::k_01004(), "Data truncated"};
} else {
LOG(ERROR) << "ConvertFromTimestampDSValue:: Buffer length is "
"insufficient for SQL_C_WCHAR.";
status_record =
StatusRecord{SQLStates::k_22003(), "Buffer length is insufficient"};
}
switch (dest_type) {
case SQL_C_CHAR:
status_record = ConvertTimestampStringToChar(timestamp_src_str, dest_buf, buffer_length, res_len);
break;

case SQL_C_WCHAR:
status_record = ConvertTimestampStringToWChar(timestamp_src_str, dest_buf, buffer_length, res_len);
break;
}

case SQL_C_BINARY: {
if (kTimestampBinaryLength <= buffer_length) {
if (res_len) {
*res_len = kTimestampBinaryLength;
}
timestamp_src_struct.fraction = timestamp_src_struct.fraction * 1000;
timestamp_src_struct.fraction = timestamp_src_struct.fraction;
std::memcpy(dest_buf, &timestamp_src_struct, kTimestampBinaryLength);

} else {
Expand Down Expand Up @@ -1214,7 +1292,7 @@ odbc_internal::StatusRecord ConvertFromDatetimeDSValue(DSValue const& src_dsval,
if (res_len) {
*res_len = kDatetimeBinaryLength;
}
datetime_src_struct.fraction = datetime_src_struct.fraction * 1000;
datetime_src_struct.fraction = datetime_src_struct.fraction;
std::memcpy(dest_buf, &datetime_src_struct, kDatetimeBinaryLength);

} else {
Expand Down
2 changes: 1 addition & 1 deletion google/cloud/odbc/bq_driver/internal/data_translation_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1105,7 +1105,7 @@ TEST(ConvertFromTimestampDSValue, convertToBinarySuccess) {
expected_timestamp.hour = 01;
expected_timestamp.minute = 59;
expected_timestamp.second = 43;
expected_timestamp.fraction = 112233000;
expected_timestamp.fraction = 112233;
char dest_buf[30];
DataBuffer dest_data = {SQL_C_BINARY, dest_buf, sizeof(dest_buf),
&result_len};
Expand Down
6 changes: 6 additions & 0 deletions google/cloud/odbc/bq_driver/internal/odbc_conn_handle.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,12 @@ void ConnectionHandle::SetUp(Section& dsn_section,
std::string job_creation_mode = dsn_section["JOBCREATIONMODE"];
dsn_.is_job_creation_required = (job_creation_mode == "1");

std::string timestamp_output_format =dsn_section["TIMESTAMPOUTPUTFORMAT"];
if (!timestamp_output_format.empty()) {
dsn_.format_options.timestamp_output_format = timestamp_output_format;
}


if (attribute_str_values_.count(SQL_ATTR_CURRENT_CATALOG) == 0) {
attribute_str_values_.insert({SQL_ATTR_CURRENT_CATALOG, dsn_.catalog});
}
Expand Down
1 change: 1 addition & 0 deletions google/cloud/odbc/bq_driver/internal/odbc_conn_handle.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ struct Dsn {
std::vector<ConnectionProperty> connection_properties;
std::uint32_t row_fetched_per_block = 100000;
std::uint32_t default_string_column_length = 16384;
google::cloud::bigquery_v2_minimal_internal::DataFormatOptions format_options;
/////////////////////////////////////////////////////////////////
// Optional Properties needed for HTAPI.
/////////////////////////////////////////////////////////////////
Expand Down
Loading
Loading