From aa3463bcfd9f64055a16c83013f90c2055272e9b Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Mon, 27 Jul 2026 20:14:26 +0000 Subject: [PATCH] Fix storage-format portability on 32-bit platforms Two bugs broke TileDB on 32-bit hosts, diagnosed from test runs on Darwin 10.6 ppc (big-endian) and i386: 1. Storage deserializers read boolean fields with read, i.e. sizeof(bool) bytes, while serializers write them as a single byte. On ABIs where bool is not 1 byte (4 bytes on Darwin/PowerPC 32-bit) this desyncs the stream: ArraySchema::deserialize shifts 3 bytes at allows_dups and every subsequent array open fails with "FilterCreate: Deserialization error; not enough data in buffer for metadata". Read and write all such fields as uint8_t (allows_dups, attribute nullable, dimension label relative_uri/is_external, enumeration ordered, current domain empty, webp FilterConfig::lossless). This is byte-identical on platforms where bool is already 1 byte. 2. timestamp_now_ms() computed tv_sec * 1000L in long arithmetic, which overflows when time_t/long are 32-bit, producing garbage fragment timestamps (20-digit names instead of 13). This broke consolidation, open-at-timestamp, fragment info, and schema evolution on i386. Widen to uint64_t before multiplying. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Rk5QiL4q9UqnC9Ww3BqapV --- tiledb/sm/array_schema/array_schema.cc | 4 +++- tiledb/sm/array_schema/attribute.cc | 4 +++- tiledb/sm/array_schema/current_domain.cc | 6 ++++-- tiledb/sm/array_schema/dimension_label.cc | 8 +++++--- tiledb/sm/array_schema/enumeration.cc | 6 ++++-- tiledb/sm/array_schema/test/unit_current_domain.cc | 4 ++-- tiledb/sm/filter/webp_filter.h | 4 +++- tiledb/sm/misc/tdb_time.cc | 4 +++- 8 files changed, 27 insertions(+), 13 deletions(-) diff --git tiledb/sm/array_schema/array_schema.cc tiledb/sm/array_schema/array_schema.cc index d707dab..7fd6aed 100644 --- tiledb/sm/array_schema/array_schema.cc +++ tiledb/sm/array_schema/array_schema.cc @@ -1181,7 +1181,9 @@ shared_ptr ArraySchema::deserialize( // Note: No security validation is possible. bool allows_dups = false; if (version >= 5) { - allows_dups = deserializer.read(); + // The field is a single byte; do not read as `bool`, whose size is + // not 1 on all ABIs (e.g. 4 bytes on Darwin/PowerPC 32-bit). + allows_dups = deserializer.read() != 0; } // Load array type diff --git tiledb/sm/array_schema/attribute.cc tiledb/sm/array_schema/attribute.cc index d69a9c6..df46c37 100644 --- tiledb/sm/array_schema/attribute.cc +++ tiledb/sm/array_schema/attribute.cc @@ -152,7 +152,9 @@ Attribute Attribute::deserialize( // Load nullable flag bool nullable = false; if (version >= 7) { - nullable = deserializer.read(); + // The field is a single byte; do not read as `bool`, whose size is + // not 1 on all ABIs (e.g. 4 bytes on Darwin/PowerPC 32-bit). + nullable = deserializer.read() != 0; } // Load validity fill value diff --git tiledb/sm/array_schema/current_domain.cc tiledb/sm/array_schema/current_domain.cc index 278bba9..cd053e4 100644 --- tiledb/sm/array_schema/current_domain.cc +++ tiledb/sm/array_schema/current_domain.cc @@ -72,7 +72,9 @@ shared_ptr CurrentDomain::deserialize( std::to_string(constants::current_domain_version) + "'"); } - auto empty = deserializer.read(); + // The field is a single byte; do not read as `bool`, whose size is + // not 1 on all ABIs (e.g. 4 bytes on Darwin/PowerPC 32-bit). + bool empty = deserializer.read() != 0; if (empty) { return make_shared(memory_tracker, disk_version); @@ -98,7 +100,7 @@ shared_ptr CurrentDomain::deserialize( void CurrentDomain::serialize(Serializer& serializer) const { serializer.write(constants::current_domain_version); - serializer.write(empty_); + serializer.write(empty_ ? 1 : 0); if (empty_) { return; diff --git tiledb/sm/array_schema/dimension_label.cc tiledb/sm/array_schema/dimension_label.cc index 5340f43..c317635 100644 --- tiledb/sm/array_schema/dimension_label.cc +++ tiledb/sm/array_schema/dimension_label.cc @@ -219,7 +219,9 @@ shared_ptr DimensionLabel::deserialize( deserializer.get_ptr(dim_label_name_size), dim_label_name_size); // Read dimension label URI - auto relative_uri = deserializer.read(); + // Boolean fields are single bytes; do not read as `bool`, whose size is + // not 1 on all ABIs (e.g. 4 bytes on Darwin/PowerPC 32-bit). + bool relative_uri = deserializer.read() != 0; auto uri_size = deserializer.read(); std::string uri(deserializer.get_ptr(uri_size), uri_size); @@ -238,7 +240,7 @@ shared_ptr DimensionLabel::deserialize( auto label_cell_val_num = deserializer.read(); // Read if dimension label is external - auto is_external = deserializer.read(); + bool is_external = deserializer.read() != 0; // Construct and return a shared pointer to a DimensionLabel return make_shared( @@ -293,7 +295,7 @@ void DimensionLabel::serialize(Serializer& serializer, uint32_t) const { serializer.write(dim_label_name_.c_str(), dim_label_name_size); // Read dimension label URI - serializer.write(relative_uri_); + serializer.write(relative_uri_ ? 1 : 0); uint64_t uri_size = (uri_.to_string().size()); serializer.write(uri_size); serializer.write(uri_.c_str(), uri_size); diff --git tiledb/sm/array_schema/enumeration.cc tiledb/sm/array_schema/enumeration.cc index 5006065..56400fd 100644 --- tiledb/sm/array_schema/enumeration.cc +++ tiledb/sm/array_schema/enumeration.cc @@ -264,7 +264,9 @@ shared_ptr Enumeration::deserialize( auto type = deserializer.read(); auto cell_val_num = deserializer.read(); - auto ordered = deserializer.read(); + // The field is a single byte; do not read as `bool`, whose size is + // not 1 on all ABIs (e.g. 4 bytes on Darwin/PowerPC 32-bit). + bool ordered = deserializer.read() != 0; auto data_size = deserializer.read(); const void* data = nullptr; @@ -444,7 +446,7 @@ void Enumeration::serialize(Serializer& serializer) const { serializer.write(static_cast(type_)); serializer.write(cell_val_num_); - serializer.write(ordered_); + serializer.write(ordered_ ? 1 : 0); serializer.write(data_.size()); if (data_.size() > 0) { serializer.write(data_.data(), data_.size()); diff --git tiledb/sm/array_schema/test/unit_current_domain.cc tiledb/sm/array_schema/test/unit_current_domain.cc index 4c008bb..fd641d1 100644 --- tiledb/sm/array_schema/test/unit_current_domain.cc +++ tiledb/sm/array_schema/test/unit_current_domain.cc @@ -172,8 +172,8 @@ storage_size_t CurrentDomainFx::calculate_serialized_size( // uint32_t - version num_bytes += sizeof(uint32_t); - // bool - empty current domain flag - num_bytes += sizeof(bool); + // uint8_t - empty current domain flag + num_bytes += sizeof(uint8_t); if (current_domain->empty()) { return num_bytes; diff --git tiledb/sm/filter/webp_filter.h tiledb/sm/filter/webp_filter.h index 8f521df..10baa92 100644 --- tiledb/sm/filter/webp_filter.h +++ tiledb/sm/filter/webp_filter.h @@ -99,7 +99,9 @@ class WebpFilter : public Filter { struct FilterConfig { float quality; WebpInputFormat format; - bool lossless; + /* uint8_t rather than bool: this struct is serialized to storage as raw + * bytes and sizeof(bool) is not 1 on all ABIs (e.g. Darwin/PowerPC). */ + uint8_t lossless; uint16_t y_extent, x_extent; }; diff --git tiledb/sm/misc/tdb_time.cc tiledb/sm/misc/tdb_time.cc index e720a65..4bfbcc7 100644 --- tiledb/sm/misc/tdb_time.cc +++ tiledb/sm/misc/tdb_time.cc @@ -52,7 +52,9 @@ uint64_t timestamp_now_ms() { struct timeval tp; memset(&tp, 0, sizeof(struct timeval)); gettimeofday(&tp, nullptr); - return static_cast(tp.tv_sec * 1000L + tp.tv_usec / 1000); + // Widen before multiplying: tv_sec * 1000 overflows on 32-bit time_t. + return static_cast(tp.tv_sec) * 1000 + + static_cast(tp.tv_usec) / 1000; #endif }