-
Notifications
You must be signed in to change notification settings - Fork 117
feat(inspect): implement SnapshotsTable scanning #801
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
3b11911
6c112cc
791474e
a5c20c2
ab62d36
6680bf9
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -23,14 +23,28 @@ | |
| /// \brief Define base APIs for metadata tables. | ||
|
|
||
| #include <memory> | ||
| #include <optional> | ||
| #include <string> | ||
|
|
||
| #include "iceberg/arrow_c_data.h" | ||
| #include "iceberg/iceberg_export.h" | ||
| #include "iceberg/result.h" | ||
| #include "iceberg/table_identifier.h" | ||
| #include "iceberg/type_fwd.h" | ||
| #include "iceberg/util/timepoint.h" | ||
|
|
||
| namespace iceberg { | ||
|
|
||
| /// \brief Parameters for snapshot selection (time travel). | ||
| struct SnapshotSelection { | ||
| /// \brief The snapshot ID to read. | ||
| std::optional<int64_t> snapshot_id; | ||
| /// \brief Read the snapshot that was current at this timestamp. | ||
| std::optional<TimePointMs> as_of_timestamp; | ||
| /// \brief Read the snapshot referenced by this named ref (branch or tag). | ||
| std::optional<std::string> ref_name; | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is |
||
| }; | ||
|
|
||
| /// \brief Base class for Iceberg metadata tables. | ||
| class ICEBERG_EXPORT MetadataTable { | ||
| public: | ||
|
|
@@ -46,6 +60,27 @@ class ICEBERG_EXPORT MetadataTable { | |
|
|
||
| virtual Kind kind() const noexcept = 0; | ||
|
|
||
| /// \brief Whether this metadata table supports time-travel queries. | ||
| /// | ||
| /// The currently supported snapshots and history metadata tables do not | ||
| /// support time travel. | ||
| bool supports_time_travel() const noexcept; | ||
|
|
||
| /// \brief Scan the metadata table using the current snapshot. | ||
| /// | ||
| /// Convenience overload — delegates to Scan(std::nullopt). | ||
| Result<ArrowArray> Scan() { return Scan(std::nullopt); } | ||
|
|
||
| /// \brief Scan the metadata table and return all rows as an Arrow struct array. | ||
| /// | ||
| /// The returned ArrowArray is a struct array where each element is one row. | ||
| /// The caller takes ownership and must call ArrowArrayRelease when done. | ||
| /// | ||
| /// The default implementation returns NotSupported. Subclasses override this | ||
| /// to materialize their data. | ||
| virtual Result<ArrowArray> Scan( | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should we return |
||
| const std::optional<SnapshotSelection>& snapshot_selection); | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It doesn't seem necessary to add a std::optional on top of SnapshotSelection if all its values are optional. |
||
|
|
||
| const TableIdentifier& name() const { return identifier_; } | ||
|
|
||
| const std::shared_ptr<Schema>& schema() const { return schema_; } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,15 +19,18 @@ | |
|
|
||
| #include "iceberg/inspect/snapshots_table.h" | ||
|
|
||
| #include <chrono> | ||
| #include <memory> | ||
| #include <utility> | ||
| #include <vector> | ||
|
|
||
| #include "iceberg/arrow_row_builder_internal.h" | ||
| #include "iceberg/schema.h" | ||
| #include "iceberg/schema_field.h" | ||
| #include "iceberg/table.h" | ||
| #include "iceberg/table_identifier.h" | ||
| #include "iceberg/type.h" | ||
| #include "iceberg/util/macros.h" | ||
|
|
||
| namespace iceberg { | ||
| namespace { | ||
|
|
@@ -65,4 +68,50 @@ Result<std::unique_ptr<SnapshotsTable>> SnapshotsTable::Make( | |
| return std::unique_ptr<SnapshotsTable>(new SnapshotsTable(std::move(table))); | ||
| } | ||
|
|
||
| Result<ArrowArray> SnapshotsTable::Scan( | ||
| const std::optional<SnapshotSelection>& /*snapshot_selection*/) { | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Silently ignoring an invalid selection diverges from Java. |
||
| ICEBERG_ASSIGN_OR_RAISE(auto builder, ArrowRowBuilder::Make(*schema())); | ||
|
|
||
| for (const auto& snapshot : source_table()->snapshots()) { | ||
| if (snapshot == nullptr) [[unlikely]] { | ||
| continue; | ||
| } | ||
|
|
||
| // column 0: committed_at (timestamptz -> int64 micros) | ||
| ICEBERG_RETURN_UNEXPECTED(AppendInt( | ||
| builder.column(0), std::chrono::duration_cast<std::chrono::microseconds>( | ||
| snapshot->timestamp_ms.time_since_epoch()) | ||
| .count())); | ||
|
|
||
| // column 1: snapshot_id (long) | ||
| ICEBERG_RETURN_UNEXPECTED(AppendInt(builder.column(1), snapshot->snapshot_id)); | ||
|
|
||
| // column 2: parent_id (long, optional) | ||
| if (snapshot->parent_snapshot_id.has_value()) { | ||
| ICEBERG_RETURN_UNEXPECTED( | ||
| AppendInt(builder.column(2), *snapshot->parent_snapshot_id)); | ||
| } else { | ||
| ICEBERG_RETURN_UNEXPECTED(AppendNull(builder.column(2))); | ||
| } | ||
|
|
||
| // column 3: operation (string, optional) | ||
| auto op = snapshot->Operation(); | ||
| if (op.has_value()) { | ||
| ICEBERG_RETURN_UNEXPECTED(AppendString(builder.column(3), *op)); | ||
| } else { | ||
| ICEBERG_RETURN_UNEXPECTED(AppendNull(builder.column(3))); | ||
| } | ||
|
|
||
| // column 4: manifest_list (string, optional) | ||
| ICEBERG_RETURN_UNEXPECTED(AppendString(builder.column(4), snapshot->manifest_list)); | ||
|
|
||
| // column 5: summary (map<string,string>) | ||
| ICEBERG_RETURN_UNEXPECTED(AppendStringMap(builder.column(5), snapshot->summary)); | ||
|
WZhuo marked this conversation as resolved.
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
|
|
||
| ICEBERG_RETURN_UNEXPECTED(builder.FinishRow()); | ||
| } | ||
|
|
||
| return std::move(builder).Finish(); | ||
| } | ||
|
|
||
| } // namespace iceberg | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -40,6 +40,13 @@ class ICEBERG_EXPORT SnapshotsTable : public MetadataTable { | |
|
|
||
| Kind kind() const noexcept override { return Kind::kSnapshots; } | ||
|
|
||
| /// \brief Scan all snapshots as rows. | ||
| /// | ||
| /// The snapshots table always returns every known snapshot, so the | ||
| /// snapshot_selection parameter is ignored. | ||
| Result<ArrowArray> Scan( | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This override hides the zero-argument |
||
| const std::optional<SnapshotSelection>& /*snapshot_selection*/) override; | ||
|
|
||
| private: | ||
| explicit SnapshotsTable(std::shared_ptr<Table> table); | ||
| }; | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| /// \file history_table_test.cc | ||
| /// Unit tests for HistoryTable. | ||
|
|
||
| #include <gmock/gmock.h> | ||
| #include <gtest/gtest.h> | ||
|
|
||
| #include "iceberg/inspect/metadata_table.h" | ||
| #include "iceberg/schema.h" | ||
| #include "iceberg/schema_field.h" | ||
| #include "iceberg/test/matchers.h" | ||
| #include "iceberg/test/metadata_table_test_base.h" | ||
| #include "iceberg/type.h" | ||
|
|
||
| namespace iceberg { | ||
| namespace { | ||
|
|
||
| std::shared_ptr<Schema> MakeHistorySchema() { | ||
| return std::make_shared<Schema>(std::vector<SchemaField>{ | ||
| SchemaField::MakeRequired(1, "made_current_at", timestamp_tz()), | ||
| SchemaField::MakeRequired(2, "snapshot_id", int64()), | ||
| SchemaField::MakeOptional(3, "parent_id", int64()), | ||
| SchemaField::MakeRequired(4, "is_current_ancestor", boolean())}); | ||
| } | ||
|
|
||
| } // namespace | ||
|
|
||
| class HistoryTableTest : public MetadataTableTestBase {}; | ||
|
|
||
| TEST_F(HistoryTableTest, SchemaMatchesIcebergSchema) { | ||
| ICEBERG_UNWRAP_OR_FAIL(auto history_table, | ||
| MetadataTable::Make(table_, MetadataTable::Kind::kHistory)); | ||
| EXPECT_TRUE(*history_table->schema() == *MakeHistorySchema()); | ||
| } | ||
|
|
||
| } // namespace iceberg |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Should it be
std::variant<std::monostate, int64_t, TimePointMs>? Otherwise we need to define the priority if bothsnapshot_idandas_of_timestampare set.