diff --git a/packages/google-cloud-bigtable/samples/admin_client/__init__.py b/packages/google-cloud-bigtable/samples/admin_client/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/__init__.py b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/instanceadmin.py b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/instanceadmin.py new file mode 100644 index 000000000000..964d2d82e923 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/instanceadmin.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python + +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Demonstrates how to run instance admin operations using BigtableInstanceAdminClient. + +Prerequisites: +- Create a Cloud Bigtable project. +- Set your Google Application Default Credentials. +""" + +import argparse + + +def run_instance_operations(project_id, instance_id, cluster_id): + # [START bigtable_check_instance_exists] + from google.api_core.exceptions import NotFound + + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableInstanceAdminClient() + instance_path = client.instance_path(project_id, instance_id) + + try: + instance = client.get_instance(name=instance_path) + print(f"Instance {instance_id} already exists.") + except NotFound: + print(f"Instance {instance_id} does not exist.") + # [END bigtable_check_instance_exists] + + # [START bigtable_create_prod_instance] + from google.api_core.exceptions import NotFound + + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableInstanceAdminClient() + project_path = client.common_project_path(project_id) + instance_path = client.instance_path(project_id, instance_id) + + cluster = bigtable_admin.Cluster( + location=client.common_location_path(project_id, "us-central1-f"), + serve_nodes=1, + default_storage_type=bigtable_admin.StorageType.SSD, + ) + instance_obj = bigtable_admin.Instance( + display_name=instance_id, + labels={"prod-label": "prod-label"}, + type_=bigtable_admin.Instance.Type.PRODUCTION, + ) + + try: + client.get_instance(name=instance_path) + except NotFound: + print("\nCreating an instance") + # Create instance with given options + operation = client.create_instance( + parent=project_path, + instance_id=instance_id, + instance=instance_obj, + clusters={cluster_id: cluster}, + ) + # Ensure the operation completes. + operation.result(timeout=480) + print(f"\nCreated instance: {instance_id}") + # [END bigtable_create_prod_instance] + + # [START bigtable_list_instances] + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableInstanceAdminClient() + project_path = client.common_project_path(project_id) + + print("\nListing instances:") + instances_response = client.list_instances(parent=project_path) + for instance in instances_response.instances: + print(instance.name.split("/")[-1]) + # [END bigtable_list_instances] + + # [START bigtable_get_instance] + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableInstanceAdminClient() + instance_path = client.instance_path(project_id, instance_id) + + inst = client.get_instance(name=instance_path) + print(f"\nName of instance: {inst.display_name}\nLabels: {dict(inst.labels)}") + # [END bigtable_get_instance] + + # [START bigtable_get_clusters] + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableInstanceAdminClient() + instance_path = client.instance_path(project_id, instance_id) + + print("\nListing clusters...") + clusters_response = client.list_clusters(parent=instance_path) + for cluster in clusters_response.clusters: + print(cluster.name.split("/")[-1]) + # [END bigtable_get_clusters] + + +def delete_instance(project_id, instance_id): + # [START bigtable_delete_instance] + from google.api_core.exceptions import NotFound + + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableInstanceAdminClient() + instance_path = client.instance_path(project_id, instance_id) + + print("\nDeleting instance") + try: + client.delete_instance(name=instance_path) + print(f"Deleted instance: {instance_id}") + except NotFound: + print(f"Instance {instance_id} does not exist.") + # [END bigtable_delete_instance] + + +def add_cluster(project_id, instance_id, cluster_id): + # [START bigtable_create_cluster] + from google.api_core.exceptions import NotFound + + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableInstanceAdminClient() + instance_path = client.instance_path(project_id, instance_id) + cluster_path = client.cluster_path(project_id, instance_id, cluster_id) + + print("\nListing clusters...") + clusters_response = client.list_clusters(parent=instance_path) + for cluster in clusters_response.clusters: + print(cluster.name.split("/")[-1]) + + new_cluster = bigtable_admin.Cluster( + location=client.common_location_path(project_id, "us-central1-a"), + serve_nodes=1, + default_storage_type=bigtable_admin.StorageType.SSD, + ) + try: + client.get_cluster(name=cluster_path) + print(f"\nCluster not created, as {cluster_id} already exists.") + except NotFound: + operation = client.create_cluster( + parent=instance_path, cluster_id=cluster_id, cluster=new_cluster + ) + # Ensure the operation completes. + operation.result(timeout=480) + print(f"\nCluster created: {cluster_id}") + # [END bigtable_create_cluster] + + +def delete_cluster(project_id, instance_id, cluster_id): + """Delete the cluster + + :type project_id: str + :param project_id: Project id of the client. + + :type instance_id: str + :param instance_id: Instance of the client. + + :type cluster_id: str + :param cluster_id: Cluster id. + """ + # [START bigtable_delete_cluster] + from google.api_core.exceptions import NotFound + + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableInstanceAdminClient() + cluster_path = client.cluster_path(project_id, instance_id, cluster_id) + + print("\nDeleting cluster") + try: + client.delete_cluster(name=cluster_path) + print(f"Cluster deleted: {cluster_id}") + except NotFound: + print(f"\nCluster {cluster_id} does not exist.") + # [END bigtable_delete_cluster] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + "command", + help="run, del-instance, add-cluster or del-cluster.", + ) + parser.add_argument("project_id", help="Your Cloud Platform project ID.") + parser.add_argument("instance_id", help="ID of the Cloud Bigtable instance.") + parser.add_argument("cluster_id", help="ID of the Cloud Bigtable cluster.") + + args = parser.parse_args() + + if args.command.lower() == "run": + run_instance_operations(args.project_id, args.instance_id, args.cluster_id) + elif args.command.lower() == "del-instance": + delete_instance(args.project_id, args.instance_id) + elif args.command.lower() == "add-cluster": + add_cluster(args.project_id, args.instance_id, args.cluster_id) + elif args.command.lower() == "del-cluster": + delete_cluster(args.project_id, args.instance_id, args.cluster_id) diff --git a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/requirements.txt b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/requirements.txt new file mode 100644 index 000000000000..bde485d13258 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-bigtable +backoff==2.2.1 diff --git a/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/test_instanceadmin.py b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/test_instanceadmin.py new file mode 100644 index 000000000000..1c5203b1a680 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/instanceadmin/test_instanceadmin.py @@ -0,0 +1,54 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid + +import backoff +from google.api_core import exceptions + +from .instanceadmin import ( + add_cluster, + delete_cluster, + delete_instance, + run_instance_operations, +) + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +INSTANCE_ID = f"instance-admin-{str(uuid.uuid4())[:8]}" +CLUSTER_ID = f"cluster-admin-{str(uuid.uuid4())[:8]}" +NEW_CLUSTER_ID = f"cluster-add-{str(uuid.uuid4())[:8]}" + + +def test_instance_operations(capsys): + try: + run_instance_operations(PROJECT, INSTANCE_ID, CLUSTER_ID) + out, _ = capsys.readouterr() + assert f"Created instance: {INSTANCE_ID}" in out + assert "Listing instances:" in out + assert "Listing clusters..." in out + + backoff_503 = backoff.on_exception(backoff.expo, exceptions.ServiceUnavailable) + backoff_503(add_cluster)(PROJECT, INSTANCE_ID, NEW_CLUSTER_ID) + out, _ = capsys.readouterr() + assert f"Cluster created: {NEW_CLUSTER_ID}" in out + + delete_cluster(PROJECT, INSTANCE_ID, NEW_CLUSTER_ID) + out, _ = capsys.readouterr() + assert f"Cluster deleted: {NEW_CLUSTER_ID}" in out + + delete_instance(PROJECT, INSTANCE_ID) + out, _ = capsys.readouterr() + assert f"Deleted instance: {INSTANCE_ID}" in out + finally: + delete_instance(PROJECT, INSTANCE_ID) diff --git a/packages/google-cloud-bigtable/samples/admin_client/metricscaler/__init__.py b/packages/google-cloud-bigtable/samples/admin_client/metricscaler/__init__.py new file mode 100644 index 000000000000..58d482ea3866 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/metricscaler/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/packages/google-cloud-bigtable/samples/admin_client/metricscaler/metricscaler.py b/packages/google-cloud-bigtable/samples/admin_client/metricscaler/metricscaler.py new file mode 100644 index 000000000000..252cd3e637b8 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/metricscaler/metricscaler.py @@ -0,0 +1,241 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sample that demonstrates how to use Cloud Monitoring metrics to +programmatically scale a Google Cloud Bigtable cluster.""" + +import argparse +import logging +import os +import time + +from google.cloud.monitoring_v3 import query +from google.protobuf import field_mask_pb2 + +from google.cloud import bigtable_admin, monitoring_v3 + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] + +logger = logging.getLogger("bigtable.metricscaler") +logger.addHandler(logging.StreamHandler()) +logger.setLevel(logging.INFO) + + +def get_cpu_load(bigtable_instance, bigtable_cluster): + """Returns the most recent Cloud Bigtable CPU load measurement. + + Returns: + float: The most recent Cloud Bigtable CPU usage metric + """ + # [START bigtable_cpu] + client = monitoring_v3.MetricServiceClient() + cpu_query = query.Query( + client, + project=PROJECT, + metric_type="bigtable.googleapis.com/cluster/cpu_load", + minutes=5, + ) + cpu_query = cpu_query.select_resources( + instance=bigtable_instance, cluster=bigtable_cluster + ) + cpu = next(cpu_query.iter()) + return cpu.points[0].value.double_value + # [END bigtable_cpu] + + +def get_storage_utilization(bigtable_instance, bigtable_cluster): + """Returns the most recent Cloud Bigtable storage utilization measurement. + + Returns: + float: The most recent Cloud Bigtable storage utilization metric + """ + # [START bigtable_metric_scaler_storage_utilization] + client = monitoring_v3.MetricServiceClient() + utilization_query = query.Query( + client, + project=PROJECT, + metric_type="bigtable.googleapis.com/cluster/storage_utilization", + minutes=5, + ) + utilization_query = utilization_query.select_resources( + instance=bigtable_instance, cluster=bigtable_cluster + ) + utilization = next(utilization_query.iter()) + return utilization.points[0].value.double_value + # [END bigtable_metric_scaler_storage_utilization] + + +def scale_bigtable(bigtable_instance, bigtable_cluster, scale_up): + """Scales the number of Cloud Bigtable nodes up or down. + + Edits the number of nodes in the Cloud Bigtable cluster to be increased + or decreased, depending on the `scale_up` boolean argument. Currently + the `incremental` strategy from `strategies.py` is used. + + + Args: + bigtable_instance (str): Cloud Bigtable instance ID to scale + bigtable_cluster (str): Cloud Bigtable cluster ID to scale + scale_up (bool): If true, scale up, otherwise scale down + """ + + # The minimum number of nodes to use. The default minimum is 3. If you have + # a lot of data, the rule of thumb is to not go below 2.5 TB per node for + # SSD clusters, and 8 TB for HDD. The + # "bigtable.googleapis.com/disk/bytes_used" metric is useful in figuring + # out the minimum number of nodes. + min_node_count = 1 + + # The maximum number of nodes to use. The default maximum is 30 nodes per + # zone. If you need more quota, you can request more by following the + # instructions at https://cloud.google.com/bigtable/quota. + max_node_count = 30 + + # The number of nodes to change the cluster by. + size_change_step = 3 + + # [START bigtable_scale] + bigtable_client = bigtable_admin.BigtableInstanceAdminClient() + instance_name = bigtable_client.instance_path(PROJECT, bigtable_instance) + instance = bigtable_client.get_instance(name=instance_name) + + if instance.type_ == bigtable_admin.Instance.Type.DEVELOPMENT: + raise ValueError("Development instances cannot be scaled.") + + cluster_name = bigtable_client.cluster_path( + PROJECT, bigtable_instance, bigtable_cluster + ) + cluster = bigtable_client.get_cluster(name=cluster_name) + + current_node_count = cluster.serve_nodes + update_mask = field_mask_pb2.FieldMask(paths=["serve_nodes"]) + + if scale_up: + if current_node_count < max_node_count: + new_node_count = min(current_node_count + size_change_step, max_node_count) + cluster.serve_nodes = new_node_count + operation = bigtable_client.partial_update_cluster( + cluster=cluster, update_mask=update_mask + ) + response = operation.result(480) + logger.info( + "Scaled up from {} to {} nodes for {}.".format( + current_node_count, new_node_count, response.name + ) + ) + else: + if current_node_count > min_node_count: + new_node_count = max(current_node_count - size_change_step, min_node_count) + cluster.serve_nodes = new_node_count + operation = bigtable_client.partial_update_cluster( + cluster=cluster, update_mask=update_mask + ) + response = operation.result(480) + logger.info( + "Scaled down from {} to {} nodes for {}.".format( + current_node_count, new_node_count, response.name + ) + ) + # [END bigtable_scale] + + +def main( + bigtable_instance, + bigtable_cluster, + high_cpu_threshold, + low_cpu_threshold, + high_storage_threshold, + short_sleep, + long_sleep, +): + """Main loop runner that autoscales Cloud Bigtable. + + Args: + bigtable_instance (str): Cloud Bigtable instance ID to autoscale + high_cpu_threshold (float): If CPU is higher than this, scale up. + low_cpu_threshold (float): If CPU is lower than this, scale down. + high_storage_threshold (float): If storage is higher than this, + scale up. + short_sleep (int): How long to sleep after no operation + long_sleep (int): How long to sleep after the number of nodes is + changed + """ + cluster_cpu = get_cpu_load(bigtable_instance, bigtable_cluster) + cluster_storage = get_storage_utilization(bigtable_instance, bigtable_cluster) + logger.info("Detected cpu of {}".format(cluster_cpu)) + logger.info("Detected storage utilization of {}".format(cluster_storage)) + try: + if cluster_cpu > high_cpu_threshold or cluster_storage > high_storage_threshold: + scale_bigtable(bigtable_instance, bigtable_cluster, True) + time.sleep(long_sleep) + elif cluster_cpu < low_cpu_threshold: + if cluster_storage < high_storage_threshold: + scale_bigtable(bigtable_instance, bigtable_cluster, False) + time.sleep(long_sleep) + else: + logger.info("CPU within threshold, sleeping.") + time.sleep(short_sleep) + except Exception as e: + logger.error("Error during scaling: %s", e) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Scales Cloud Bigtable clusters based on CPU usage." + ) + parser.add_argument( + "bigtable_instance", help="ID of the Cloud Bigtable instance to connect to." + ) + parser.add_argument( + "bigtable_cluster", help="ID of the Cloud Bigtable cluster to connect to." + ) + parser.add_argument( + "--high_cpu_threshold", + help="If Cloud Bigtable CPU usage is above this threshold, scale up", + default=0.6, + ) + parser.add_argument( + "--low_cpu_threshold", + help="If Cloud Bigtable CPU usage is below this threshold, scale down", + default=0.2, + ) + parser.add_argument( + "--high_storage_threshold", + help="If Cloud Bigtable storage utilization is above this threshold, scale up", + default=0.6, + ) + parser.add_argument( + "--short_sleep", + help="How long to sleep in seconds between checking metrics after no " + "scale operation", + default=60, + ) + parser.add_argument( + "--long_sleep", + help="How long to sleep in seconds between checking metrics after a " + "scaling operation", + default=60 * 10, + ) + args = parser.parse_args() + + while True: + main( + args.bigtable_instance, + args.bigtable_cluster, + float(args.high_cpu_threshold), + float(args.low_cpu_threshold), + float(args.high_storage_threshold), + int(args.short_sleep), + int(args.long_sleep), + ) diff --git a/packages/google-cloud-bigtable/samples/admin_client/metricscaler/metricscaler_test.py b/packages/google-cloud-bigtable/samples/admin_client/metricscaler/metricscaler_test.py new file mode 100644 index 000000000000..abb0b108bc4c --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/metricscaler/metricscaler_test.py @@ -0,0 +1,235 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit and system tests for metricscaler.py""" + +import os +import uuid +from unittest.mock import Mock, patch + +import pytest +from google.api_core.exceptions import NotFound +from test_utils.retry import RetryResult + +from google.cloud import bigtable_admin + +from . import metricscaler +from .metricscaler import get_cpu_load, get_storage_utilization, main, scale_bigtable + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +BIGTABLE_ZONE = os.environ.get("BIGTABLE_ZONE", "us-central1-b") +SIZE_CHANGE_STEP = 3 +INSTANCE_ID_FORMAT = "metric-scale-test-{}" +BIGTABLE_INSTANCE = INSTANCE_ID_FORMAT.format(str(uuid.uuid4())[:10]) +BIGTABLE_DEV_INSTANCE = INSTANCE_ID_FORMAT.format(str(uuid.uuid4())[:10]) + + +# System tests to verify API calls succeed + + +@patch.object(metricscaler, "query") +def test_get_cpu_load(monitoring_v3_query): + iter_mock = monitoring_v3_query.Query().select_resources().iter + iter_mock.return_value = iter([Mock(points=[Mock(value=Mock(double_value=1.0))])]) + assert float(get_cpu_load(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE)) > 0.0 + + +@patch.object(metricscaler, "query") +def test_get_storage_utilization(monitoring_v3_query): + iter_mock = monitoring_v3_query.Query().select_resources().iter + iter_mock.return_value = iter([Mock(points=[Mock(value=Mock(double_value=1.0))])]) + assert float(get_storage_utilization(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE)) > 0.0 + + +@pytest.fixture() +def instance(): + cluster_id = BIGTABLE_INSTANCE + + client = bigtable_admin.BigtableInstanceAdminClient() + project_path = client.common_project_path(PROJECT) + instance_path = client.instance_path(PROJECT, BIGTABLE_INSTANCE) + + cluster = bigtable_admin.Cluster( + location=client.common_location_path(PROJECT, BIGTABLE_ZONE), + serve_nodes=1, + default_storage_type=bigtable_admin.StorageType.SSD, + ) + instance_obj = bigtable_admin.Instance( + display_name=BIGTABLE_INSTANCE, + type_=bigtable_admin.Instance.Type.PRODUCTION, + labels={"prod-label": "prod-label"}, + ) + + try: + client.get_instance(name=instance_path) + except NotFound: + operation = client.create_instance( + parent=project_path, + instance_id=BIGTABLE_INSTANCE, + instance=instance_obj, + clusters={cluster_id: cluster}, + ) + response = operation.result(480) + print(f"Successfully created {response.name}") + + yield + + client.delete_instance(name=instance_path) + + +@pytest.fixture() +def dev_instance(): + cluster_id = BIGTABLE_DEV_INSTANCE + + client = bigtable_admin.BigtableInstanceAdminClient() + project_path = client.common_project_path(PROJECT) + instance_path = client.instance_path(PROJECT, BIGTABLE_DEV_INSTANCE) + + cluster = bigtable_admin.Cluster( + location=client.common_location_path(PROJECT, BIGTABLE_ZONE), + default_storage_type=bigtable_admin.StorageType.SSD, + ) + instance_obj = bigtable_admin.Instance( + display_name=BIGTABLE_DEV_INSTANCE, + type_=bigtable_admin.Instance.Type.DEVELOPMENT, + labels={"dev-label": "dev-label"}, + ) + + try: + client.get_instance(name=instance_path) + except NotFound: + operation = client.create_instance( + parent=project_path, + instance_id=BIGTABLE_DEV_INSTANCE, + instance=instance_obj, + clusters={cluster_id: cluster}, + ) + response = operation.result(480) + print(f"Successfully created {response.name}") + + yield + + client.delete_instance(name=instance_path) + + +class ClusterNodeCountPredicate: + def __init__(self, expected_node_count): + self.expected_node_count = expected_node_count + + def __call__(self, cluster): + expected = self.expected_node_count + print(f"Expected node count: {expected}; found: {cluster.serve_nodes}") + return cluster.serve_nodes == expected + + +def test_scale_bigtable(instance): + bigtable_client = bigtable_admin.BigtableInstanceAdminClient() + cluster_path = bigtable_client.cluster_path( + PROJECT, BIGTABLE_INSTANCE, BIGTABLE_INSTANCE + ) + + def get_cluster(): + return bigtable_client.get_cluster(name=cluster_path) + + _nonzero_node_count = RetryResult( + lambda c: c.serve_nodes > 0, + max_tries=10, + ) + cluster = _nonzero_node_count(get_cluster)() + + original_node_count = cluster.serve_nodes + + scale_bigtable(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, True) + + scaled_node_count_predicate = ClusterNodeCountPredicate( + original_node_count + SIZE_CHANGE_STEP + ) + scaled_node_count_predicate.__name__ = "scaled_node_count_predicate" + _scaled_node_count = RetryResult( + scaled_node_count_predicate, + max_tries=10, + ) + _scaled_node_count(get_cluster)() + + scale_bigtable(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, False) + + restored_node_count_predicate = ClusterNodeCountPredicate(original_node_count) + restored_node_count_predicate.__name__ = "restored_node_count_predicate" + _restored_node_count = RetryResult( + restored_node_count_predicate, + max_tries=10, + ) + _restored_node_count(get_cluster)() + + +def test_handle_dev_instance(capsys, dev_instance): + with pytest.raises(ValueError): + scale_bigtable(BIGTABLE_DEV_INSTANCE, BIGTABLE_DEV_INSTANCE, True) + + +@patch("time.sleep") +@patch.object(metricscaler, "get_storage_utilization") +@patch.object(metricscaler, "get_cpu_load") +@patch.object(metricscaler, "scale_bigtable") +def test_main(scale_bigtable, get_cpu_load, get_storage_utilization, sleep): + SHORT_SLEEP = 5 + LONG_SLEEP = 10 + + # Test okay CPU, okay storage utilization + get_cpu_load.return_value = 0.5 + get_storage_utilization.return_value = 0.5 + + main(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, 0.6, 0.3, 0.6, SHORT_SLEEP, LONG_SLEEP) + scale_bigtable.assert_not_called() + scale_bigtable.reset_mock() + + # Test high CPU, okay storage utilization + get_cpu_load.return_value = 0.7 + get_storage_utilization.return_value = 0.5 + main(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, 0.6, 0.3, 0.6, SHORT_SLEEP, LONG_SLEEP) + scale_bigtable.assert_called_once_with(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, True) + scale_bigtable.reset_mock() + + # Test low CPU, okay storage utilization + get_storage_utilization.return_value = 0.5 + get_cpu_load.return_value = 0.2 + main(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, 0.6, 0.3, 0.6, SHORT_SLEEP, LONG_SLEEP) + scale_bigtable.assert_called_once_with(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, False) + scale_bigtable.reset_mock() + + # Test okay CPU, high storage utilization + get_cpu_load.return_value = 0.5 + get_storage_utilization.return_value = 0.7 + + main(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, 0.6, 0.3, 0.6, SHORT_SLEEP, LONG_SLEEP) + scale_bigtable.assert_called_once_with(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, True) + scale_bigtable.reset_mock() + + # Test high CPU, high storage utilization + get_cpu_load.return_value = 0.7 + get_storage_utilization.return_value = 0.7 + main(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, 0.6, 0.3, 0.6, SHORT_SLEEP, LONG_SLEEP) + scale_bigtable.assert_called_once_with(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, True) + scale_bigtable.reset_mock() + + # Test low CPU, high storage utilization + get_cpu_load.return_value = 0.2 + get_storage_utilization.return_value = 0.7 + main(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, 0.6, 0.3, 0.6, SHORT_SLEEP, LONG_SLEEP) + scale_bigtable.assert_called_once_with(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, True) + scale_bigtable.reset_mock() + + +if __name__ == "__main__": + test_get_cpu_load() diff --git a/packages/google-cloud-bigtable/samples/admin_client/metricscaler/requirements.txt b/packages/google-cloud-bigtable/samples/admin_client/metricscaler/requirements.txt new file mode 100644 index 000000000000..4a2b04016e67 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/metricscaler/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-bigtable +google-cloud-monitoring==2.29.0 diff --git a/packages/google-cloud-bigtable/samples/admin_client/tableadmin/__init__.py b/packages/google-cloud-bigtable/samples/admin_client/tableadmin/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/google-cloud-bigtable/samples/admin_client/tableadmin/requirements.txt b/packages/google-cloud-bigtable/samples/admin_client/tableadmin/requirements.txt new file mode 100644 index 000000000000..47acfcb87fb0 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/tableadmin/requirements.txt @@ -0,0 +1 @@ +google-cloud-bigtable diff --git a/packages/google-cloud-bigtable/samples/admin_client/tableadmin/tableadmin.py b/packages/google-cloud-bigtable/samples/admin_client/tableadmin/tableadmin.py new file mode 100644 index 000000000000..c82a4ace9228 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/tableadmin/tableadmin.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python + +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Demonstrates how to connect to Cloud Bigtable and run table admin operations using BigtableTableAdminClient. + +Prerequisites: +- Create a Cloud Bigtable instance. +- Set your Google Application Default Credentials. +""" + +import argparse + +from ...utils import create_table_cm + + +def run_table_operations(project_id, instance_id, table_id): + with create_table_cm(project_id, instance_id, table_id, verbose=False): + # [START bigtable_list_tables] + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableTableAdminClient() + instance_path = client.instance_path(project_id, instance_id) + + tables = client.list_tables(parent=instance_path) + print("Listing tables in current project...") + if tables.tables: + for tbl in tables.tables: + print(tbl.name.split("/")[-1]) + else: + print("No table exists in current project...") + # [END bigtable_list_tables] + + # [START bigtable_create_family_gc_max_age] + from google.protobuf import duration_pb2 + + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableTableAdminClient() + table_path = client.table_path(project_id, instance_id, table_id) + + print("Creating column family cf1 with MaxAge GC Rule...") + # Create a column family with GC policy : maximum age + # where age = current time minus cell timestamp + + # Define the GC rule to retain data with max age of 5 days + max_age_rule = bigtable_admin.GcRule( + max_age=duration_pb2.Duration(seconds=5 * 86400) + ) + cf1 = bigtable_admin.ColumnFamily(gc_rule=max_age_rule) + mod1 = bigtable_admin.ModifyColumnFamiliesRequest.Modification( + id="cf1", create=cf1 + ) + client.modify_column_families(name=table_path, modifications=[mod1]) + print("Created column family cf1 with MaxAge GC Rule.") + # [END bigtable_create_family_gc_max_age] + + # [START bigtable_create_family_gc_max_versions] + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableTableAdminClient() + table_path = client.table_path(project_id, instance_id, table_id) + + print("Creating column family cf2 with max versions GC rule...") + # Create a column family with GC policy : most recent N versions + # where 1 = most recent version + + # Define the GC policy to retain only the most recent 2 versions + max_versions_rule = bigtable_admin.GcRule(max_num_versions=2) + cf2 = bigtable_admin.ColumnFamily(gc_rule=max_versions_rule) + mod2 = bigtable_admin.ModifyColumnFamiliesRequest.Modification( + id="cf2", create=cf2 + ) + client.modify_column_families(name=table_path, modifications=[mod2]) + print("Created column family cf2 with Max Versions GC Rule.") + # [END bigtable_create_family_gc_max_versions] + + # [START bigtable_create_family_gc_union] + from google.protobuf import duration_pb2 + + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableTableAdminClient() + table_path = client.table_path(project_id, instance_id, table_id) + + print("Creating column family cf3 with union GC rule...") + # Create a column family with GC policy to drop data that matches + # at least one condition. + # Define a GC rule to drop cells older than 5 days or not the + # most recent version + union_rule = bigtable_admin.GcRule( + union=bigtable_admin.GcRule.Union( + rules=[ + bigtable_admin.GcRule( + max_age=duration_pb2.Duration(seconds=5 * 86400) + ), + bigtable_admin.GcRule(max_num_versions=2), + ] + ) + ) + cf3 = bigtable_admin.ColumnFamily(gc_rule=union_rule) + mod3 = bigtable_admin.ModifyColumnFamiliesRequest.Modification( + id="cf3", create=cf3 + ) + client.modify_column_families(name=table_path, modifications=[mod3]) + print("Created column family cf3 with Union GC rule") + # [END bigtable_create_family_gc_union] + + # [START bigtable_create_family_gc_intersection] + from google.protobuf import duration_pb2 + + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableTableAdminClient() + table_path = client.table_path(project_id, instance_id, table_id) + + print("Creating column family cf4 with Intersection GC rule...") + # Create a column family with GC policy to drop data that matches + # all conditions + # GC rule: Drop cells older than 5 days AND older than the most + # recent 2 versions + intersection_rule = bigtable_admin.GcRule( + intersection=bigtable_admin.GcRule.Intersection( + rules=[ + bigtable_admin.GcRule( + max_age=duration_pb2.Duration(seconds=5 * 86400) + ), + bigtable_admin.GcRule(max_num_versions=2), + ] + ) + ) + cf4 = bigtable_admin.ColumnFamily(gc_rule=intersection_rule) + mod4 = bigtable_admin.ModifyColumnFamiliesRequest.Modification( + id="cf4", create=cf4 + ) + client.modify_column_families(name=table_path, modifications=[mod4]) + print("Created column family cf4 with Intersection GC rule.") + # [END bigtable_create_family_gc_intersection] + + # [START bigtable_create_family_gc_nested] + from google.protobuf import duration_pb2 + + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableTableAdminClient() + table_path = client.table_path(project_id, instance_id, table_id) + + print("Creating column family cf5 with a Nested GC rule...") + # Create a column family with nested GC policies. + # Create a nested GC rule: + # Drop cells that are either older than the 10 recent versions + # OR + # Drop cells that are older than a month AND older than the + # 2 recent versions + rule1 = bigtable_admin.GcRule(max_num_versions=10) + rule2 = bigtable_admin.GcRule( + intersection=bigtable_admin.GcRule.Intersection( + rules=[ + bigtable_admin.GcRule( + max_age=duration_pb2.Duration(seconds=30 * 86400) + ), + bigtable_admin.GcRule(max_num_versions=2), + ] + ) + ) + nested_rule = bigtable_admin.GcRule( + union=bigtable_admin.GcRule.Union(rules=[rule1, rule2]) + ) + cf5 = bigtable_admin.ColumnFamily(gc_rule=nested_rule) + mod5 = bigtable_admin.ModifyColumnFamiliesRequest.Modification( + id="cf5", create=cf5 + ) + client.modify_column_families(name=table_path, modifications=[mod5]) + print("Created column family cf5 with a Nested GC rule.") + # [END bigtable_create_family_gc_nested] + + # [START bigtable_list_column_families] + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableTableAdminClient() + table_path = client.table_path(project_id, instance_id, table_id) + + print("Printing Column Family and GC Rule for all column families...") + table_obj = client.get_table(name=table_path) + for column_family_name, cf_obj in sorted(table_obj.column_families.items()): + print("Column Family:", column_family_name) + print("GC Rule:") + print(cf_obj.gc_rule) + # Sample output: + # Column Family: cf4 + # GC Rule: + # intersection { + # rules { + # max_age { + # seconds: 432000 + # } + # } + # rules { + # max_num_versions: 2 + # } + # } + # [END bigtable_list_column_families] + + # [START bigtable_update_gc_rule] + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableTableAdminClient() + table_path = client.table_path(project_id, instance_id, table_id) + + print("Updating column family cf1 GC rule...") + # Update the column family cf1 to update the GC rule + updated_cf1 = bigtable_admin.ColumnFamily( + gc_rule=bigtable_admin.GcRule(max_num_versions=1) + ) + mod_update = bigtable_admin.ModifyColumnFamiliesRequest.Modification( + id="cf1", update=updated_cf1 + ) + client.modify_column_families(name=table_path, modifications=[mod_update]) + print("Updated column family cf1 GC rule\n") + # [END bigtable_update_gc_rule] + + # [START bigtable_delete_family] + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableTableAdminClient() + table_path = client.table_path(project_id, instance_id, table_id) + + print("Delete a column family cf2...") + # Delete a column family + mod_delete = bigtable_admin.ModifyColumnFamiliesRequest.Modification( + id="cf2", drop=True + ) + client.modify_column_families(name=table_path, modifications=[mod_delete]) + print("Column family cf2 deleted successfully.") + # [END bigtable_delete_family] + + print( + 'execute command "python tableadmin.py delete [project_id] \ + [instance_id] --table [tableName]" to delete the table.' + ) + + +def delete_table(project_id, instance_id, table_id): + """Delete bigtable. + + :type project_id: str + :param project_id: Project id of the client. + + :type instance_id: str + :param instance_id: Instance of the client. + + :type table_id: str + :param table_id: Table id to create table. + """ + # [START bigtable_delete_table] + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableTableAdminClient() + table_path = client.table_path(project_id, instance_id, table_id) + + # Delete the entire table + print("Deleting {} table.".format(table_id)) + client.delete_table(name=table_path) + print("Deleted {} table.".format(table_id)) + # [END bigtable_delete_table] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument("command", help="run or delete.") + parser.add_argument( + "--table", help="Cloud Bigtable Table name.", default="Hello-Bigtable" + ) + parser.add_argument("project_id", help="Your Cloud Platform project ID.") + parser.add_argument( + "instance_id", help="ID of the Cloud Bigtable instance to connect to." + ) + + args = parser.parse_args() + + if args.command.lower() == "run": + run_table_operations(args.project_id, args.instance_id, args.table) + elif args.command.lower() == "delete": + delete_table(args.project_id, args.instance_id, args.table) + else: + print( + "Command should be either run or delete.\n Use argument -h,\ + --help to show help and exit." + ) diff --git a/packages/google-cloud-bigtable/samples/admin_client/tableadmin/tableadmin_test.py b/packages/google-cloud-bigtable/samples/admin_client/tableadmin/tableadmin_test.py new file mode 100644 index 000000000000..ea3c4cbc17d1 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/admin_client/tableadmin/tableadmin_test.py @@ -0,0 +1,45 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid + +from ...utils import create_table_cm +from .tableadmin import delete_table, run_table_operations + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] +TABLE_ID = f"table-admin-{str(uuid.uuid4())[:16]}" + + +def test_run_table_operations(capsys): + run_table_operations(PROJECT, BIGTABLE_INSTANCE, TABLE_ID) + out, _ = capsys.readouterr() + assert "Listing tables in current project..." in out + assert "Created column family cf1 with MaxAge GC Rule." in out + assert "Created column family cf2 with Max Versions GC Rule." in out + assert "Created column family cf3 with Union GC rule" in out + assert "Created column family cf4 with Intersection GC rule." in out + assert "Created column family cf5 with a Nested GC rule." in out + assert "Printing Column Family and GC Rule for all column families..." in out + assert "Updated column family cf1 GC rule" in out + assert "Column family cf2 deleted successfully." in out + + +def test_delete_table(capsys): + table_id = f"table-admin-to-delete-{str(uuid.uuid4())[:16]}" + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, table_id, verbose=False): + delete_table(PROJECT, BIGTABLE_INSTANCE, table_id) + out, _ = capsys.readouterr() + assert f"Deleting {table_id} table." in out + assert f"Deleted {table_id} table." in out diff --git a/packages/google-cloud-bigtable/samples/data_client/__init__.py b/packages/google-cloud-bigtable/samples/data_client/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/google-cloud-bigtable/samples/data_client/hello/__init__.py b/packages/google-cloud-bigtable/samples/data_client/hello/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/google-cloud-bigtable/samples/data_client/hello/main.py b/packages/google-cloud-bigtable/samples/data_client/hello/main.py new file mode 100644 index 000000000000..64d0ce608277 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/hello/main.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Demonstrates how to connect to Cloud Bigtable and run some basic operations with the sync data APIs + +Prerequisites: + +- Create a Cloud Bigtable instance. + https://cloud.google.com/bigtable/docs/creating-instance +- Set your Google Application Default Credentials. + https://developers.google.com/identity/protocols/application-default-credentials +""" + +import argparse + +# [START bigtable_hw_imports] +from google.cloud import bigtable +from google.cloud.bigtable.data import row_filters + +# [END bigtable_hw_imports] +from ...utils import wait_for_table + +# use to ignore warnings +row_filters + + +def main(project_id, instance_id, table_id): + # [START bigtable_hw_connect] + client = bigtable.data.BigtableDataClient(project=project_id) + table = client.get_table(instance_id, table_id) + # [END bigtable_hw_connect] + + # [START bigtable_hw_create_table] + from google.api_core.exceptions import AlreadyExists + + from google.cloud import bigtable_admin + + # the data client only supports the data API. Table creation is an admin operation + # use admin client to create the table + print("Creating the {} table.".format(table_id)) + admin_client = bigtable_admin.BigtableTableAdminClient() + instance_path = admin_client.instance_path(project_id, instance_id) + + print("Creating column family cf1 with Max Version GC rule...") + # Create a column family with GC policy : most recent N versions + # Define the GC policy to retain only the most recent 2 versions + max_versions_rule = bigtable_admin.GcRule(max_num_versions=2) + column_family_id = "cf1" + column_families = { + column_family_id: bigtable_admin.ColumnFamily(gc_rule=max_versions_rule) + } + try: + admin_client.create_table( + parent=instance_path, + table_id=table_id, + table=bigtable_admin.Table(column_families=column_families), + ) + except AlreadyExists: + print("Table {} already exists.".format(table_id)) + # [END bigtable_hw_create_table] + + try: + # let table creation complete + wait_for_table(table) + # [START bigtable_hw_write_rows] + print("Writing some greetings to the table.") + greetings = [b"Hello World!", b"Hello Cloud Bigtable!", b"Hello Python!"] + mutations = [] + column = b"greeting" + for i, value in enumerate(greetings): + # Note: This example uses sequential numeric IDs for simplicity, + # but this can result in poor performance in a production + # application. Since rows are stored in sorted order by key, + # sequential keys can result in poor distribution of operations + # across nodes. + # + # We recommend that you use bytestrings directly for row keys + # where possible, rather than encoding strings. + # + # For more information about how to design a Bigtable schema for + # the best performance, see the documentation: + # + # https://cloud.google.com/bigtable/docs/schema-design + row_key = f"greeting{i}".encode() + row_mutation = bigtable.data.RowMutationEntry( + row_key, bigtable.data.SetCell(column_family_id, column, value) + ) + mutations.append(row_mutation) + table.bulk_mutate_rows(mutations) + # [END bigtable_hw_write_rows] + + # [START bigtable_hw_create_filter] + # Create a filter to only retrieve the most recent version of the cell + # for each column across entire row. + row_filter = bigtable.data.row_filters.CellsColumnLimitFilter(1) + # [END bigtable_hw_create_filter] + + # [START bigtable_hw_get_with_filter] + # [START bigtable_hw_get_by_key] + print("Getting a single greeting by row key.") + key = "greeting0".encode() + + row = table.read_row(key, row_filter=row_filter) + cell = row.cells[0] + print(cell.value.decode("utf-8")) + # [END bigtable_hw_get_by_key] + # [END bigtable_hw_get_with_filter] + + # [START bigtable_hw_scan_with_filter] + # [START bigtable_hw_scan_all] + print("Scanning for all greetings:") + query = bigtable.data.ReadRowsQuery(row_filter=row_filter) + for row in table.read_rows_stream(query): + cell = row.cells[0] + print(cell.value.decode("utf-8")) + # [END bigtable_hw_scan_all] + # [END bigtable_hw_scan_with_filter] + finally: + # [START bigtable_hw_delete_table] + from google.cloud import bigtable_admin + + # the data client only supports the data API. Table deletion is an admin operation + # use admin client to delete the table + admin_client = bigtable_admin.BigtableTableAdminClient() + table_path = admin_client.table_path(project_id, instance_id, table_id) + + print("Deleting the {} table.".format(table_id)) + admin_client.delete_table(name=table_path) + table.close() + client.close() + # [END bigtable_hw_delete_table] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument("project_id", help="Your Cloud Platform project ID.") + parser.add_argument( + "instance_id", help="ID of the Cloud Bigtable instance to connect to." + ) + parser.add_argument( + "--table", help="Table to create and destroy.", default="Hello-Bigtable" + ) + + args = parser.parse_args() + main(args.project_id, args.instance_id, args.table) diff --git a/packages/google-cloud-bigtable/samples/data_client/hello/main_test.py b/packages/google-cloud-bigtable/samples/data_client/hello/main_test.py new file mode 100644 index 000000000000..77cab55a025d --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/hello/main_test.py @@ -0,0 +1,35 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid + +from .main import main + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] +TABLE_ID = f"hello-world-test-{str(uuid.uuid4())[:16]}" + + +def test_main(capsys): + main(PROJECT, BIGTABLE_INSTANCE, TABLE_ID) + + out, _ = capsys.readouterr() + assert "Creating the {} table.".format(TABLE_ID) in out + assert "Writing some greetings to the table." in out + assert "Getting a single greeting by row key." in out + assert "Hello World!" in out + assert "Scanning for all greetings" in out + assert "Hello Cloud Bigtable!" in out + assert "Deleting the {} table.".format(TABLE_ID) in out diff --git a/packages/google-cloud-bigtable/samples/data_client/hello/requirements.txt b/packages/google-cloud-bigtable/samples/data_client/hello/requirements.txt new file mode 100644 index 000000000000..35b55807c842 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/hello/requirements.txt @@ -0,0 +1,2 @@ +google-cloud-bigtable +google-cloud-core==2.5.0 diff --git a/packages/google-cloud-bigtable/samples/data_client/quickstart/__init__.py b/packages/google-cloud-bigtable/samples/data_client/quickstart/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/google-cloud-bigtable/samples/data_client/quickstart/main.py b/packages/google-cloud-bigtable/samples/data_client/quickstart/main.py new file mode 100644 index 000000000000..ef9bf85fe0df --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/quickstart/main.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START bigtable_quickstart] +import argparse + +from google.cloud.bigtable.data import BigtableDataClient + + +def main(project_id="project-id", instance_id="instance-id", table_id="my-table"): + # Create a Cloud Bigtable client. + client = BigtableDataClient(project=project_id) + + # Open an existing table. + table = client.get_table(instance_id, table_id) + + row_key = "r1" + row = table.read_row(row_key) + + column_family_id = "cf1" + column_id = b"c1" + value = row.get_cells(column_family_id, column_id)[0].value.decode("utf-8") + + table.close() + client.close() + + print("Row key: {}\nData: {}".format(row_key, value)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument("project_id", help="Your Cloud Platform project ID.") + parser.add_argument( + "instance_id", help="ID of the Cloud Bigtable instance to connect to." + ) + parser.add_argument( + "--table", help="Existing table used in the quickstart.", default="my-table" + ) + + args = parser.parse_args() + main(args.project_id, args.instance_id, args.table) + +# [END bigtable_quickstart] diff --git a/packages/google-cloud-bigtable/samples/data_client/quickstart/main_test.py b/packages/google-cloud-bigtable/samples/data_client/quickstart/main_test.py new file mode 100644 index 000000000000..9b42cf553e16 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/quickstart/main_test.py @@ -0,0 +1,48 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid +from typing import Generator + +import pytest + +from google.cloud.bigtable.data import BigtableDataClient, SetCell + +from ...utils import create_table_cm +from .main import main + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] +TABLE_ID = f"quickstart-test-{str(uuid.uuid4())[:16]}" + + +@pytest.fixture +def table_id() -> Generator[str, None, None]: + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"cf1": None}): + _populate_table(TABLE_ID) + yield TABLE_ID + + +def _populate_table(table_id: str): + with BigtableDataClient(project=PROJECT) as client: + with client.get_table(BIGTABLE_INSTANCE, table_id) as table: + table.mutate_row("r1", SetCell("cf1", "c1", "test-value")) + + +def test_main(capsys, table_id): + main(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + assert "Row key: r1\nData: test-value\n" in out diff --git a/packages/google-cloud-bigtable/samples/data_client/quickstart/requirements.txt b/packages/google-cloud-bigtable/samples/data_client/quickstart/requirements.txt new file mode 100644 index 000000000000..47acfcb87fb0 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/quickstart/requirements.txt @@ -0,0 +1 @@ +google-cloud-bigtable diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/__init__.py b/packages/google-cloud-bigtable/samples/data_client/snippets/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/data_client_snippets.py b/packages/google-cloud-bigtable/samples/data_client/snippets/data_client_snippets.py new file mode 100644 index 000000000000..aa7e35694802 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/snippets/data_client_snippets.py @@ -0,0 +1,371 @@ +#!/usr/bin/env python + +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def write_simple(table): + # [START bigtable_writes_simple] + from google.cloud.bigtable.data import BigtableDataClient, SetCell + + def write_simple(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + family_id = "stats_summary" + row_key = b"phone#4c410523#20190501" + + cell_mutation = SetCell(family_id, "connected_cell", 1) + wifi_mutation = SetCell(family_id, "connected_wifi", 1) + os_mutation = SetCell(family_id, "os_build", "PQ2A.190405.003") + + table.mutate_row(row_key, cell_mutation) + table.mutate_row(row_key, wifi_mutation) + table.mutate_row(row_key, os_mutation) + + # [END bigtable_writes_simple] + write_simple(table.client.project, table.instance_id, table.table_id) + + +def write_batch(table): + # [START bigtable_writes_batch] + from google.cloud.bigtable.data import BigtableDataClient + from google.cloud.bigtable.data.exceptions import MutationsExceptionGroup + from google.cloud.bigtable.data.mutations import RowMutationEntry, SetCell + + def write_batch(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + family_id = "stats_summary" + try: + with table.mutations_batcher() as batcher: + mutation_list = [ + SetCell(family_id, "connected_cell", 1), + SetCell(family_id, "connected_wifi", 1), + SetCell(family_id, "os_build", "12155.0.0-rc1"), + ] + # batcher.append adds the RowMutationEntry to the batcher's + # queue to be written in the next flush. + batcher.append( + RowMutationEntry("tablet#a0b81f74#20190501", mutation_list) + ) + batcher.append( + RowMutationEntry("tablet#a0b81f74#20190502", mutation_list) + ) + except MutationsExceptionGroup as e: + # MutationsExceptionGroup contains a FailedMutationEntryError for + # each mutation that failed. + for sub_exception in e.exceptions: + failed_entry: RowMutationEntry = sub_exception.entry + cause: Exception = sub_exception.__cause__ + print( + f"Failed mutation: {failed_entry.row_key} with error: {cause!r}" + ) + + # [END bigtable_writes_batch] + write_batch(table.client.project, table.instance_id, table.table_id) + + +def write_increment(table): + # [START bigtable_writes_increment] + from google.cloud.bigtable.data import BigtableDataClient + from google.cloud.bigtable.data.read_modify_write_rules import IncrementRule + + def write_increment(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + family_id = "stats_summary" + row_key = "phone#4c410523#20190501" + + # Decrement the connected_wifi value by 1. + increment_rule = IncrementRule( + family_id, "connected_wifi", increment_amount=-1 + ) + result_row = table.read_modify_write_row(row_key, increment_rule) + + # check result + cell = result_row[0] + print(f"{cell.row_key} value: {int(cell)}") + + # [END bigtable_writes_increment] + write_increment(table.client.project, table.instance_id, table.table_id) + + +def write_conditional(table): + # [START bigtable_writes_conditional] + from google.cloud.bigtable.data import BigtableDataClient, SetCell, row_filters + + def write_conditional(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + family_id = "stats_summary" + row_key = "phone#4c410523#20190501" + + row_filter = row_filters.RowFilterChain( + filters=[ + row_filters.FamilyNameRegexFilter(family_id), + row_filters.ColumnQualifierRegexFilter("os_build"), + row_filters.ValueRegexFilter("PQ2A\\..*"), + ] + ) + + if_true = SetCell(family_id, "os_name", "android") + result = table.check_and_mutate_row( + row_key, + row_filter, + true_case_mutations=if_true, + false_case_mutations=None, + ) + if result is True: + print("The row os_name was set to android") + + # [END bigtable_writes_conditional] + write_conditional(table.client.project, table.instance_id, table.table_id) + + +def write_aggregate(table): + # [START bigtable_writes_aggregate] + import time + + from google.cloud.bigtable.data import BigtableDataClient + from google.cloud.bigtable.data.exceptions import MutationsExceptionGroup + from google.cloud.bigtable.data.mutations import AddToCell, RowMutationEntry + + def write_aggregate(project_id, instance_id, table_id): + """Increments a value in a Bigtable table using AddToCell mutation.""" + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + row_key = "unique_device_ids_1" + try: + with table.mutations_batcher() as batcher: + # The AddToCell mutation increments the value of a cell. + # The `counters` family must be set up to be an aggregate + # family with an int64 input type. + reading = AddToCell( + family="counters", + qualifier="odometer", + value=32304, + # Convert nanoseconds to microseconds + timestamp_micros=time.time_ns() // 1000, + ) + batcher.append( + RowMutationEntry(row_key.encode("utf-8"), [reading]) + ) + except MutationsExceptionGroup as e: + # MutationsExceptionGroup contains a FailedMutationEntryError for + # each mutation that failed. + for sub_exception in e.exceptions: + failed_entry: RowMutationEntry = sub_exception.entry + cause: Exception = sub_exception.__cause__ + print( + f"Failed mutation for row {failed_entry.row_key!r} with error: {cause!r}" + ) + + # [END bigtable_writes_aggregate] + write_aggregate(table.client.project, table.instance_id, table.table_id) + + +def read_row(table): + # [START bigtable_reads_row] + from google.cloud.bigtable.data import BigtableDataClient + + def read_row(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + row_key = "phone#4c410523#20190501" + row = table.read_row(row_key) + print(row) + + # [END bigtable_reads_row] + read_row(table.client.project, table.instance_id, table.table_id) + + +def read_row_partial(table): + # [START bigtable_reads_row_partial] + from google.cloud.bigtable.data import BigtableDataClient, row_filters + + def read_row_partial(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + row_key = "phone#4c410523#20190501" + col_filter = row_filters.ColumnQualifierRegexFilter(b"os_build") + + row = table.read_row(row_key, row_filter=col_filter) + print(row) + + # [END bigtable_reads_row_partial] + read_row_partial(table.client.project, table.instance_id, table.table_id) + + +def read_rows_multiple(table): + # [START bigtable_reads_rows] + from google.cloud.bigtable.data import BigtableDataClient, ReadRowsQuery + + def read_rows(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + query = ReadRowsQuery( + row_keys=[b"phone#4c410523#20190501", b"phone#4c410523#20190502"] + ) + for row in table.read_rows_stream(query): + print(row) + + # [END bigtable_reads_rows] + read_rows(table.client.project, table.instance_id, table.table_id) + + +def read_row_range(table): + # [START bigtable_reads_row_range] + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + RowRange, + ) + + def read_row_range(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + row_range = RowRange( + start_key=b"phone#4c410523#20190501", + end_key=b"phone#4c410523#201906201", + ) + query = ReadRowsQuery(row_ranges=[row_range]) + + for row in table.read_rows_stream(query): + print(row) + + # [END bigtable_reads_row_range] + read_row_range(table.client.project, table.instance_id, table.table_id) + + +def read_row_ranges(table): + # [START bigtable_reads_row_ranges] + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + RowRange, + ) + + def read_row_ranges(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + row_ranges = [ + RowRange( + start_key=b"phone#4c410523#20190501", + end_key=b"phone#4c410523#201906201", + ), + RowRange( + start_key=b"phone#5c10102#20190501", + end_key=b"phone#5c10102#201906201", + ), + ] + query = ReadRowsQuery(row_ranges=row_ranges) + + for row in table.read_rows_stream(query): + print(row) + + # [END bigtable_reads_row_ranges] + read_row_ranges(table.client.project, table.instance_id, table.table_id) + + +def read_with_prefix(table): + # [START bigtable_reads_prefix] + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + RowRange, + ) + + def read_prefix(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + prefix = "phone#" + end_key = prefix[:-1] + chr(ord(prefix[-1]) + 1) + prefix_range = RowRange(start_key=prefix, end_key=end_key) + query = ReadRowsQuery(row_ranges=[prefix_range]) + + for row in table.read_rows_stream(query): + print(row) + + # [END bigtable_reads_prefix] + read_prefix(table.client.project, table.instance_id, table.table_id) + + +def read_with_filter(table): + # [START bigtable_reads_filter] + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + def read_with_filter(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + row_filter = row_filters.ValueRegexFilter(b"PQ2A.*$") + query = ReadRowsQuery(row_filter=row_filter) + + for row in table.read_rows_stream(query): + print(row) + + # [END bigtable_reads_filter] + read_with_filter(table.client.project, table.instance_id, table.table_id) + + +def execute_query(table): + # [START bigtable_execute_query] + from google.cloud.bigtable.data import BigtableDataClient + + def execute_query(project_id, instance_id, table_id): + with BigtableDataClient(project=project_id) as client: + query = ( + "SELECT _key, stats_summary['os_build'], " + "stats_summary['connected_cell'], " + "stats_summary['connected_wifi'] " + f"from `{table_id}` WHERE _key=@row_key" + ) + result = client.execute_query( + query, + instance_id, + parameters={"row_key": b"phone#4c410523#20190501"}, + ) + results = list(result) + print(results) + + # [END bigtable_execute_query] + execute_query(table.client.project, table.instance_id, table.table_id) + + +# [START bigtable_reads_print] +def print_row(row): + from google.cloud._helpers import _datetime_from_microseconds + + print("Reading data for {}:".format(row.row_key.decode("utf-8"))) + last_family = None + for cell in row.cells: + if last_family != cell.family: + print("Column Family {}".format(cell.family)) + last_family = cell.family + + labels = " [{}]".format(",".join(cell.labels)) if len(cell.labels) else "" + print( + "\t{}: {} @{}{}".format( + cell.qualifier.decode("utf-8"), + cell.value.decode("utf-8"), + _datetime_from_microseconds(cell.timestamp_micros), + labels, + ) + ) + print("") + + +# [END bigtable_reads_print] diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/data_client_snippets_test.py b/packages/google-cloud-bigtable/samples/data_client/snippets/data_client_snippets_test.py new file mode 100644 index 000000000000..54eea31c5d78 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/snippets/data_client_snippets_test.py @@ -0,0 +1,109 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid + +import pytest + +from ...utils import create_table_cm +from . import data_client_snippets as data_snippets + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] +TABLE_ID = f"data-client-sync-{str(uuid.uuid4())[:16]}" + + +@pytest.fixture(scope="session") +def column_family_config(): + from google.cloud.bigtable_admin_v2 import types + + int_aggregate_type = types.Type.Aggregate( + input_type=types.Type(int64_type={"encoding": {"big_endian_bytes": {}}}), + sum={}, + ) + + return { + "family": types.ColumnFamily(), + "stats_summary": types.ColumnFamily(), + "counters": types.ColumnFamily( + value_type=types.Type(aggregate_type=int_aggregate_type) + ), + } + + +@pytest.fixture(scope="session") +def table_id(column_family_config): + with create_table_cm(PROJECT, BIGTABLE_INSTANCE, TABLE_ID, column_family_config): + yield TABLE_ID + + +@pytest.fixture +def table(table_id): + from google.cloud.bigtable.data import BigtableDataClient + + with BigtableDataClient(project=PROJECT) as client: + with client.get_table(BIGTABLE_INSTANCE, table_id) as table: + yield table + + +def test_write_simple(table): + data_snippets.write_simple(table) + + +def test_write_batch(table): + data_snippets.write_batch(table) + + +def test_write_increment(table): + data_snippets.write_increment(table) + + +def test_write_conditional(table): + data_snippets.write_conditional(table) + + +def test_write_aggregate(table): + data_snippets.write_aggregate(table) + + +def test_read_row(table): + data_snippets.read_row(table) + + +def test_read_row_partial(table): + data_snippets.read_row_partial(table) + + +def test_read_rows_multiple(table): + data_snippets.read_rows_multiple(table) + + +def test_read_row_range(table): + data_snippets.read_row_range(table) + + +def test_read_row_ranges(table): + data_snippets.read_row_ranges(table) + + +def test_read_with_prefix(table): + data_snippets.read_with_prefix(table) + + +def test_read_with_filter(table): + data_snippets.read_with_filter(table) + + +def test_execute_query(table): + data_snippets.execute_query(table) diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/deletes_snippets.py b/packages/google-cloud-bigtable/samples/data_client/snippets/deletes_snippets.py new file mode 100644 index 000000000000..c069252c8afd --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/snippets/deletes_snippets.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python + +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START bigtable_delete_from_column] +def delete_from_column(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + DeleteRangeFromColumn, + ) + + client = BigtableDataClient(project=project_id) + table = client.get_table(instance_id, table_id) + + table.mutate_row( + "phone#4c410523#20190501", + DeleteRangeFromColumn(family="cell_plan", qualifier=b"data_plan_01gb"), + ) + + table.close() + client.close() + + +# [END bigtable_delete_from_column] + + +# [START bigtable_delete_from_column_family] +def delete_from_column_family(project_id, instance_id, table_id): + from google.cloud.bigtable.data import BigtableDataClient, DeleteAllFromFamily + + client = BigtableDataClient(project=project_id) + table = client.get_table(instance_id, table_id) + + table.mutate_row("phone#4c410523#20190501", DeleteAllFromFamily("cell_plan")) + + table.close() + client.close() + + +# [END bigtable_delete_from_column_family] + + +# [START bigtable_delete_from_row] +def delete_from_row(project_id, instance_id, table_id): + from google.cloud.bigtable.data import BigtableDataClient, DeleteAllFromRow + + client = BigtableDataClient(project=project_id) + table = client.get_table(instance_id, table_id) + + table.mutate_row("phone#4c410523#20190501", DeleteAllFromRow()) + + table.close() + client.close() + + +# [END bigtable_delete_from_row] + + +# [START bigtable_streaming_and_batching] +def streaming_and_batching(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + DeleteRangeFromColumn, + ReadRowsQuery, + RowMutationEntry, + ) + + client = BigtableDataClient(project=project_id) + table = client.get_table(instance_id, table_id) + + with table.mutations_batcher() as batcher: + for row in table.read_rows_stream(ReadRowsQuery(limit=10)): + batcher.append( + RowMutationEntry( + row.row_key, + DeleteRangeFromColumn( + family="cell_plan", qualifier=b"data_plan_01gb" + ), + ) + ) + + table.close() + client.close() + + +# [END bigtable_streaming_and_batching] + + +# [START bigtable_check_and_mutate] +def check_and_mutate(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + DeleteRangeFromColumn, + ) + from google.cloud.bigtable.data.row_filters import LiteralValueFilter + + client = BigtableDataClient(project=project_id) + table = client.get_table(instance_id, table_id) + + table.check_and_mutate_row( + "phone#4c410523#20190501", + predicate=LiteralValueFilter("PQ2A.190405.003"), + true_case_mutations=DeleteRangeFromColumn( + family="cell_plan", qualifier=b"data_plan_01gb" + ), + ) + + table.close() + client.close() + + +# [END bigtable_check_and_mutate] + + +# [START bigtable_drop_row_range] +def drop_row_range(project_id, instance_id, table_id): + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableTableAdminClient() + table_path = client.table_path(project_id, instance_id, table_id) + row_key_prefix = b"phone#4c410523" + client.drop_row_range( + request={ + "name": table_path, + "row_key_prefix": row_key_prefix, + }, + timeout=200, + ) + + +# [END bigtable_drop_row_range] + + +# [START bigtable_delete_column_family] +def delete_column_family(project_id, instance_id, table_id): + from google.cloud import bigtable_admin + + client = bigtable_admin.BigtableTableAdminClient() + table_path = client.table_path(project_id, instance_id, table_id) + column_family_id = "stats_summary" + modification = bigtable_admin.ModifyColumnFamiliesRequest.Modification( + id=column_family_id, drop=True + ) + client.modify_column_families(name=table_path, modifications=[modification]) + + +# [END bigtable_delete_column_family] diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/deletes_snippets_test.py b/packages/google-cloud-bigtable/samples/data_client/snippets/deletes_snippets_test.py new file mode 100644 index 000000000000..54b69321cce7 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/snippets/deletes_snippets_test.py @@ -0,0 +1,269 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os +import uuid +from typing import Generator + +import pytest +from google.cloud._helpers import _microseconds_from_datetime + +from ...utils import create_table_cm +from . import deletes_snippets + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] +TABLE_ID = f"mobile-time-series-deletes-{str(uuid.uuid4())[:16]}" + + +@pytest.fixture(scope="module", autouse=True) +def table_id() -> Generator[str, None, None]: + with create_table_cm( + PROJECT, + BIGTABLE_INSTANCE, + TABLE_ID, + {"stats_summary": None, "cell_plan": None}, + verbose=False, + ): + _populate_table(TABLE_ID) + yield TABLE_ID + + +def _populate_table(table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + RowMutationEntry, + SetCell, + ) + + timestamp = datetime.datetime(2019, 5, 1) + timestamp_minus_hr = timestamp - datetime.timedelta(hours=1) + + with BigtableDataClient(project=PROJECT) as client: + with client.get_table(BIGTABLE_INSTANCE, table_id) as table: + with table.mutations_batcher() as batcher: + batcher.append( + RowMutationEntry( + "phone#4c410523#20190501", + [ + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190405.003", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_01gb", + "true", + _microseconds_from_datetime(timestamp_minus_hr), + ), + SetCell( + "cell_plan", + "data_plan_01gb", + "false", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_05gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + batcher.append( + RowMutationEntry( + "phone#4c410523#20190502", + [ + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190405.004", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_05gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + batcher.append( + RowMutationEntry( + "phone#4c410523#20190505", + [ + SetCell( + "stats_summary", + "connected_cell", + 0, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190406.000", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_05gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + batcher.append( + RowMutationEntry( + "phone#5c10102#20190501", + [ + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190401.002", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_10gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + batcher.append( + RowMutationEntry( + "phone#5c10102#20190502", + [ + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 0, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190406.000", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_10gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + + +def assert_output_match(capsys, expected): + out, _ = capsys.readouterr() + assert out == expected + + +def test_delete_from_column(capsys, table_id): + deletes_snippets.delete_from_column(PROJECT, BIGTABLE_INSTANCE, table_id) + assert_output_match(capsys, "") + + +def test_delete_from_column_family(capsys, table_id): + deletes_snippets.delete_from_column_family(PROJECT, BIGTABLE_INSTANCE, table_id) + assert_output_match(capsys, "") + + +def test_delete_from_row(capsys, table_id): + deletes_snippets.delete_from_row(PROJECT, BIGTABLE_INSTANCE, table_id) + assert_output_match(capsys, "") + + +def test_streaming_and_batching(capsys, table_id): + deletes_snippets.streaming_and_batching(PROJECT, BIGTABLE_INSTANCE, table_id) + assert_output_match(capsys, "") + + +def test_check_and_mutate(capsys, table_id): + deletes_snippets.check_and_mutate(PROJECT, BIGTABLE_INSTANCE, table_id) + assert_output_match(capsys, "") + + +def test_drop_row_range(capsys, table_id): + deletes_snippets.drop_row_range(PROJECT, BIGTABLE_INSTANCE, table_id) + assert_output_match(capsys, "") + + +def test_delete_column_family(capsys, table_id): + deletes_snippets.delete_column_family(PROJECT, BIGTABLE_INSTANCE, table_id) + assert_output_match(capsys, "") diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/filter_snippets.py b/packages/google-cloud-bigtable/samples/data_client/snippets/filter_snippets.py new file mode 100644 index 000000000000..2fd7909f3357 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/snippets/filter_snippets.py @@ -0,0 +1,393 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START bigtable_filters_limit_row_sample] +def filter_limit_row_sample(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery(row_filter=row_filters.RowSampleFilter(0.75)) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_row_sample] +# [START bigtable_filters_limit_row_regex] +def filter_limit_row_regex(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery( + row_filter=row_filters.RowKeyRegexFilter(".*#20190501$".encode("utf-8")) + ) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_row_regex] +# [START bigtable_filters_limit_cells_per_col] +def filter_limit_cells_per_col(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery(row_filter=row_filters.CellsColumnLimitFilter(2)) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_cells_per_col] +# [START bigtable_filters_limit_cells_per_row] +def filter_limit_cells_per_row(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery(row_filter=row_filters.CellsRowLimitFilter(2)) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_cells_per_row] +# [START bigtable_filters_limit_cells_per_row_offset] +def filter_limit_cells_per_row_offset(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery(row_filter=row_filters.CellsRowOffsetFilter(2)) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_cells_per_row_offset] +# [START bigtable_filters_limit_col_family_regex] +def filter_limit_col_family_regex(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery( + row_filter=row_filters.FamilyNameRegexFilter("stats_.*$".encode("utf-8")) + ) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_col_family_regex] +# [START bigtable_filters_limit_col_qualifier_regex] +def filter_limit_col_qualifier_regex(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery( + row_filter=row_filters.ColumnQualifierRegexFilter( + "connected_.*$".encode("utf-8") + ) + ) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_col_qualifier_regex] +# [START bigtable_filters_limit_col_range] +def filter_limit_col_range(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery( + row_filter=row_filters.ColumnRangeFilter( + "cell_plan", b"data_plan_01gb", b"data_plan_10gb", inclusive_end=False + ) + ) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_col_range] +# [START bigtable_filters_limit_value_range] +def filter_limit_value_range(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery( + row_filter=row_filters.ValueRangeFilter(b"PQ2A.190405", b"PQ2A.190406") + ) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_value_range] +# [START bigtable_filters_limit_value_regex] + + +def filter_limit_value_regex(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery( + row_filter=row_filters.ValueRegexFilter("PQ2A.*$".encode("utf-8")) + ) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_value_regex] +# [START bigtable_filters_limit_timestamp_range] +def filter_limit_timestamp_range(project_id, instance_id, table_id): + import datetime + + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + end = datetime.datetime(2019, 5, 1) + + query = ReadRowsQuery(row_filter=row_filters.TimestampRangeFilter(end=end)) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_timestamp_range] +# [START bigtable_filters_limit_block_all] +def filter_limit_block_all(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery(row_filter=row_filters.BlockAllFilter(True)) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_block_all] +# [START bigtable_filters_limit_pass_all] +def filter_limit_pass_all(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery(row_filter=row_filters.PassAllFilter(True)) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_limit_pass_all] +# [START bigtable_filters_modify_strip_value] +def filter_modify_strip_value(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery(row_filter=row_filters.StripValueTransformerFilter(True)) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_modify_strip_value] +# [START bigtable_filters_modify_apply_label] +def filter_modify_apply_label(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery(row_filter=row_filters.ApplyLabelFilter(label="labelled")) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_modify_apply_label] +# [START bigtable_filters_composing_chain] +def filter_composing_chain(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery( + row_filter=row_filters.RowFilterChain( + filters=[ + row_filters.CellsColumnLimitFilter(1), + row_filters.FamilyNameRegexFilter("cell_plan"), + ] + ) + ) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_composing_chain] +# [START bigtable_filters_composing_interleave] +def filter_composing_interleave(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery( + row_filter=row_filters.RowFilterUnion( + filters=[ + row_filters.ValueRegexFilter("true"), + row_filters.ColumnQualifierRegexFilter("os_build"), + ] + ) + ) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_composing_interleave] +# [START bigtable_filters_composing_condition] +def filter_composing_condition(project_id, instance_id, table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + ReadRowsQuery, + row_filters, + ) + + query = ReadRowsQuery( + row_filter=row_filters.ConditionalRowFilter( + predicate_filter=row_filters.RowFilterChain( + filters=[ + row_filters.ColumnQualifierRegexFilter("data_plan_10gb"), + row_filters.ValueRegexFilter("true"), + ] + ), + true_filter=row_filters.ApplyLabelFilter(label="passed-filter"), + false_filter=row_filters.ApplyLabelFilter(label="filtered-out"), + ) + ) + + with BigtableDataClient(project=project_id) as client: + with client.get_table(instance_id, table_id) as table: + for row in table.read_rows(query): + print_row(row) + + +# [END bigtable_filters_composing_condition] + + +# [START bigtable_filters_print] +def print_row(row): + from google.cloud._helpers import _datetime_from_microseconds + + print("Reading data for {}:".format(row.row_key.decode("utf-8"))) + last_family = None + for cell in row.cells: + if last_family != cell.family: + print("Column Family {}".format(cell.family)) + last_family = cell.family + + labels = " [{}]".format(",".join(cell.labels)) if len(cell.labels) else "" + print( + "\t{}: {} @{}{}".format( + cell.qualifier.decode("utf-8"), + cell.value.decode("utf-8"), + _datetime_from_microseconds(cell.timestamp_micros), + labels, + ) + ) + print("") + + +# [END bigtable_filters_print] diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/filter_snippets_test.py b/packages/google-cloud-bigtable/samples/data_client/snippets/filter_snippets_test.py new file mode 100644 index 000000000000..71aaa582cede --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/snippets/filter_snippets_test.py @@ -0,0 +1,373 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import inspect +import os +import uuid +from typing import Generator + +import pytest +from google.cloud._helpers import _microseconds_from_datetime + +from ...utils import create_table_cm +from . import filter_snippets +from .snapshots.snap_filters_test import snapshots + +PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"] +TABLE_ID = f"mobile-time-series-filters-{str(uuid.uuid4())[:16]}" + + +@pytest.fixture(scope="module", autouse=True) +def table_id() -> Generator[str, None, None]: + with create_table_cm( + PROJECT, BIGTABLE_INSTANCE, TABLE_ID, {"stats_summary": None, "cell_plan": None} + ): + _populate_table(TABLE_ID) + yield TABLE_ID + + +def _populate_table(table_id): + from google.cloud.bigtable.data import ( + BigtableDataClient, + RowMutationEntry, + SetCell, + ) + + timestamp = datetime.datetime(2019, 5, 1) + timestamp_minus_hr = timestamp - datetime.timedelta(hours=1) + + with BigtableDataClient(project=PROJECT) as client: + with client.get_table(BIGTABLE_INSTANCE, table_id) as table: + with table.mutations_batcher() as batcher: + batcher.append( + RowMutationEntry( + "phone#4c410523#20190501", + [ + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190405.003", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_01gb", + "true", + _microseconds_from_datetime(timestamp_minus_hr), + ), + SetCell( + "cell_plan", + "data_plan_01gb", + "false", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_05gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + batcher.append( + RowMutationEntry( + "phone#4c410523#20190502", + [ + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190405.004", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_05gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + batcher.append( + RowMutationEntry( + "phone#4c410523#20190505", + [ + SetCell( + "stats_summary", + "connected_cell", + 0, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190406.000", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_05gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + batcher.append( + RowMutationEntry( + "phone#5c10102#20190501", + [ + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190401.002", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_10gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + batcher.append( + RowMutationEntry( + "phone#5c10102#20190502", + [ + SetCell( + "stats_summary", + "connected_cell", + 1, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "connected_wifi", + 0, + _microseconds_from_datetime(timestamp), + ), + SetCell( + "stats_summary", + "os_build", + "PQ2A.190406.000", + _microseconds_from_datetime(timestamp), + ), + SetCell( + "cell_plan", + "data_plan_10gb", + "true", + _microseconds_from_datetime(timestamp), + ), + ], + ) + ) + + +def test_filter_limit_row_sample(capsys, table_id): + filter_snippets.filter_limit_row_sample(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + assert "Reading data for" in out + + +def test_filter_limit_row_regex(capsys, table_id): + filter_snippets.filter_limit_row_regex(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_cells_per_col(capsys, table_id): + filter_snippets.filter_limit_cells_per_col(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_cells_per_row(capsys, table_id): + filter_snippets.filter_limit_cells_per_row(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_cells_per_row_offset(capsys, table_id): + filter_snippets.filter_limit_cells_per_row_offset( + PROJECT, BIGTABLE_INSTANCE, table_id + ) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_col_family_regex(capsys, table_id): + filter_snippets.filter_limit_col_family_regex(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_col_qualifier_regex(capsys, table_id): + filter_snippets.filter_limit_col_qualifier_regex( + PROJECT, BIGTABLE_INSTANCE, table_id + ) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_col_range(capsys, table_id): + filter_snippets.filter_limit_col_range(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_value_range(capsys, table_id): + filter_snippets.filter_limit_value_range(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_value_regex(capsys, table_id): + filter_snippets.filter_limit_value_regex(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_timestamp_range(capsys, table_id): + filter_snippets.filter_limit_timestamp_range(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_block_all(capsys, table_id): + filter_snippets.filter_limit_block_all(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_limit_pass_all(capsys, table_id): + filter_snippets.filter_limit_pass_all(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_modify_strip_value(capsys, table_id): + filter_snippets.filter_modify_strip_value(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_modify_apply_label(capsys, table_id): + filter_snippets.filter_modify_apply_label(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_composing_chain(capsys, table_id): + filter_snippets.filter_composing_chain(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_composing_interleave(capsys, table_id): + filter_snippets.filter_composing_interleave(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected + + +def test_filter_composing_condition(capsys, table_id): + filter_snippets.filter_composing_condition(PROJECT, BIGTABLE_INSTANCE, table_id) + + out, _ = capsys.readouterr() + expected = snapshots[inspect.currentframe().f_code.co_name] + assert out == expected diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/requirements.txt b/packages/google-cloud-bigtable/samples/data_client/snippets/requirements.txt new file mode 100644 index 000000000000..47acfcb87fb0 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/snippets/requirements.txt @@ -0,0 +1 @@ +google-cloud-bigtable diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/snapshots/__init__.py b/packages/google-cloud-bigtable/samples/data_client/snippets/snapshots/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/google-cloud-bigtable/samples/data_client/snippets/snapshots/snap_filters_test.py b/packages/google-cloud-bigtable/samples/data_client/snippets/snapshots/snap_filters_test.py new file mode 100644 index 000000000000..0547ddddd858 --- /dev/null +++ b/packages/google-cloud-bigtable/samples/data_client/snippets/snapshots/snap_filters_test.py @@ -0,0 +1,503 @@ +# -*- coding: utf-8 -*- +# this was previously implemented using the `snapshottest` package (https://goo.gl/zC4yUc), +# which is not compatible with Python 3.12. So we moved to a standard dictionary storing +# expected outputs for each test +from __future__ import unicode_literals + +snapshots = {} + +snapshots["test_filter_limit_row_regex"] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 +\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_limit_cells_per_col" +] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 +\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_limit_cells_per_row" +] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 +\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_limit_cells_per_row_offset" +] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family stats_summary +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family stats_summary +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family stats_summary +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family stats_summary +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_limit_col_family_regex" +] = """Reading data for phone#4c410523#20190501: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_limit_col_qualifier_regex" +] = """Reading data for phone#4c410523#20190501: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 + +""" + +snapshots["test_filter_limit_col_range"] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 +\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_limit_value_range" +] = """Reading data for phone#4c410523#20190501: +Column Family stats_summary +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family stats_summary +\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_limit_value_regex" +] = """Reading data for phone#4c410523#20190501: +Column Family stats_summary +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family stats_summary +\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family stats_summary +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family stats_summary +\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family stats_summary +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_limit_timestamp_range" +] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 + +""" + +snapshots["test_filter_limit_block_all"] = "" + +snapshots["test_filter_limit_pass_all"] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 +\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_modify_strip_value" +] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: @2019-05-01 00:00:00+00:00 +\tdata_plan_01gb: @2019-04-30 23:00:00+00:00 +\tdata_plan_05gb: @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: @2019-05-01 00:00:00+00:00 +\tconnected_wifi: @2019-05-01 00:00:00+00:00 +\tos_build: @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family cell_plan +\tdata_plan_05gb: @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: @2019-05-01 00:00:00+00:00 +\tconnected_wifi: @2019-05-01 00:00:00+00:00 +\tos_build: @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family cell_plan +\tdata_plan_05gb: @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: @2019-05-01 00:00:00+00:00 +\tconnected_wifi: @2019-05-01 00:00:00+00:00 +\tos_build: @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family cell_plan +\tdata_plan_10gb: @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: @2019-05-01 00:00:00+00:00 +\tconnected_wifi: @2019-05-01 00:00:00+00:00 +\tos_build: @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family cell_plan +\tdata_plan_10gb: @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tconnected_cell: @2019-05-01 00:00:00+00:00 +\tconnected_wifi: @2019-05-01 00:00:00+00:00 +\tos_build: @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_modify_apply_label" +] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 [labelled] +\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 [labelled] +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 [labelled] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 [labelled] + +Reading data for phone#4c410523#20190502: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 [labelled] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] +\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 [labelled] + +Reading data for phone#4c410523#20190505: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 [labelled] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 [labelled] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 [labelled] + +Reading data for phone#5c10102#20190501: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 [labelled] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] +\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 [labelled] + +Reading data for phone#5c10102#20190502: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 [labelled] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [labelled] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 [labelled] +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 [labelled] + +""" + +snapshots["test_filter_composing_chain"] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_composing_interleave" +] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190502: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 + +Reading data for phone#4c410523#20190505: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190501: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 + +Reading data for phone#5c10102#20190502: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 +Column Family stats_summary +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 + +""" + +snapshots[ + "test_filter_composing_condition" +] = """Reading data for phone#4c410523#20190501: +Column Family cell_plan +\tdata_plan_01gb: false @2019-05-01 00:00:00+00:00 [filtered-out] +\tdata_plan_01gb: true @2019-04-30 23:00:00+00:00 [filtered-out] +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 [filtered-out] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [filtered-out] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [filtered-out] +\tos_build: PQ2A.190405.003 @2019-05-01 00:00:00+00:00 [filtered-out] + +Reading data for phone#4c410523#20190502: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 [filtered-out] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [filtered-out] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [filtered-out] +\tos_build: PQ2A.190405.004 @2019-05-01 00:00:00+00:00 [filtered-out] + +Reading data for phone#4c410523#20190505: +Column Family cell_plan +\tdata_plan_05gb: true @2019-05-01 00:00:00+00:00 [filtered-out] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 [filtered-out] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [filtered-out] +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 [filtered-out] + +Reading data for phone#5c10102#20190501: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 [passed-filter] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [passed-filter] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [passed-filter] +\tos_build: PQ2A.190401.002 @2019-05-01 00:00:00+00:00 [passed-filter] + +Reading data for phone#5c10102#20190502: +Column Family cell_plan +\tdata_plan_10gb: true @2019-05-01 00:00:00+00:00 [passed-filter] +Column Family stats_summary +\tconnected_cell: \x00\x00\x00\x00\x00\x00\x00\x01 @2019-05-01 00:00:00+00:00 [passed-filter] +\tconnected_wifi: \x00\x00\x00\x00\x00\x00\x00\x00 @2019-05-01 00:00:00+00:00 [passed-filter] +\tos_build: PQ2A.190406.000 @2019-05-01 00:00:00+00:00 [passed-filter] + +""" diff --git a/packages/google-cloud-bigtable/samples/data_client_async/hello/async_main.py b/packages/google-cloud-bigtable/samples/data_client_async/hello/async_main.py index be687b07b95f..db23f35d842f 100644 --- a/packages/google-cloud-bigtable/samples/data_client_async/hello/async_main.py +++ b/packages/google-cloud-bigtable/samples/data_client_async/hello/async_main.py @@ -46,30 +46,37 @@ async def main(project_id, instance_id, table_id): # [END bigtable_async_hw_connect] # [START bigtable_async_hw_create_table] - from google.cloud.bigtable import column_family + from google.api_core.exceptions import AlreadyExists - # the async client only supports the data API. Table creation as an admin operation + from google.cloud import bigtable_admin + + # the async client only supports the data API. Table creation is an admin operation # use admin client to create the table print("Creating the {} table.".format(table_id)) - admin_client = bigtable.Client(project=project_id, admin=True) - admin_instance = admin_client.instance(instance_id) - admin_table = admin_instance.table(table_id) + admin_client = bigtable_admin.BigtableTableAdminAsyncClient() + instance_path = admin_client.instance_path(project_id, instance_id) print("Creating column family cf1 with Max Version GC rule...") # Create a column family with GC policy : most recent N versions # Define the GC policy to retain only the most recent 2 versions - max_versions_rule = column_family.MaxVersionsGCRule(2) - column_family_id = b"cf1" - column_families = {column_family_id: max_versions_rule} - if not admin_table.exists(): - admin_table.create(column_families=column_families) - else: + max_versions_rule = bigtable_admin.GcRule(max_num_versions=2) + column_family_id = "cf1" + column_families = { + column_family_id: bigtable_admin.ColumnFamily(gc_rule=max_versions_rule) + } + try: + await admin_client.create_table( + parent=instance_path, + table_id=table_id, + table=bigtable_admin.Table(column_families=column_families), + ) + except AlreadyExists: print("Table {} already exists.".format(table_id)) # [END bigtable_async_hw_create_table] try: # let table creation complete - wait_for_table(admin_table) + wait_for_table(table) # [START bigtable_async_hw_write_rows] print("Writing some greetings to the table.") greetings = [b"Hello World!", b"Hello Cloud Bigtable!", b"Hello Python!"] @@ -125,12 +132,16 @@ async def main(project_id, instance_id, table_id): # [END bigtable_async_hw_scan_with_filter] finally: # [START bigtable_async_hw_delete_table] - # the async client only supports the data API. Table deletion as an admin operation - # use admin client to create the table + from google.cloud import bigtable_admin + + # the async client only supports the data API. Table deletion is an admin operation + # use admin client to delete the table + admin_client = bigtable_admin.BigtableTableAdminAsyncClient() + table_path = admin_client.table_path(project_id, instance_id, table_id) + print("Deleting the {} table.".format(table_id)) - admin_table.delete() - await table.close() - await client.close() + await admin_client.delete_table(name=table_path) + await admin_client.close() # [END bigtable_async_hw_delete_table] diff --git a/packages/google-cloud-bigtable/samples/data_client_async/snippets/data_client_snippets_async.py b/packages/google-cloud-bigtable/samples/data_client_async/snippets/data_client_snippets_async.py index 77448bf7e085..15a4e562614d 100644 --- a/packages/google-cloud-bigtable/samples/data_client_async/snippets/data_client_snippets_async.py +++ b/packages/google-cloud-bigtable/samples/data_client_async/snippets/data_client_snippets_async.py @@ -247,6 +247,36 @@ async def read_row_range(project_id, instance_id, table_id): await read_row_range(table.client.project, table.instance_id, table.table_id) +async def read_row_ranges(table): + # [START bigtable_async_reads_row_ranges] + from google.cloud.bigtable.data import ( + BigtableDataClientAsync, + ReadRowsQuery, + RowRange, + ) + + async def read_row_ranges(project_id, instance_id, table_id): + async with BigtableDataClientAsync(project=project_id) as client: + async with client.get_table(instance_id, table_id) as table: + row_ranges = [ + RowRange( + start_key=b"phone#4c410523#20190501", + end_key=b"phone#4c410523#201906201", + ), + RowRange( + start_key=b"phone#5c10102#20190501", + end_key=b"phone#5c10102#201906201", + ), + ] + query = ReadRowsQuery(row_ranges=row_ranges) + + async for row in await table.read_rows_stream(query): + print(row) + + # [END bigtable_async_reads_row_ranges] + await read_row_ranges(table.client.project, table.instance_id, table.table_id) + + async def read_with_prefix(table): # [START bigtable_async_reads_prefix] from google.cloud.bigtable.data import ( @@ -313,3 +343,29 @@ async def execute_query(project_id, instance_id, table_id): # [END bigtable_async_execute_query] await execute_query(table.client.project, table.instance_id, table.table_id) + + +# [START bigtable_async_reads_print] +def print_row(row): + from google.cloud._helpers import _datetime_from_microseconds + + print("Reading data for {}:".format(row.row_key.decode("utf-8"))) + last_family = None + for cell in row.cells: + if last_family != cell.family: + print("Column Family {}".format(cell.family)) + last_family = cell.family + + labels = " [{}]".format(",".join(cell.labels)) if len(cell.labels) else "" + print( + "\t{}: {} @{}{}".format( + cell.qualifier.decode("utf-8"), + cell.value.decode("utf-8"), + _datetime_from_microseconds(cell.timestamp_micros), + labels, + ) + ) + print("") + + +# [END bigtable_async_reads_print] diff --git a/packages/google-cloud-bigtable/samples/data_client_async/snippets/data_client_snippets_async_test.py b/packages/google-cloud-bigtable/samples/data_client_async/snippets/data_client_snippets_async_test.py index 6742d2260a83..d7e4d46b2402 100644 --- a/packages/google-cloud-bigtable/samples/data_client_async/snippets/data_client_snippets_async_test.py +++ b/packages/google-cloud-bigtable/samples/data_client_async/snippets/data_client_snippets_async_test.py @@ -102,6 +102,11 @@ async def test_read_row_range(table): await data_snippets.read_row_range(table) +@pytest.mark.asyncio +async def test_read_row_ranges(table): + await data_snippets.read_row_ranges(table) + + @pytest.mark.asyncio async def test_read_with_prefix(table): await data_snippets.read_with_prefix(table) diff --git a/packages/google-cloud-bigtable/samples/data_client_async/snippets/filter_snippets_async.py b/packages/google-cloud-bigtable/samples/data_client_async/snippets/filter_snippets_async.py index 899d4c5c78e9..d2e7da1bb57c 100644 --- a/packages/google-cloud-bigtable/samples/data_client_async/snippets/filter_snippets_async.py +++ b/packages/google-cloud-bigtable/samples/data_client_async/snippets/filter_snippets_async.py @@ -367,6 +367,7 @@ async def filter_composing_condition(project_id, instance_id, table_id): # [END bigtable_filters_composing_condition_asyncio] +# [START bigtable_filters_print_asyncio] def print_row(row): from google.cloud._helpers import _datetime_from_microseconds @@ -387,3 +388,6 @@ def print_row(row): ) ) print("") + + +# [END bigtable_filters_print_asyncio] diff --git a/packages/google-cloud-bigtable/samples/legacy_client/hello/main.py b/packages/google-cloud-bigtable/samples/legacy_client/hello/main.py index c971280b9f77..10473f65bd89 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/hello/main.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/hello/main.py @@ -26,7 +26,7 @@ import argparse -# [START bigtable_hw_imports] +# [START bigtable_hw_imports_legacy] from datetime import datetime, timezone from google.cloud import bigtable @@ -34,7 +34,7 @@ from ...utils import wait_for_table -# [END bigtable_hw_imports] +# [END bigtable_hw_imports_legacy] # use to avoid warnings row_filters @@ -42,14 +42,14 @@ def main(project_id, instance_id, table_id): - # [START bigtable_hw_connect] + # [START bigtable_hw_connect_legacy] # The client must be created with admin=True because it will create a # table. client = bigtable.Client(project=project_id, admin=True) instance = client.instance(instance_id) - # [END bigtable_hw_connect] + # [END bigtable_hw_connect_legacy] - # [START bigtable_hw_create_table] + # [START bigtable_hw_create_table_legacy] print("Creating the {} table.".format(table_id)) table = instance.table(table_id) @@ -63,13 +63,13 @@ def main(project_id, instance_id, table_id): table.create(column_families=column_families) else: print("Table {} already exists.".format(table_id)) - # [END bigtable_hw_create_table] + # [END bigtable_hw_create_table_legacy] try: # let table creation complete wait_for_table(table) - # [START bigtable_hw_write_rows] + # [START bigtable_hw_write_rows_legacy] print("Writing some greetings to the table.") greetings = [b"Hello World!", b"Hello Cloud Bigtable!", b"Hello Python!"] rows = [] @@ -98,27 +98,27 @@ def main(project_id, instance_id, table_id): ) rows.append(row) table.mutate_rows(rows) - # [END bigtable_hw_write_rows] + # [END bigtable_hw_write_rows_legacy] - # [START bigtable_hw_create_filter] + # [START bigtable_hw_create_filter_legacy] # Create a filter to only retrieve the most recent version of the cell # for each column across entire row. row_filter = bigtable.row_filters.CellsColumnLimitFilter(1) - # [END bigtable_hw_create_filter] + # [END bigtable_hw_create_filter_legacy] - # [START bigtable_hw_get_with_filter] - # [START bigtable_hw_get_by_key] + # [START bigtable_hw_get_with_filter_legacy] + # [START bigtable_hw_get_by_key_legacy] print("Getting a single greeting by row key.") key = b"greeting0" row = table.read_row(key, row_filter) cell = row.cells[column_family_id.decode("utf-8")][column][0] print(cell.value.decode("utf-8")) - # [END bigtable_hw_get_by_key] - # [END bigtable_hw_get_with_filter] + # [END bigtable_hw_get_by_key_legacy] + # [END bigtable_hw_get_with_filter_legacy] - # [START bigtable_hw_scan_with_filter] - # [START bigtable_hw_scan_all] + # [START bigtable_hw_scan_with_filter_legacy] + # [START bigtable_hw_scan_all_legacy] print("Scanning for all greetings:") partial_rows = table.read_rows(filter_=row_filter) @@ -126,14 +126,14 @@ def main(project_id, instance_id, table_id): column_family_id_str = column_family_id.decode("utf-8") cell = row.cells[column_family_id_str][column][0] print(cell.value.decode("utf-8")) - # [END bigtable_hw_scan_all] - # [END bigtable_hw_scan_with_filter] + # [END bigtable_hw_scan_all_legacy] + # [END bigtable_hw_scan_with_filter_legacy] finally: - # [START bigtable_hw_delete_table] + # [START bigtable_hw_delete_table_legacy] print("Deleting the {} table.".format(table_id)) table.delete() - # [END bigtable_hw_delete_table] + # [END bigtable_hw_delete_table_legacy] if __name__ == "__main__": diff --git a/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py b/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py index 7341bfc46f19..53f1bfbb1906 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/instanceadmin/instanceadmin.py @@ -55,14 +55,14 @@ def run_instance_operations(project_id, instance_id, cluster_id): labels = {"prod-label": "prod-label"} instance = client.instance(instance_id, labels=labels) - # [START bigtable_check_instance_exists] + # [START bigtable_check_instance_exists_legacy] if not instance.exists(): print("Instance {} does not exist.".format(instance_id)) else: print("Instance {} already exists.".format(instance_id)) - # [END bigtable_check_instance_exists] + # [END bigtable_check_instance_exists_legacy] - # [START bigtable_create_prod_instance] + # [START bigtable_create_prod_instance_legacy] cluster = instance.cluster( cluster_id, location_id=location_id, @@ -76,27 +76,27 @@ def run_instance_operations(project_id, instance_id, cluster_id): # Ensure the operation completes. operation.result(timeout=480) print("\nCreated instance: {}".format(instance_id)) - # [END bigtable_create_prod_instance] + # [END bigtable_create_prod_instance_legacy] - # [START bigtable_list_instances] + # [START bigtable_list_instances_legacy] print("\nListing instances:") for instance_local in client.list_instances()[0]: print(instance_local.instance_id) - # [END bigtable_list_instances] + # [END bigtable_list_instances_legacy] - # [START bigtable_get_instance] + # [START bigtable_get_instance_legacy] print( "\nName of instance: {}\nLabels: {}".format( instance.display_name, instance.labels ) ) - # [END bigtable_get_instance] + # [END bigtable_get_instance_legacy] - # [START bigtable_get_clusters] + # [START bigtable_get_clusters_legacy] print("\nListing clusters...") for cluster in instance.list_clusters()[0]: print(cluster.cluster_id) - # [END bigtable_get_clusters] + # [END bigtable_get_clusters_legacy] def delete_instance(project_id, instance_id): @@ -111,14 +111,14 @@ def delete_instance(project_id, instance_id): client = bigtable.Client(project=project_id, admin=True) instance = client.instance(instance_id) - # [START bigtable_delete_instance] + # [START bigtable_delete_instance_legacy] print("\nDeleting instance") if not instance.exists(): print("Instance {} does not exist.".format(instance_id)) else: instance.delete() print("Deleted instance: {}".format(instance_id)) - # [END bigtable_delete_instance] + # [END bigtable_delete_instance_legacy] def add_cluster(project_id, instance_id, cluster_id): @@ -144,7 +144,7 @@ def add_cluster(project_id, instance_id, cluster_id): print("Instance {} does not exist.".format(instance_id)) else: print("\nAdding cluster to instance {}".format(instance_id)) - # [START bigtable_create_cluster] + # [START bigtable_create_cluster_legacy] print("\nListing clusters...") for cluster in instance.list_clusters()[0]: print(cluster.cluster_id) @@ -161,7 +161,7 @@ def add_cluster(project_id, instance_id, cluster_id): # Ensure the operation completes. operation.result(timeout=480) print("\nCluster created: {}".format(cluster_id)) - # [END bigtable_create_cluster] + # [END bigtable_create_cluster_legacy] def delete_cluster(project_id, instance_id, cluster_id): @@ -181,7 +181,7 @@ def delete_cluster(project_id, instance_id, cluster_id): instance = client.instance(instance_id) cluster = instance.cluster(cluster_id) - # [START bigtable_delete_cluster] + # [START bigtable_delete_cluster_legacy] print("\nDeleting cluster") if cluster.exists(): cluster.delete() @@ -189,7 +189,7 @@ def delete_cluster(project_id, instance_id, cluster_id): else: print("\nCluster {} does not exist.".format(cluster_id)) - # [END bigtable_delete_cluster] + # [END bigtable_delete_cluster_legacy] if __name__ == "__main__": diff --git a/packages/google-cloud-bigtable/samples/legacy_client/metricscaler/metricscaler.py b/packages/google-cloud-bigtable/samples/legacy_client/metricscaler/metricscaler.py index 1f89e6aacc15..3703d895794c 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/metricscaler/metricscaler.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/metricscaler/metricscaler.py @@ -38,7 +38,7 @@ def get_cpu_load(bigtable_instance, bigtable_cluster): Returns: float: The most recent Cloud Bigtable CPU usage metric """ - # [START bigtable_cpu] + # [START bigtable_cpu_legacy] client = monitoring_v3.MetricServiceClient() cpu_query = query.Query( client, @@ -51,7 +51,7 @@ def get_cpu_load(bigtable_instance, bigtable_cluster): ) cpu = next(cpu_query.iter()) return cpu.points[0].value.double_value - # [END bigtable_cpu] + # [END bigtable_cpu_legacy] def get_storage_utilization(bigtable_instance, bigtable_cluster): @@ -60,7 +60,7 @@ def get_storage_utilization(bigtable_instance, bigtable_cluster): Returns: float: The most recent Cloud Bigtable storage utilization metric """ - # [START bigtable_metric_scaler_storage_utilization] + # [START bigtable_metric_scaler_storage_utilization_legacy] client = monitoring_v3.MetricServiceClient() utilization_query = query.Query( client, @@ -73,7 +73,7 @@ def get_storage_utilization(bigtable_instance, bigtable_cluster): ) utilization = next(utilization_query.iter()) return utilization.points[0].value.double_value - # [END bigtable_metric_scaler_storage_utilization] + # [END bigtable_metric_scaler_storage_utilization_legacy] def scale_bigtable(bigtable_instance, bigtable_cluster, scale_up): @@ -105,7 +105,7 @@ def scale_bigtable(bigtable_instance, bigtable_cluster, scale_up): # The number of nodes to change the cluster by. size_change_step = 3 - # [START bigtable_scale] + # [START bigtable_scale_legacy] bigtable_client = bigtable.Client(admin=True) instance = bigtable_client.instance(bigtable_instance) instance.reload() @@ -140,7 +140,7 @@ def scale_bigtable(bigtable_instance, bigtable_cluster, scale_up): current_node_count, new_node_count, response.name ) ) - # [END bigtable_scale] + # [END bigtable_scale_legacy] def main( diff --git a/packages/google-cloud-bigtable/samples/legacy_client/quickstart/main.py b/packages/google-cloud-bigtable/samples/legacy_client/quickstart/main.py index 50bfe639426c..a8bfd79a7b36 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/quickstart/main.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/quickstart/main.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# [START bigtable_quickstart] +# [START bigtable_quickstart_legacy] import argparse from google.cloud import bigtable @@ -54,4 +54,4 @@ def main(project_id="project-id", instance_id="instance-id", table_id="my-table" args = parser.parse_args() main(args.project_id, args.instance_id, args.table) -# [END bigtable_quickstart] +# [END bigtable_quickstart_legacy] diff --git a/packages/google-cloud-bigtable/samples/legacy_client/snippets/deletes/deletes_snippets.py b/packages/google-cloud-bigtable/samples/legacy_client/snippets/deletes/deletes_snippets.py index faafe7e319ac..f959aeb95349 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/snippets/deletes/deletes_snippets.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/snippets/deletes/deletes_snippets.py @@ -14,7 +14,7 @@ # limitations under the License. -# [START bigtable_delete_from_column] +# [START bigtable_delete_from_column_legacy] def delete_from_column(project_id, instance_id, table_id): from google.cloud.bigtable import Client @@ -26,10 +26,10 @@ def delete_from_column(project_id, instance_id, table_id): row.commit() -# [END bigtable_delete_from_column] +# [END bigtable_delete_from_column_legacy] -# [START bigtable_delete_from_column_family] +# [START bigtable_delete_from_column_family_legacy] def delete_from_column_family(project_id, instance_id, table_id): from google.cloud.bigtable import Client @@ -41,10 +41,10 @@ def delete_from_column_family(project_id, instance_id, table_id): row.commit() -# [END bigtable_delete_from_column_family] +# [END bigtable_delete_from_column_family_legacy] -# [START bigtable_delete_from_row] +# [START bigtable_delete_from_row_legacy] def delete_from_row(project_id, instance_id, table_id): from google.cloud.bigtable import Client @@ -56,10 +56,10 @@ def delete_from_row(project_id, instance_id, table_id): row.commit() -# [END bigtable_delete_from_row] +# [END bigtable_delete_from_row_legacy] -# [START bigtable_streaming_and_batching] +# [START bigtable_streaming_and_batching_legacy] def streaming_and_batching(project_id, instance_id, table_id): from google.cloud.bigtable import Client @@ -75,10 +75,10 @@ def streaming_and_batching(project_id, instance_id, table_id): batcher.mutate(direct_row) -# [END bigtable_streaming_and_batching] +# [END bigtable_streaming_and_batching_legacy] -# [START bigtable_check_and_mutate] +# [START bigtable_check_and_mutate_legacy] def check_and_mutate(project_id, instance_id, table_id): from google.cloud.bigtable import Client @@ -91,10 +91,10 @@ def check_and_mutate(project_id, instance_id, table_id): row.commit() -# [END bigtable_check_and_mutate] +# [END bigtable_check_and_mutate_legacy] -# [START bigtable_drop_row_range] +# [START bigtable_drop_row_range_legacy] def drop_row_range(project_id, instance_id, table_id): from google.cloud.bigtable import Client @@ -105,10 +105,10 @@ def drop_row_range(project_id, instance_id, table_id): table.drop_by_prefix(row_key_prefix, timeout=200) -# [END bigtable_drop_row_range] +# [END bigtable_drop_row_range_legacy] -# [START bigtable_delete_column_family] +# [START bigtable_delete_column_family_legacy] def delete_column_family(project_id, instance_id, table_id): from google.cloud.bigtable import Client @@ -120,10 +120,10 @@ def delete_column_family(project_id, instance_id, table_id): column_family_obj.delete() -# [END bigtable_delete_column_family] +# [END bigtable_delete_column_family_legacy] -# [START bigtable_delete_table] +# [START bigtable_delete_table_legacy] def delete_table(project_id, instance_id, table_id): from google.cloud.bigtable import Client @@ -133,4 +133,4 @@ def delete_table(project_id, instance_id, table_id): table.delete() -# [END bigtable_delete_table] +# [END bigtable_delete_table_legacy] diff --git a/packages/google-cloud-bigtable/samples/legacy_client/snippets/filters/filter_snippets.py b/packages/google-cloud-bigtable/samples/legacy_client/snippets/filters/filter_snippets.py index f2a1a0fd0a06..911c64f97922 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/snippets/filters/filter_snippets.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/snippets/filters/filter_snippets.py @@ -14,7 +14,7 @@ # limitations under the License. -# [START bigtable_filters_limit_row_sample] +# [START bigtable_filters_limit_row_sample_legacy] def filter_limit_row_sample(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable import row_filters @@ -28,8 +28,8 @@ def filter_limit_row_sample(project_id, instance_id, table_id): print_row(row) -# [END bigtable_filters_limit_row_sample] -# [START bigtable_filters_limit_row_regex] +# [END bigtable_filters_limit_row_sample_legacy] +# [START bigtable_filters_limit_row_regex_legacy] def filter_limit_row_regex(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable import row_filters @@ -45,8 +45,8 @@ def filter_limit_row_regex(project_id, instance_id, table_id): print_row(row) -# [END bigtable_filters_limit_row_regex] -# [START bigtable_filters_limit_cells_per_col] +# [END bigtable_filters_limit_row_regex_legacy] +# [START bigtable_filters_limit_cells_per_col_legacy] def filter_limit_cells_per_col(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable import row_filters @@ -60,8 +60,8 @@ def filter_limit_cells_per_col(project_id, instance_id, table_id): print_row(row) -# [END bigtable_filters_limit_cells_per_col] -# [START bigtable_filters_limit_cells_per_row] +# [END bigtable_filters_limit_cells_per_col_legacy] +# [START bigtable_filters_limit_cells_per_row_legacy] def filter_limit_cells_per_row(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable import row_filters @@ -75,8 +75,8 @@ def filter_limit_cells_per_row(project_id, instance_id, table_id): print_row(row) -# [END bigtable_filters_limit_cells_per_row] -# [START bigtable_filters_limit_cells_per_row_offset] +# [END bigtable_filters_limit_cells_per_row_legacy] +# [START bigtable_filters_limit_cells_per_row_offset_legacy] def filter_limit_cells_per_row_offset(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable import row_filters @@ -90,8 +90,8 @@ def filter_limit_cells_per_row_offset(project_id, instance_id, table_id): print_row(row) -# [END bigtable_filters_limit_cells_per_row_offset] -# [START bigtable_filters_limit_col_family_regex] +# [END bigtable_filters_limit_cells_per_row_offset_legacy] +# [START bigtable_filters_limit_col_family_regex_legacy] def filter_limit_col_family_regex(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable import row_filters @@ -107,8 +107,8 @@ def filter_limit_col_family_regex(project_id, instance_id, table_id): print_row(row) -# [END bigtable_filters_limit_col_family_regex] -# [START bigtable_filters_limit_col_qualifier_regex] +# [END bigtable_filters_limit_col_family_regex_legacy] +# [START bigtable_filters_limit_col_qualifier_regex_legacy] def filter_limit_col_qualifier_regex(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable import row_filters @@ -124,8 +124,8 @@ def filter_limit_col_qualifier_regex(project_id, instance_id, table_id): print_row(row) -# [END bigtable_filters_limit_col_qualifier_regex] -# [START bigtable_filters_limit_col_range] +# [END bigtable_filters_limit_col_qualifier_regex_legacy] +# [START bigtable_filters_limit_col_range_legacy] def filter_limit_col_range(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable import row_filters @@ -143,8 +143,8 @@ def filter_limit_col_range(project_id, instance_id, table_id): print_row(row) -# [END bigtable_filters_limit_col_range] -# [START bigtable_filters_limit_value_range] +# [END bigtable_filters_limit_col_range_legacy] +# [START bigtable_filters_limit_value_range_legacy] def filter_limit_value_range(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable import row_filters @@ -161,8 +161,8 @@ def filter_limit_value_range(project_id, instance_id, table_id): print_row(row) -# [END bigtable_filters_limit_value_range] -# [START bigtable_filters_limit_value_regex] +# [END bigtable_filters_limit_value_range_legacy] +# [START bigtable_filters_limit_value_regex_legacy] def filter_limit_value_regex(project_id, instance_id, table_id): @@ -180,8 +180,8 @@ def filter_limit_value_regex(project_id, instance_id, table_id): print_row(row) -# [END bigtable_filters_limit_value_regex] -# [START bigtable_filters_limit_timestamp_range] +# [END bigtable_filters_limit_value_regex_legacy] +# [START bigtable_filters_limit_timestamp_range_legacy] def filter_limit_timestamp_range(project_id, instance_id, table_id): import datetime @@ -201,8 +201,8 @@ def filter_limit_timestamp_range(project_id, instance_id, table_id): print_row(row) -# [END bigtable_filters_limit_timestamp_range] -# [START bigtable_filters_limit_block_all] +# [END bigtable_filters_limit_timestamp_range_legacy] +# [START bigtable_filters_limit_block_all_legacy] def filter_limit_block_all(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable import row_filters @@ -216,8 +216,8 @@ def filter_limit_block_all(project_id, instance_id, table_id): print_row(row) -# [END bigtable_filters_limit_block_all] -# [START bigtable_filters_limit_pass_all] +# [END bigtable_filters_limit_block_all_legacy] +# [START bigtable_filters_limit_pass_all_legacy] def filter_limit_pass_all(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable import row_filters @@ -231,8 +231,8 @@ def filter_limit_pass_all(project_id, instance_id, table_id): print_row(row) -# [END bigtable_filters_limit_pass_all] -# [START bigtable_filters_modify_strip_value] +# [END bigtable_filters_limit_pass_all_legacy] +# [START bigtable_filters_modify_strip_value_legacy] def filter_modify_strip_value(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable import row_filters @@ -246,8 +246,8 @@ def filter_modify_strip_value(project_id, instance_id, table_id): print_row(row) -# [END bigtable_filters_modify_strip_value] -# [START bigtable_filters_modify_apply_label] +# [END bigtable_filters_modify_strip_value_legacy] +# [START bigtable_filters_modify_apply_label_legacy] def filter_modify_apply_label(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable import row_filters @@ -261,8 +261,8 @@ def filter_modify_apply_label(project_id, instance_id, table_id): print_row(row) -# [END bigtable_filters_modify_apply_label] -# [START bigtable_filters_composing_chain] +# [END bigtable_filters_modify_apply_label_legacy] +# [START bigtable_filters_composing_chain_legacy] def filter_composing_chain(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable import row_filters @@ -283,8 +283,8 @@ def filter_composing_chain(project_id, instance_id, table_id): print_row(row) -# [END bigtable_filters_composing_chain] -# [START bigtable_filters_composing_interleave] +# [END bigtable_filters_composing_chain_legacy] +# [START bigtable_filters_composing_interleave_legacy] def filter_composing_interleave(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable import row_filters @@ -305,8 +305,8 @@ def filter_composing_interleave(project_id, instance_id, table_id): print_row(row) -# [END bigtable_filters_composing_interleave] -# [START bigtable_filters_composing_condition] +# [END bigtable_filters_composing_interleave_legacy] +# [START bigtable_filters_composing_condition_legacy] def filter_composing_condition(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable import row_filters @@ -331,10 +331,10 @@ def filter_composing_condition(project_id, instance_id, table_id): print_row(row) -# [END bigtable_filters_composing_condition] +# [END bigtable_filters_composing_condition_legacy] -# [START bigtable_filters_print] +# [START bigtable_filters_print_legacy] def print_row(row): print("Reading data for {}:".format(row.row_key.decode("utf-8"))) for cf, cols in sorted(row.cells.items()): @@ -355,4 +355,4 @@ def print_row(row): print("") -# [END bigtable_filters_print] +# [END bigtable_filters_print_legacy] diff --git a/packages/google-cloud-bigtable/samples/legacy_client/snippets/reads/read_snippets.py b/packages/google-cloud-bigtable/samples/legacy_client/snippets/reads/read_snippets.py index 1d4ee3d8e650..bb14dfbafd5b 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/snippets/reads/read_snippets.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/snippets/reads/read_snippets.py @@ -14,7 +14,7 @@ # limitations under the License. -# [START bigtable_reads_row] +# [START bigtable_reads_row_legacy] def read_row(project_id, instance_id, table_id): from google.cloud import bigtable @@ -28,10 +28,10 @@ def read_row(project_id, instance_id, table_id): print_row(row) -# [END bigtable_reads_row] +# [END bigtable_reads_row_legacy] -# [START bigtable_reads_row_partial] +# [START bigtable_reads_row_partial_legacy] def read_row_partial(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable import row_filters @@ -47,8 +47,8 @@ def read_row_partial(project_id, instance_id, table_id): print_row(row) -# [END bigtable_reads_row_partial] -# [START bigtable_reads_rows] +# [END bigtable_reads_row_partial_legacy] +# [START bigtable_reads_rows_legacy] def read_rows(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable.row_set import RowSet @@ -66,8 +66,8 @@ def read_rows(project_id, instance_id, table_id): print_row(row) -# [END bigtable_reads_rows] -# [START bigtable_reads_row_range] +# [END bigtable_reads_rows_legacy] +# [START bigtable_reads_row_range_legacy] def read_row_range(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable.row_set import RowSet @@ -86,8 +86,8 @@ def read_row_range(project_id, instance_id, table_id): print_row(row) -# [END bigtable_reads_row_range] -# [START bigtable_reads_row_ranges] +# [END bigtable_reads_row_range_legacy] +# [START bigtable_reads_row_ranges_legacy] def read_row_ranges(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable.row_set import RowSet @@ -109,8 +109,8 @@ def read_row_ranges(project_id, instance_id, table_id): print_row(row) -# [END bigtable_reads_row_ranges] -# [START bigtable_reads_prefix] +# [END bigtable_reads_row_ranges_legacy] +# [START bigtable_reads_prefix_legacy] def read_prefix(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable.row_set import RowSet @@ -129,8 +129,8 @@ def read_prefix(project_id, instance_id, table_id): print_row(row) -# [END bigtable_reads_prefix] -# [START bigtable_reads_filter] +# [END bigtable_reads_prefix_legacy] +# [START bigtable_reads_filter_legacy] def read_filter(project_id, instance_id, table_id): from google.cloud import bigtable from google.cloud.bigtable import row_filters @@ -144,10 +144,10 @@ def read_filter(project_id, instance_id, table_id): print_row(row) -# [END bigtable_reads_filter] +# [END bigtable_reads_filter_legacy] -# [START bigtable_reads_print] +# [START bigtable_reads_print_legacy] def print_row(row): print("Reading data for {}:".format(row.row_key.decode("utf-8"))) for cf, cols in sorted(row.cells.items()): @@ -168,4 +168,4 @@ def print_row(row): print("") -# [END bigtable_reads_print] +# [END bigtable_reads_print_legacy] diff --git a/packages/google-cloud-bigtable/samples/legacy_client/snippets/writes/write_batch.py b/packages/google-cloud-bigtable/samples/legacy_client/snippets/writes/write_batch.py index a583bb7134e1..a703c9ed67af 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/snippets/writes/write_batch.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/snippets/writes/write_batch.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# [START bigtable_writes_batch] +# [START bigtable_writes_batch_legacy] from datetime import datetime, timezone from google.cloud import bigtable @@ -43,4 +43,4 @@ def write_batch(project_id, instance_id, table_id): print("Successfully wrote 2 rows.") -# [END bigtable_writes_batch] +# [END bigtable_writes_batch_legacy] diff --git a/packages/google-cloud-bigtable/samples/legacy_client/snippets/writes/write_conditionally.py b/packages/google-cloud-bigtable/samples/legacy_client/snippets/writes/write_conditionally.py index b6f05fba77f4..b4b5194fb785 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/snippets/writes/write_conditionally.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/snippets/writes/write_conditionally.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# [START bigtable_writes_conditional] +# [START bigtable_writes_conditional_legacy] from datetime import datetime, timezone from google.cloud import bigtable @@ -43,4 +43,4 @@ def write_conditional(project_id, instance_id, table_id): print("Successfully updated row's os_name.") -# [END bigtable_writes_conditional] +# [END bigtable_writes_conditional_legacy] diff --git a/packages/google-cloud-bigtable/samples/legacy_client/snippets/writes/write_increment.py b/packages/google-cloud-bigtable/samples/legacy_client/snippets/writes/write_increment.py index ac8e2d16af34..3b282d1bf962 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/snippets/writes/write_increment.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/snippets/writes/write_increment.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# [START bigtable_writes_increment] +# [START bigtable_writes_increment_legacy] from google.cloud import bigtable @@ -33,4 +33,4 @@ def write_increment(project_id, instance_id, table_id): print("Successfully updated row {}.".format(row_key)) -# [END bigtable_writes_increment] +# [END bigtable_writes_increment_legacy] diff --git a/packages/google-cloud-bigtable/samples/legacy_client/snippets/writes/write_simple.py b/packages/google-cloud-bigtable/samples/legacy_client/snippets/writes/write_simple.py index fb7074bc526e..46313a675dba 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/snippets/writes/write_simple.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/snippets/writes/write_simple.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# [START bigtable_writes_simple] +# [START bigtable_writes_simple_legacy] from datetime import datetime, timezone from google.cloud import bigtable @@ -39,4 +39,4 @@ def write_simple(project_id, instance_id, table_id): print("Successfully wrote row {}.".format(row_key)) -# [END bigtable_writes_simple] +# [END bigtable_writes_simple_legacy] diff --git a/packages/google-cloud-bigtable/samples/legacy_client/tableadmin/tableadmin.py b/packages/google-cloud-bigtable/samples/legacy_client/tableadmin/tableadmin.py index a9b8446db7e5..8c6e76340432 100644 --- a/packages/google-cloud-bigtable/samples/legacy_client/tableadmin/tableadmin.py +++ b/packages/google-cloud-bigtable/samples/legacy_client/tableadmin/tableadmin.py @@ -55,7 +55,7 @@ def run_table_operations(project_id, instance_id, table_id): client = bigtable.Client(project=project_id, admin=True) instance = client.instance(instance_id) with create_table_cm(project_id, instance_id, table_id, verbose=False) as table: - # [START bigtable_list_tables] + # [START bigtable_list_tables_legacy] tables = instance.list_tables() print("Listing tables in current project...") if tables != []: @@ -63,9 +63,9 @@ def run_table_operations(project_id, instance_id, table_id): print(tbl.table_id) else: print("No table exists in current project...") - # [END bigtable_list_tables] + # [END bigtable_list_tables_legacy] - # [START bigtable_create_family_gc_max_age] + # [START bigtable_create_family_gc_max_age_legacy] print("Creating column family cf1 with with MaxAge GC Rule...") # Create a column family with GC policy : maximum age # where age = current time minus cell timestamp @@ -76,9 +76,9 @@ def run_table_operations(project_id, instance_id, table_id): column_family1 = table.column_family("cf1", max_age_rule) column_family1.create() print("Created column family cf1 with MaxAge GC Rule.") - # [END bigtable_create_family_gc_max_age] + # [END bigtable_create_family_gc_max_age_legacy] - # [START bigtable_create_family_gc_max_versions] + # [START bigtable_create_family_gc_max_versions_legacy] print("Creating column family cf2 with max versions GC rule...") # Create a column family with GC policy : most recent N versions # where 1 = most recent version @@ -89,9 +89,9 @@ def run_table_operations(project_id, instance_id, table_id): column_family2 = table.column_family("cf2", max_versions_rule) column_family2.create() print("Created column family cf2 with Max Versions GC Rule.") - # [END bigtable_create_family_gc_max_versions] + # [END bigtable_create_family_gc_max_versions_legacy] - # [START bigtable_create_family_gc_union] + # [START bigtable_create_family_gc_union_legacy] print("Creating column family cf3 with union GC rule...") # Create a column family with GC policy to drop data that matches # at least one condition. @@ -107,9 +107,9 @@ def run_table_operations(project_id, instance_id, table_id): column_family3 = table.column_family("cf3", union_rule) column_family3.create() print("Created column family cf3 with Union GC rule") - # [END bigtable_create_family_gc_union] + # [END bigtable_create_family_gc_union_legacy] - # [START bigtable_create_family_gc_intersection] + # [START bigtable_create_family_gc_intersection_legacy] print("Creating column family cf4 with Intersection GC rule...") # Create a column family with GC policy to drop data that matches # all conditions @@ -125,9 +125,9 @@ def run_table_operations(project_id, instance_id, table_id): column_family4 = table.column_family("cf4", intersection_rule) column_family4.create() print("Created column family cf4 with Intersection GC rule.") - # [END bigtable_create_family_gc_intersection] + # [END bigtable_create_family_gc_intersection_legacy] - # [START bigtable_create_family_gc_nested] + # [START bigtable_create_family_gc_nested_legacy] print("Creating column family cf5 with a Nested GC rule...") # Create a column family with nested GC policies. # Create a nested GC rule: @@ -148,9 +148,9 @@ def run_table_operations(project_id, instance_id, table_id): column_family5 = table.column_family("cf5", nested_rule) column_family5.create() print("Created column family cf5 with a Nested GC rule.") - # [END bigtable_create_family_gc_nested] + # [END bigtable_create_family_gc_nested_legacy] - # [START bigtable_list_column_families] + # [START bigtable_list_column_families_legacy] print("Printing Column Family and GC Rule for all column families...") column_families = table.list_column_families() for column_family_name, gc_rule in sorted(column_families.items()): @@ -172,30 +172,30 @@ def run_table_operations(project_id, instance_id, table_id): # } # } # } - # [END bigtable_list_column_families] + # [END bigtable_list_column_families_legacy] print("Print column family cf1 GC rule before update...") print("Column Family: cf1") print(column_family1.to_pb()) - # [START bigtable_update_gc_rule] + # [START bigtable_update_gc_rule_legacy] print("Updating column family cf1 GC rule...") # Update the column family cf1 to update the GC rule column_family1 = table.column_family("cf1", column_family.MaxVersionsGCRule(1)) column_family1.update() print("Updated column family cf1 GC rule\n") - # [END bigtable_update_gc_rule] + # [END bigtable_update_gc_rule_legacy] print("Print column family cf1 GC rule after update...") print("Column Family: cf1") print(column_family1.to_pb()) - # [START bigtable_delete_family] + # [START bigtable_delete_family_legacy] print("Delete a column family cf2...") # Delete a column family column_family2.delete() print("Column family cf2 deleted successfully.") - # [END bigtable_delete_family] + # [END bigtable_delete_family_legacy] print( 'execute command "python tableadmin.py delete [project_id] \ @@ -220,7 +220,7 @@ def delete_table(project_id, instance_id, table_id): instance = client.instance(instance_id) table = instance.table(table_id) - # [START bigtable_delete_table] + # [START bigtable_delete_table_legacy] # Delete the entire table print("Checking if table {} exists...".format(table_id)) @@ -231,7 +231,7 @@ def delete_table(project_id, instance_id, table_id): print("Deleted {} table.".format(table_id)) else: print("Table {} does not exists.".format(table_id)) - # [END bigtable_delete_table] + # [END bigtable_delete_table_legacy] if __name__ == "__main__": diff --git a/packages/google-cloud-bigtable/samples/utils.py b/packages/google-cloud-bigtable/samples/utils.py index d093d0427cbf..62cf1049d401 100644 --- a/packages/google-cloud-bigtable/samples/utils.py +++ b/packages/google-cloud-bigtable/samples/utils.py @@ -17,7 +17,7 @@ from google.api_core import exceptions from google.api_core.retry import Retry, if_exception_type -from google.cloud import bigtable +from google.cloud import bigtable, bigtable_admin from google.cloud.bigtable.column_family import ColumnFamily from google.cloud.bigtable_admin_v2.types import ColumnFamily as ColumnFamily_pb @@ -44,11 +44,12 @@ def __enter__(self): return self._table def __exit__(self, *args): - if self._table.exists(): + admin_client = bigtable_admin.BigtableTableAdminClient() + try: if self._verbose: print(f"deleting table: {self._table.table_id}") - delete_retry(self._table.delete()) - else: + delete_retry(admin_client.delete_table)(name=self._table.name) + except exceptions.NotFound: if self._verbose: print(f"table {self._table.table_id} not found") @@ -57,12 +58,18 @@ def create_table(project, instance_id, table_id, column_families={}): """ Creates a new table, and blocks until it reaches a ready state """ + admin_client = bigtable_admin.BigtableTableAdminClient() + instance_path = admin_client.instance_path(project, instance_id) + table_path = admin_client.table_path(project, instance_id, table_id) + + try: + admin_client.delete_table(name=table_path) + except exceptions.NotFound: + pass + client = bigtable.Client(project=project, admin=True) instance = client.instance(instance_id) - table = instance.table(table_id) - if table.exists(): - table.delete() # convert column families to pb if needed pb_families = { @@ -72,16 +79,15 @@ def create_table(project, instance_id, table_id, column_families={}): for (id, rule) in column_families.items() } - # create table using gapic layer - instance._client.table_admin_client.create_table( + admin_client.create_table( request={ - "parent": instance.name, + "parent": instance_path, "table_id": table_id, "table": {"column_families": pb_families}, } ) - wait_for_table(table) + wait_for_table(table_path) return table @@ -101,5 +107,15 @@ def wait_for_table(table): Because this method is wrapped with an api_core.Retry decorator, it will retry with backoff if the table is not ready """ - if not table.exists(): - raise exceptions.NotFound + if hasattr(table, "exists"): + if not table.exists(): + raise exceptions.NotFound + else: + table_path = table.table_name if hasattr(table, "table_name") else str(table) + admin_client = bigtable_admin.BigtableTableAdminClient() + admin_client.get_table( + request={ + "name": table_path, + "view": bigtable_admin.Table.View.NAME_ONLY, + } + )