From 26023e9fa5e2426bf4d90d4cef5896515f0a2a7d Mon Sep 17 00:00:00 2001 From: Sayali Bhavsar Date: Mon, 27 Jul 2026 15:13:57 +0530 Subject: [PATCH 1/5] Initial commit: add CockroachDB benchmark wrapper Add wrapper scripts, configuration, and documentation for running CockroachDB benchmarks in the CPT pipeline. --- LICENSE | 280 ++++++++++ README.md | 285 ++++++++++ cockroachdb.json | 20 + cockroachdb/cockroachdb_run | 521 ++++++++++++++++++ cockroachdb/openmetrics_cockroachdb_reset.txt | 8 + cockroachdb/test_opts | 5 + license | 17 + result_schema.py | 19 + 8 files changed, 1155 insertions(+) create mode 100644 LICENSE create mode 100644 README.md create mode 100644 cockroachdb.json create mode 100755 cockroachdb/cockroachdb_run create mode 100644 cockroachdb/openmetrics_cockroachdb_reset.txt create mode 100644 cockroachdb/test_opts create mode 100644 license create mode 100644 result_schema.py diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d8cf7d4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,280 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS diff --git a/README.md b/README.md new file mode 100644 index 0000000..8e079fd --- /dev/null +++ b/README.md @@ -0,0 +1,285 @@ +# CockroachDB Benchmark Wrapper + +## Description + +This wrapper facilitates the automated execution of CockroachDB benchmarks using the `cockroach workload` tool. CockroachDB is used here as a system-level benchmark that stresses CPU, memory, kernel scheduler, and storage subsystems by driving SQL workloads against a single-node cluster. The primary metric is operations per second (ops/sec). + +The wrapper provides: +- Automated CockroachDB binary download and execution. +- Support for x86_64 and aarch64 architectures. +- Configurable workloads (KV and MoVR) with adjustable concurrency, duration, and ramp time. +- Configurable CockroachDB version selection. +- Multi-iteration testing with trimmed mean and deviation calculation. +- Result collection, processing, and verification. +- CSV and JSON output formats. +- System configuration metadata capture. +- Integration with test_tools framework. +- Optional Performance Co-Pilot (PCP) integration. + +## Command-Line Options + +``` +CockroachDB Wrapper Options: + --cockroach_version : CockroachDB version to download (e.g., "v24.3.33"). + Defaults to v22.2.0. Versions >= v23.2 require x86-64-v2 CPU support. + --concurrency : Comma-separated list of concurrency levels to test + (e.g., "128,256,512,1024"). Defaults to 128,256,512,1024. + --duration : How long each workload runs (e.g., "90s", "120s"). + Defaults to 90s. + --opts_file : Workload configuration file to use. + Defaults to test_opts in the script directory. + --ramp : Ramp-up period before measurement begins (e.g., "10s"). + Defaults to 10s. + +General test_tools options: + --debug: Enables bash -x output, useful for debugging issues with wrappers. + --home_parent : Parent home directory. If not set, defaults to current working directory. + --host_config : Host configuration name, defaults to current hostname. + --iterations : Number of times to run the test, defaults to 1. + --run_user: User that is actually running the test on the test system. Defaults to current user. + --sys_type: Type of system working with (aws, azure, hostname). Defaults to hostname. + --sysname: Name of the system running, used in determining config files. Defaults to hostname. + --tuned_setting: Used in naming the results directory. For RHEL, defaults to current active tuned profile. + For non-RHEL systems, defaults to 'none'. + --use_pcp: Enable Performance Co-Pilot monitoring during test execution. + --tools_git : Git repo to retrieve the required tools from. + Default: https://github.com/redhat-performance/test_tools-wrappers + --usage: Display this usage message. +``` + +## What the Script Does + +The `cockroachdb_run` script performs the following workflow: + +1. **Environment Setup**: + - Clones the test_tools-wrappers repository if not present (default: ~/test_tools). + - Tries wget, then curl, then git clone to obtain the tools. + - Sources error codes and general setup utilities. + - Gathers system hardware information. + +2. **Package Installation**: + - Installs required dependencies via package_tool: bc, zip. + - Dependencies are defined in cockroachdb.json for different OS variants (RHEL, Ubuntu, SLES, Amazon Linux). + +3. **CockroachDB Installation**: + - Downloads the CockroachDB binary tarball from `binaries.cockroachdb.com`. + - Selects the correct binary for the system architecture (x86_64 or aarch64). + - Extracts the binary to `cockroach-build/` in the script directory. + - Skips download if the binary is already present. + +4. **PCP Setup** (optional): + - If `--use_pcp` is specified, initializes Performance Co-Pilot monitoring. + - Creates a timestamped PCP data directory at `/tmp/pcp_/`. + - Resets all OpenMetrics values between workload runs. + +5. **Test Execution**: + - Reads the test configuration file (`test_opts` by default) line by line. + - Each line defines a workload name and corresponding `cockroach workload run` flags. + - For each workload and concurrency level: + - Starts a fresh CockroachDB single-node insecure cluster on localhost. + - Runs the workload with configured duration and ramp time. + - Parses ops/sec(cum) from the final summary line of the workload output. + - Stops CockroachDB and removes the data directory. + - Records start and end timestamps for each workload run. + - Runs each configuration for the specified number of iterations (`--iterations`). + +6. **Data Collection**: + - Extracts ops/sec from each run's raw output. + - Generates per-iteration CSV files (`cockroachdb_iter_N.csv`) with system metadata header. + - Optionally records PCP performance data (iteration, concurrency, average). + +7. **Verification**: + - Converts CSV to JSON via `csv_to_json`. + - Validates results against Pydantic schema (`result_schema.py`) ensuring: + - All workload names match expected Workload enum values. + - All Average values are valid floats. + - Timestamps are valid datetime objects. + +8. **Combined Report**: + - Generates `results_cockroachdb.csv` combining all iterations. + - Computes average ops/sec across iterations for each workload+concurrency combination. + - For 5 or more iterations, applies trimmed mean (drops highest and lowest values). + - Computes deviation as a percentage (standard deviation / average x 100). + - Includes system metadata header, timestamps, and all result fields. + +9. **Output**: + - Creates timestamped results directory in `${HOME}/export_results/cockroachdb_`. + - Saves all raw output files, processed CSV/JSON, and system metadata. + - Optionally saves PCP performance data. + - Archives results to configured storage location. + +## Dependencies + +Location of underlying workload: Downloaded from https://binaries.cockroachdb.com. + +**General packages required**: bc, zip + +To run: +```bash +git clone https://github.com/redhat-performance/cockroachdb-wrapper +cd cockroachdb-wrapper/cockroachdb +./cockroachdb_run +``` + +The script will automatically detect your CPU architecture and download the appropriate CockroachDB binary. + +## The CockroachDB Benchmark + +CockroachDB is a distributed SQL database. In this wrapper it is used as a system-level benchmark, not a database benchmark. A single-node insecure cluster runs on localhost and workloads drive SQL operations to stress the system. + +### Workloads + +The default `test_opts` file defines 5 workload configurations: + +**KV Workloads** create a table `kv (k BIGINT PRIMARY KEY, v BYTES)` and run concurrent `SELECT` and `UPSERT` operations with configurable read/write ratios: + +| Name | Description | +|------|-------------| +| kv_95pct_reads | KV workload with 95% reads, 5% writes | +| kv_50pct_reads | KV workload with 50% reads, 50% writes | +| kv_60pct_reads | KV workload with 60% reads, 40% writes | +| kv_10pct_reads | KV workload with 10% reads, 90% writes | + +**MoVR Workload** simulates a ride-sharing application with 6 tables (users, vehicles, rides, promo_codes, user_promo_codes, vehicle_location_histories): + +| Name | Description | +|------|-------------| +| movr | MoVR ride-sharing simulation workload | + +### Key Parameters + +1. **Concurrency**: Number of concurrent worker goroutines driving SQL operations. Default levels: 128, 256, 512, 1024. Higher concurrency stresses the system more heavily. + +2. **Duration**: How long each workload runs after the ramp-up period. Default: 90 seconds. + +3. **Ramp**: Warm-up period before measurement begins. Default: 10 seconds. Operations during ramp are not counted in the final ops/sec metric. + +4. **Performance Metric**: The primary metric is **ops/sec(cum)** — cumulative operations per second over the measurement period. Higher values indicate better system throughput. + +### CockroachDB Version Compatibility + +- **v22.2.0** (default): Works on all platforms. Requires glibc >= 2.17, no special CPU instruction requirements. +- **v23.2+**: Requires **x86-64-v2** CPU support (SSE4.2, SSSE3, POPCNT). All modern CPUs (Intel 3rd gen+, AMD Ryzen+) support this, but older hardware will fail with `Fatal glibc error: CPU does not support x86-64-v2`. + +## Output Files + +The results directory contains: + +- **results_cockroachdb.csv**: Combined CSV with system metadata, averaged results, and deviation across iterations. +- **cockroachdb_iter_N.csv**: Per-iteration CSV with individual workload results and timestamps. +- **raw_\_c\_iter\.out**: Raw output from each `cockroach workload run` invocation. +- **cockroach_start.log**: CockroachDB server startup log. +- **meta_data*.yml**: System metadata (CPU info, memory, NUMA topology, kernel version). +- **PCP data** (if `--use_pcp` option used): Performance Co-Pilot monitoring data. + +## Examples + +### Basic run with defaults +```bash +./cockroachdb_run +``` +This runs with: +- CockroachDB v22.2.0 +- All 5 workloads from test_opts +- Concurrency levels 128, 256, 512, 1024 +- 90 second duration with 10 second ramp +- 1 iteration + +### Quick test run +```bash +./cockroachdb_run --concurrency "128" --duration 10s --ramp 5s +``` +Runs all workloads at a single concurrency level with short duration for quick validation. + +### Run with a newer CockroachDB version +```bash +./cockroachdb_run --cockroach_version v24.3.33 +``` +Downloads and uses CockroachDB v24.3.33 instead of the default v22.2.0. + +### Run multiple iterations +```bash +./cockroachdb_run --iterations 3 +``` +Runs the full test 3 times and averages results with deviation calculation. + +### Run multiple iterations with trimmed mean +```bash +./cockroachdb_run --iterations 5 +``` +Runs 5 iterations. The highest and lowest values are dropped before averaging (trimmed mean). + +### Run with PCP monitoring +```bash +./cockroachdb_run --use_pcp +``` +Collects Performance Co-Pilot data during the run. + +### Run with debug output +```bash +./cockroachdb_run --debug --concurrency "128" --duration 10s --ramp 5s +``` +Enables bash -x output for troubleshooting. + +### Combination example +```bash +./cockroachdb_run --cockroach_version v24.3.33 --concurrency "128,256" --duration 60s --iterations 3 --use_pcp +``` +Uses CockroachDB v24.3.33, runs 2 concurrency levels, 60 second duration, 3 iterations, with PCP monitoring. + +## How Result Averaging Works + +The script computes combined results across multiple iterations: + +### Average (ops/sec) +1. Collects ops/sec values from each iteration for every workload+concurrency combination. +2. For fewer than 5 iterations: computes a simple arithmetic mean. +3. For 5 or more iterations: applies a **trimmed mean** — drops the highest and lowest values, then averages the remaining samples. This reduces the impact of outliers. + +### Deviation +1. Computes standard deviation across all iteration values. +2. Expresses deviation as a percentage of the average: `(stddev / average) x 100`. +3. For single-iteration runs, deviation is 0. +4. Low deviation (< 2%) indicates consistent, reliable results. High deviation suggests system instability or interference from other workloads. + +## Return Codes + +The script uses standardized error codes from test_tools error_codes: +- **0**: Success +- **101**: Git clone failure (test_tools) +- **102**: CockroachDB binary download failure +- **103**: CockroachDB failed to start + +Exit codes indicate specific failure points for automated testing workflows. + +## Notes + +### Architecture Support +- **x86_64**: Full support. Default binary from `cockroach-.linux-amd64.tgz`. +- **aarch64**: Full support. Uses `cockroach-.linux-3.7.10-gnu-aarch64.tgz`. + +### CockroachDB Cluster Configuration +- Runs in **single-node insecure mode** on localhost (127.0.0.1:26257). +- Uses 25% of system memory for CockroachDB cache (`--cache .25`). +- Data directory is created and destroyed for each workload+concurrency combination to ensure a clean state. +- HTTP admin UI is available on port 8080 during test runs. + +### Workload Configuration +- Custom workloads can be defined by creating a new test_opts file and using `--opts_file`. +- Each line format: `workload_name,cockroach workload run flags` +- The `--drop` flag is always passed to workload run, which drops and recreates the workload database each time. + +### Performance Tips +- Run multiple iterations to verify consistency and get meaningful deviation values. +- Ensure the system is idle (no other workloads) for best results. +- For production benchmarking, use 5+ iterations to enable trimmed mean. +- Consider the active tuned profile on RHEL systems. +- Higher concurrency levels may be limited by available CPU cores. + +### Troubleshooting +- If CockroachDB fails to start, check `cockroach_start.log` for errors. +- If download fails, verify internet access to `binaries.cockroachdb.com`. +- If ops/sec is 0, check raw output files for workload errors. +- If using v23.2+, verify CPU supports x86-64-v2 instructions. +- Use `--debug` to enable bash -x output for detailed execution tracing. +- If `save_results` fails, ensure `zip` is installed. diff --git a/cockroachdb.json b/cockroachdb.json new file mode 100644 index 0000000..b4eb172 --- /dev/null +++ b/cockroachdb.json @@ -0,0 +1,20 @@ +{ + "dependencies": { + "rhel": [ + "bc", + "zip" + ], + "ubuntu": [ + "bc", + "zip" + ], + "sles": [ + "bc", + "zip" + ], + "amzn": [ + "bc", + "zip" + ] + } +} diff --git a/cockroachdb/cockroachdb_run b/cockroachdb/cockroachdb_run new file mode 100755 index 0000000..6fcafee --- /dev/null +++ b/cockroachdb/cockroachdb_run @@ -0,0 +1,521 @@ +#!/bin/bash +# +# License +# +#================================================= +# Copyright (C) 2026 Sayali Bhavsar sbhavsar@redhat.com +#================================================= +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# Automates execution of CockroachDB benchmarks. Downloads the CockroachDB +# binary, starts a single-node insecure cluster, runs the specified workloads, +# collects results, and tears everything down. +# + +#================================================= +cockroachdb_wrapper_version="v1.00" +test_name="cockroachdb" +cockroach_version="v22.2.0" +#================================================= +results_file="results_${test_name}.csv" +arguments="$@" +script_dir=$(realpath $(dirname $0)) +pcpdir="" + +duration="90s" +ramp="10s" +concurrency="" +opts_file="" + +exit_out() +{ + echo $1 + exit $2 +} + +if [ ! -f "/tmp/${test_name}.out" ]; then + command="${0} $@" + echo $command + $command &> /tmp/${test_name}.out + rtc=$? + cat /tmp/${test_name}.out + rm /tmp/${test_name}.out + exit $rtc +fi + +curdir=$(dirname $(realpath $0)) +if [[ $0 == "./"* ]]; then + chars=`echo $0 | awk -v RS='/' 'END{print NR-1}'` + if [[ $chars == 1 ]]; then + run_dir=`pwd` + else + run_dir=`echo $0 | cut -d'/' -f 1-${chars} | cut -d'.' -f2-` + run_dir="${curdir}${run_dir}" + fi +elif [[ $0 != "/"* ]]; then + dir=`echo $0 | rev | cut -d'/' -f2- | rev` + run_dir="${curdir}/${dir}" +else + chars=`echo $0 | awk -v RS='/' 'END{print NR-1}'` + run_dir=`echo $0 | cut -d'/' -f 1-${chars}` + if [[ $run_dir != "/"* ]]; then + run_dir=${curdir}/${run_dir} + fi +fi +cd $run_dir + +show_usage=0 + +TOOLS_BIN="$HOME/test_tools" +export TOOLS_BIN + +usage() +{ + echo "Usage $1:" + echo " --cockroach_version : CockroachDB version to download (e.g., \"v24.3.33\")." + echo " Defaults to v22.2.0. Versions >= v23.2 require x86-64-v2 CPU support." + echo " --concurrency : Comma-separated list of concurrency levels to test" + echo " (e.g., \"128,256,512,1024\"). Defaults to 128,256,512,1024." + echo " --duration : How long each workload runs (e.g., \"90s\", \"120s\")." + echo " Defaults to 90s." + echo " --opts_file : Workload configuration file to use." + echo " Defaults to test_opts in the script directory." + echo " --ramp : Ramp-up period before measurement begins (e.g., \"10s\")." + echo " Defaults to 10s." + source $TOOLS_BIN/general_setup --usage + exit $E_USAGE +} + +attempt_tools_generic() +{ + method="$1" + if [[ ! -d "$TOOLS_BIN" ]]; then + $method ${tools_git}/archive/refs/heads/main.zip + if [[ $? -eq 0 ]]; then + unzip -q main.zip + mv test_tools-wrappers-main ${TOOLS_BIN} + rm main.zip + fi + fi +} + +attempt_tools_git() +{ + if [[ ! -d "$TOOLS_BIN" ]]; then + git clone $tools_git "$TOOLS_BIN" + if [ $? -ne 0 ]; then + exit_out "Error: pulling git $tools_git failed." 101 + fi + fi +} + +install_test_tools() +{ + # + # Clone the repo that contains the common code and tools + # + tools_git=https://github.com/redhat-performance/test_tools-wrappers + found=0 + for arg in "$@"; do + if [ $found -eq 1 ]; then + tools_git=$arg + found=0 + fi + if [[ $arg == "--tools_git" ]]; then + found=1 + fi + + # + # Check for usage here before calling the common parsers. + # Otherwise the script exits without showing test-specific options. + # + if [[ $arg == "--usage" ]]; then + show_usage=1 + fi + done + + # + # Only clone if the test tools directory does not already exist. + # + attempt_tools_generic "wget" + attempt_tools_generic "curl -L -O " + attempt_tools_git + + if [ $show_usage -eq 1 ]; then + usage $1 + fi +} + +# +# Download and extract the CockroachDB binary if not already present. +# +install_cockroachdb() +{ + cockroach_dir="${run_dir}/cockroach-build" + if [[ -d "$cockroach_dir" ]]; then + echo "CockroachDB already installed at $cockroach_dir" + return + fi + + arch=$(uname -m) + if [[ $arch == "aarch64" ]]; then + tarball="cockroach-${cockroach_version}.linux-3.7.10-gnu-aarch64.tgz" + else + tarball="cockroach-${cockroach_version}.linux-amd64.tgz" + fi + + echo "Downloading CockroachDB ${cockroach_version}..." + url="https://binaries.cockroachdb.com/${tarball}" + if ! wget -q "$url" -O "$tarball" 2>/dev/null; then + if ! curl -sL "$url" -o "$tarball" 2>/dev/null; then + exit_out "Failed to download CockroachDB from $url" 102 + fi + fi + + tar -xf "$tarball" + dirname=$(echo "$tarball" | sed "s/.tgz//") + mv "$dirname" "$cockroach_dir" + rm -f "$tarball" + echo "CockroachDB installed to $cockroach_dir" +} + +# +# Start CockroachDB in single-node insecure mode. +# +start_cockroachdb() +{ + cockroach_bin="${cockroach_dir}/cockroach" + echo "Starting CockroachDB single-node cluster..." + $cockroach_bin start-single-node --cache .25 --insecure --background \ + --store=${run_dir}/cockroach-data --listen-addr=127.0.0.1:26257 \ + --http-addr=127.0.0.1:8080 > ${run_dir}/cockroach_start.log 2>&1 + + sleep 5 + + if ! $cockroach_bin node status --insecure --host=127.0.0.1:26257 > /dev/null 2>&1; then + exit_out "CockroachDB failed to start" 103 + fi + echo "CockroachDB is running" +} + +# +# Stop CockroachDB and remove the data directory. +# +stop_cockroachdb() +{ + cockroach_bin="${cockroach_dir}/cockroach" + echo "Stopping CockroachDB..." + $cockroach_bin quit --insecure --host=127.0.0.1:26257 > /dev/null 2>&1 + sleep 2 + + # Kill any remaining cockroach processes + pkill -f "cockroach start-single-node" 2>/dev/null + sleep 1 + + rm -rf ${run_dir}/cockroach-data + echo "CockroachDB stopped and data cleaned up" +} + +# +# Run a single workload at a given concurrency level. +# Parse ops/sec from the cockroach workload output. +# +# The cockroach workload output looks like: +# _elapsed___errors_____ops(total)___ops/sec(cum)__avg(ms)__p50(ms)__p95(ms)__p99(ms)_pMax(ms)__result +# 90.0s 0 16325 181.4 3.7 0.6 10.0 65.0 125.8 +# +# The ops/sec(cum) value comes from the final summary line (column 4). +# +run_single_workload() +{ + workload_name=$1 + workload_opts=$2 + conc=$3 + iter=$4 + + out_file="${run_dir}/raw_${workload_name}_c${conc}_iter${iter}.out" + + $cockroach_bin workload run $workload_opts \ + --concurrency $conc --duration $duration --drop --ramp $ramp \ + 'postgresql://root@127.0.0.1:26257?sslmode=disable' \ + > $out_file 2>&1 + + dur_num="${duration%s}" + ops_sec=$(grep -E "^\s*${dur_num}(\.[0-9]+)?s" "$out_file" | tail -1 | awk '{print $4}') + if [[ -z $ops_sec ]]; then + ops_sec=0 + fi + echo $ops_sec +} + +# +# Run all workloads across all concurrency levels for one iteration. +# +execute_cockroachdb() +{ + iter=${1} + out_file="${run_dir}/cockroachdb_iter_${iter}" + + run_concurrency=$(echo $concurrency | sed "s/,/ /g") + while IFS= read -r test_info + do + workload_name=$(echo $test_info | cut -d, -f1) + workload_opts=$(echo $test_info | cut -d, -f2) + + for conc in $run_concurrency; + do + if [[ $to_use_pcp -eq 1 ]]; then + start_pcp_subset + fi + + start_cockroachdb + start_time=$(retrieve_time_stamp) + + echo "Running: $workload_name at concurrency $conc (iteration $iter)" + ops_sec=$(run_single_workload "$workload_name" "$workload_opts" "$conc" "$iter") + + end_time=$(retrieve_time_stamp) + stop_cockroachdb + + echo "${workload_name},${conc},${ops_sec},0,${start_time},${end_time}" >> ${out_file}.csv + + if [[ $to_use_pcp -eq 1 ]]; then + results2pcp_add_value "iteration:${iter}" + results2pcp_add_value "concurrency:${conc}" + results2pcp_add_value "average:${ops_sec}" + results2pcp_add_value_commit + reset_pcp_om + stop_pcp_subset + fi + done + done < "$opts_file" + + # + # Write the CSV header and validate results for this iteration. + # + tmpcsv=$(mktemp) + $TOOLS_BIN/test_header_info --front_matter --results_file $tmpcsv \ + --host $to_configuration --sys_type $to_sys_type \ + --tuned $to_tuned_setting --results_version $cockroachdb_wrapper_version \ + --test_name $test_name --field_header "Workload,Concurrency,Average,Deviation" + cat ${out_file}.csv >> $tmpcsv + mv $tmpcsv ${out_file}.csv + + ${TOOLS_BIN}/csv_to_json $to_json_flags --csv_file ${out_file}.csv --output_file cockroachdb_verify.json + test_rtc=$? + if [[ $test_rtc -ne 0 ]]; then + exit_out "${TOOLS_BIN}/csv_to_json failed for ${out_file}.csv" $test_rtc + fi + ${TOOLS_BIN}/verify_results $to_verify_flags --schema_file $script_dir/../result_schema.py --class_name Cockroachdb_Results --file cockroachdb_verify.json + test_rtc=$? + if [[ $test_rtc -ne 0 ]]; then + echo Test failure detected: $out_file + fi +} + +# +# Compute a trimmed mean for each workload+concurrency combination across +# multiple iterations. For 5 or more iterations the highest and lowest +# values are dropped before averaging. +# +generate_combine_report() +{ + rm -f $results_file + + $TOOLS_BIN/test_header_info --front_matter --results_file $results_file \ + --host $to_configuration --sys_type $to_sys_type \ + --tuned $to_tuned_setting --results_version $cockroachdb_wrapper_version \ + --test_name $test_name --field_header "Workload,Concurrency,Average,Deviation" + + # Build a list of unique workload,concurrency pairs from the first iteration. + combos=$(grep -v "^#" cockroachdb_iter_1.csv | tail -n +2 | cut -d, -f1,2) + + for combo in $combos + do + workload_name=$(echo $combo | cut -d, -f1) + conc=$(echo $combo | cut -d, -f2) + high=0 + low="" + sum=0 + values="" + val_sep="" + start_time=$(grep "^${workload_name},${conc}," cockroachdb_iter_1.csv | cut -d, -f5) + end_time=$(grep "^${workload_name},${conc}," cockroachdb_iter_${to_times_to_run}.csv | cut -d, -f6) + for iter in $(seq 1 1 $to_times_to_run); do + file="cockroachdb_iter_${iter}.csv" + rval=$(grep "^${workload_name},${conc}," $file | cut -d, -f3 | cut -d. -f1) + if [[ -z $rval ]]; then + echo Error in parsing cockroachdb_iter_${iter}.csv + continue + fi + let "sum=${sum}+${rval}" + values="${values}${val_sep}${rval}" + val_sep=" " + if [[ $rval -gt $high ]]; then + high=$rval + fi + if [[ $low == "" ]] || [[ $rval -lt $low ]]; then + low=$rval + fi + done + if [[ $to_times_to_run -gt 4 ]]; then + let "samples=${to_times_to_run}-2" + let "sum=${sum}-${high}-${low}" + else + samples=${to_times_to_run} + fi + avg=$(echo "scale=1; $sum/$samples" | bc) + if [[ $to_times_to_run -gt 1 ]]; then + sumsq=0 + for v in $values; do + sumsq=$(echo "$sumsq + ($v - $avg) * ($v - $avg)" | bc) + done + stddev=$(echo "scale=2; sqrt($sumsq / $to_times_to_run)" | bc -l) + if [[ $(echo "$avg > 0" | bc) -eq 1 ]]; then + deviation=$(echo "scale=2; $stddev * 100 / $avg" | bc) + else + deviation="0" + fi + else + deviation="0" + fi + echo ${workload_name},${conc},${avg},${deviation},${start_time},${end_time} >> $results_file + done +} + +install_test_tools "$@" + +# +# Variables set by general setup. +# +# TOOLS_BIN: points to the tool directory +# to_home_root: home directory +# to_configuration: configuration information +# to_times_to_run: number of times to run the test +# to_run_label: Label for the run +# to_user: User on the test system running the test +# to_sys_type: for results info, basically aws, azure or local +# to_sysname: name of the system +# to_tuned_setting: tuned setting +# + +pushd $curdir 2> /dev/null +source "$TOOLS_BIN/general_setup" "$@" +popd 2> /dev/null +# Gather hardware information +$TOOLS_BIN/gather_data ${curdir} + +ARGUMENT_LIST=( + "cockroach_version" + "concurrency" + "duration" + "opts_file" + "ramp" +) + +NO_ARGUMENTS=( + "usage" +) + +# read arguments +opts=$(getopt \ + --longoptions "$(printf "%s:," "${ARGUMENT_LIST[@]}")" \ + --longoptions "$(printf "%s," "${NO_ARGUMENTS[@]}")" \ + --name "$(basename "$0")" \ + --options "h" \ + -- "$@" +) + +eval set --$opts + +while [[ $# -gt 0 ]]; do + case "$1" in + --cockroach_version) + cockroach_version=$2 + shift 2 + ;; + --concurrency) + concurrency=$2 + shift 2 + ;; + --duration) + duration=$2 + shift 2 + ;; + --opts_file) + opts_file=$2 + shift 2 + ;; + --ramp) + ramp=$2 + shift 2 + ;; + --usage) + usage $0 + ;; + -h) + usage $0 + ;; + --) + break + ;; + *) + echo option not found $1 + usage $0 + ;; + esac +done + +if [[ $concurrency == "" ]]; then + concurrency="128,256,512,1024" +fi + +if [[ $opts_file == "" ]]; then + opts_file="${run_dir}/test_opts" +fi + +package_tool --no_packages $to_no_pkg_install --wrapper_config $curdir/../cockroachdb.json + +install_cockroachdb + +# Set up PCP if enabled +if [[ $to_use_pcp -eq 1 ]]; then + source $TOOLS_BIN/pcp/pcp_commands.inc + setup_pcp + pcp_cfg=$TOOLS_BIN/pcp/default.cfg + pcpdir=/tmp/pcp_`date "+%Y.%m.%d-%H.%M.%S"` + echo "Start PCP" + echo start_pcp ${pcpdir}/ ${test_name} $pcp_cfg + start_pcp ${pcpdir}/ ${test_name} $pcp_cfg +fi + +rm -rf ${run_dir}/*csv ${run_dir}/raw_*.out +for iter in $(seq 1 1 $to_times_to_run); do + execute_cockroachdb $iter +done +generate_combine_report + +# Shutdown PCP and clean up +if [[ $to_use_pcp -eq 1 ]]; then + shutdown_pcp +fi + +${TOOLS_BIN}/save_results --curdir $curdir --home_root $to_home_root \ + --other_files "*_summary,run*log,test_results_report,${pcpdir},*csv,raw_*.out" \ + --results $results_file --test_name cockroachdb \ + --tuned_setting=$to_tuned_setting --version $cockroachdb_wrapper_version --user $to_user + +exit $E_SUCCESS diff --git a/cockroachdb/openmetrics_cockroachdb_reset.txt b/cockroachdb/openmetrics_cockroachdb_reset.txt new file mode 100644 index 0000000..6c80a13 --- /dev/null +++ b/cockroachdb/openmetrics_cockroachdb_reset.txt @@ -0,0 +1,8 @@ +iteration 0 +running 0 +numthreads 0 +runtime NaN +throughput NaN +latency NaN +concurrency NaN +average NaN diff --git a/cockroachdb/test_opts b/cockroachdb/test_opts new file mode 100644 index 0000000..819c5d8 --- /dev/null +++ b/cockroachdb/test_opts @@ -0,0 +1,5 @@ +kv_95pct_reads,kv --read-percent 95 +kv_50pct_reads,kv --read-percent 50 +kv_60pct_reads,kv --read-percent 60 +kv_10pct_reads,kv --read-percent 10 +movr,movr diff --git a/license b/license new file mode 100644 index 0000000..26394f2 --- /dev/null +++ b/license @@ -0,0 +1,17 @@ +# +# Copyright (C) 2026 Sayali Bhavsar sbhavsar@redhat.com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# diff --git a/result_schema.py b/result_schema.py new file mode 100644 index 0000000..1af8182 --- /dev/null +++ b/result_schema.py @@ -0,0 +1,19 @@ +import pydantic +import datetime + +from enum import Enum + +class Workload(Enum): + kv_95pct_reads = "kv_95pct_reads" + kv_50pct_reads = "kv_50pct_reads" + kv_60pct_reads = "kv_60pct_reads" + kv_10pct_reads = "kv_10pct_reads" + movr = "movr" + +class Cockroachdb_Results(pydantic.BaseModel): + Workload: Workload + Concurrency: int = pydantic.Field(gt=0) + Average: float = pydantic.Field(allow_inf_nan=False) + Deviation: float = pydantic.Field(allow_inf_nan=False) + Start_Date: datetime.datetime + End_Date: datetime.datetime From dc11c48d65b07f2601622f3d0a013f3f4b922b35 Mon Sep 17 00:00:00 2001 From: Sayali Bhavsar Date: Fri, 7 Aug 2026 14:45:28 +0530 Subject: [PATCH 2/5] code review changes --- cockroachdb/cockroachdb_run | 5 ++--- license | 17 ----------------- result_schema.py | 4 ++-- 3 files changed, 4 insertions(+), 22 deletions(-) delete mode 100644 license diff --git a/cockroachdb/cockroachdb_run b/cockroachdb/cockroachdb_run index 6fcafee..5031626 100755 --- a/cockroachdb/cockroachdb_run +++ b/cockroachdb/cockroachdb_run @@ -164,7 +164,7 @@ install_test_tools() # install_cockroachdb() { - cockroach_dir="${run_dir}/cockroach-build" + cockroach_dir="${run_dir}/cockroach-build-${cockroach_version}" if [[ -d "$cockroach_dir" ]]; then echo "CockroachDB already installed at $cockroach_dir" return @@ -358,8 +358,7 @@ generate_combine_report() file="cockroachdb_iter_${iter}.csv" rval=$(grep "^${workload_name},${conc}," $file | cut -d, -f3 | cut -d. -f1) if [[ -z $rval ]]; then - echo Error in parsing cockroachdb_iter_${iter}.csv - continue + exit_out "Error: failed to parse ${workload_name},${conc} from cockroachdb_iter_${iter}.csv" 104 fi let "sum=${sum}+${rval}" values="${values}${val_sep}${rval}" diff --git a/license b/license deleted file mode 100644 index 26394f2..0000000 --- a/license +++ /dev/null @@ -1,17 +0,0 @@ -# -# Copyright (C) 2026 Sayali Bhavsar sbhavsar@redhat.com -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# diff --git a/result_schema.py b/result_schema.py index 1af8182..41eb773 100644 --- a/result_schema.py +++ b/result_schema.py @@ -13,7 +13,7 @@ class Workload(Enum): class Cockroachdb_Results(pydantic.BaseModel): Workload: Workload Concurrency: int = pydantic.Field(gt=0) - Average: float = pydantic.Field(allow_inf_nan=False) - Deviation: float = pydantic.Field(allow_inf_nan=False) + Average: float = pydantic.Field(allow_inf_nan=False, ge=0) + Deviation: float = pydantic.Field(allow_inf_nan=False, ge=0) Start_Date: datetime.datetime End_Date: datetime.datetime From 52e12148a11fe70f3f111fdfc1e64e3614eaf5a1 Mon Sep 17 00:00:00 2001 From: Sayali Bhavsar Date: Mon, 10 Aug 2026 16:04:14 +0530 Subject: [PATCH 3/5] Fix CSV leading-zero formatting, PCP metric capture order, and use simple mean --- cockroachdb/cockroachdb_run | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/cockroachdb/cockroachdb_run b/cockroachdb/cockroachdb_run index 5031626..0a7ca51 100755 --- a/cockroachdb/cockroachdb_run +++ b/cockroachdb/cockroachdb_run @@ -287,6 +287,10 @@ execute_cockroachdb() echo "Running: $workload_name at concurrency $conc (iteration $iter)" ops_sec=$(run_single_workload "$workload_name" "$workload_opts" "$conc" "$iter") + if [[ $to_use_pcp -eq 0 ]]; then + rm -f "${run_dir}/raw_${workload_name}_c${conc}_iter${iter}.out" + fi + end_time=$(retrieve_time_stamp) stop_cockroachdb @@ -297,8 +301,8 @@ execute_cockroachdb() results2pcp_add_value "concurrency:${conc}" results2pcp_add_value "average:${ops_sec}" results2pcp_add_value_commit - reset_pcp_om stop_pcp_subset + reset_pcp_om fi done done < "$opts_file" @@ -326,11 +330,6 @@ execute_cockroachdb() fi } -# -# Compute a trimmed mean for each workload+concurrency combination across -# multiple iterations. For 5 or more iterations the highest and lowest -# values are dropped before averaging. -# generate_combine_report() { rm -f $results_file @@ -340,15 +339,12 @@ generate_combine_report() --tuned $to_tuned_setting --results_version $cockroachdb_wrapper_version \ --test_name $test_name --field_header "Workload,Concurrency,Average,Deviation" - # Build a list of unique workload,concurrency pairs from the first iteration. combos=$(grep -v "^#" cockroachdb_iter_1.csv | tail -n +2 | cut -d, -f1,2) for combo in $combos do workload_name=$(echo $combo | cut -d, -f1) conc=$(echo $combo | cut -d, -f2) - high=0 - low="" sum=0 values="" val_sep="" @@ -363,28 +359,16 @@ generate_combine_report() let "sum=${sum}+${rval}" values="${values}${val_sep}${rval}" val_sep=" " - if [[ $rval -gt $high ]]; then - high=$rval - fi - if [[ $low == "" ]] || [[ $rval -lt $low ]]; then - low=$rval - fi done - if [[ $to_times_to_run -gt 4 ]]; then - let "samples=${to_times_to_run}-2" - let "sum=${sum}-${high}-${low}" - else - samples=${to_times_to_run} - fi - avg=$(echo "scale=1; $sum/$samples" | bc) + avg=$(echo "scale=1; $sum/$to_times_to_run" | bc | sed 's/^\./0./') if [[ $to_times_to_run -gt 1 ]]; then sumsq=0 for v in $values; do sumsq=$(echo "$sumsq + ($v - $avg) * ($v - $avg)" | bc) done - stddev=$(echo "scale=2; sqrt($sumsq / $to_times_to_run)" | bc -l) + stddev=$(echo "scale=2; sqrt($sumsq / $to_times_to_run)" | bc -l | sed 's/^\./0./') if [[ $(echo "$avg > 0" | bc) -eq 1 ]]; then - deviation=$(echo "scale=2; $stddev * 100 / $avg" | bc) + deviation=$(echo "scale=2; $stddev * 100 / $avg" | bc | sed 's/^\./0./') else deviation="0" fi From a2d4cb033d45791ff6bc79743acafea550981434 Mon Sep 17 00:00:00 2001 From: Sayali Bhavsar Date: Mon, 10 Aug 2026 19:24:07 +0530 Subject: [PATCH 4/5] code review changes2 --- cockroachdb/cockroachdb_run | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cockroachdb/cockroachdb_run b/cockroachdb/cockroachdb_run index 0a7ca51..6cae559 100755 --- a/cockroachdb/cockroachdb_run +++ b/cockroachdb/cockroachdb_run @@ -361,6 +361,7 @@ generate_combine_report() val_sep=" " done avg=$(echo "scale=1; $sum/$to_times_to_run" | bc | sed 's/^\./0./') + deviation=0 if [[ $to_times_to_run -gt 1 ]]; then sumsq=0 for v in $values; do @@ -369,11 +370,7 @@ generate_combine_report() stddev=$(echo "scale=2; sqrt($sumsq / $to_times_to_run)" | bc -l | sed 's/^\./0./') if [[ $(echo "$avg > 0" | bc) -eq 1 ]]; then deviation=$(echo "scale=2; $stddev * 100 / $avg" | bc | sed 's/^\./0./') - else - deviation="0" fi - else - deviation="0" fi echo ${workload_name},${conc},${avg},${deviation},${start_time},${end_time} >> $results_file done From 093487c4ae13b88384f7d0abc59c63034b894bbd Mon Sep 17 00:00:00 2001 From: Sayali Bhavsar Date: Mon, 24 Aug 2026 17:07:48 +0530 Subject: [PATCH 5/5] Address dvalinrh code review feedback --- README.md | 11 ++++---- cockroachdb/cockroachdb_run | 54 ++++++++++++++++++++++++++++++------- 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 8e079fd..65e230e 100644 --- a/README.md +++ b/README.md @@ -244,11 +244,12 @@ The script computes combined results across multiple iterations: ## Return Codes -The script uses standardized error codes from test_tools error_codes: -- **0**: Success -- **101**: Git clone failure (test_tools) -- **102**: CockroachDB binary download failure -- **103**: CockroachDB failed to start +The script uses standardized error codes from `test_tools/error_codes`: +- **0** (`E_SUCCESS`): Success +- **101** (`E_GENERAL`): General failure — test_tools git clone failure, CockroachDB binary download failure, or CockroachDB failed to start +- **106** (`E_INVAL_DATA`): Failed to parse a workload's result value while combining iteration results + +Other exit codes surfaced by the script (usage errors, argument parsing, PCP failures, validation failures) come from `general_setup` and the shared `test_tools` utilities it invokes (`csv_to_json`, `verify_results`); see `test_tools/error_codes` for the full list. Exit codes indicate specific failure points for automated testing workflows. diff --git a/cockroachdb/cockroachdb_run b/cockroachdb/cockroachdb_run index 6cae559..b67ca20 100755 --- a/cockroachdb/cockroachdb_run +++ b/cockroachdb/cockroachdb_run @@ -82,6 +82,18 @@ show_usage=0 TOOLS_BIN="$HOME/test_tools" export TOOLS_BIN +# +# error_codes is normally sourced by general_setup, but a few failures below +# can happen before general_setup runs. Provide matching defaults so named +# codes are always available, then pick up the authoritative values once +# error_codes exists. +# +E_GENERAL=101 +E_INVAL_DATA=106 +if [[ -f "$TOOLS_BIN/error_codes" ]]; then + source "$TOOLS_BIN/error_codes" +fi + usage() { echo "Usage $1:" @@ -117,7 +129,7 @@ attempt_tools_git() if [[ ! -d "$TOOLS_BIN" ]]; then git clone $tools_git "$TOOLS_BIN" if [ $? -ne 0 ]; then - exit_out "Error: pulling git $tools_git failed." 101 + exit_out "Error: pulling git $tools_git failed." $E_GENERAL fi fi } @@ -179,9 +191,11 @@ install_cockroachdb() echo "Downloading CockroachDB ${cockroach_version}..." url="https://binaries.cockroachdb.com/${tarball}" - if ! wget -q "$url" -O "$tarball" 2>/dev/null; then - if ! curl -sL "$url" -o "$tarball" 2>/dev/null; then - exit_out "Failed to download CockroachDB from $url" 102 + wget -q "$url" -O "$tarball" 2>/dev/null + if [[ $? -ne 0 ]]; then + curl -sL "$url" -o "$tarball" 2>/dev/null + if [[ $? -ne 0 ]]; then + exit_out "Failed to download CockroachDB from $url" $E_GENERAL fi fi @@ -203,10 +217,18 @@ start_cockroachdb() --store=${run_dir}/cockroach-data --listen-addr=127.0.0.1:26257 \ --http-addr=127.0.0.1:8080 > ${run_dir}/cockroach_start.log 2>&1 - sleep 5 + ready=0 + for i in $(seq 1 30); do + $cockroach_bin node status --insecure --host=127.0.0.1:26257 > /dev/null 2>&1 + if [[ $? -eq 0 ]]; then + ready=1 + break + fi + sleep 1 + done - if ! $cockroach_bin node status --insecure --host=127.0.0.1:26257 > /dev/null 2>&1; then - exit_out "CockroachDB failed to start" 103 + if [[ $ready -eq 0 ]]; then + exit_out "CockroachDB failed to start" $E_GENERAL fi echo "CockroachDB is running" } @@ -219,7 +241,14 @@ stop_cockroachdb() cockroach_bin="${cockroach_dir}/cockroach" echo "Stopping CockroachDB..." $cockroach_bin quit --insecure --host=127.0.0.1:26257 > /dev/null 2>&1 - sleep 2 + + for i in $(seq 1 10); do + pgrep -f "cockroach start-single-node" > /dev/null 2>&1 + if [[ $? -ne 0 ]]; then + break + fi + sleep 1 + done # Kill any remaining cockroach processes pkill -f "cockroach start-single-node" 2>/dev/null @@ -252,10 +281,14 @@ run_single_workload() --concurrency $conc --duration $duration --drop --ramp $ramp \ 'postgresql://root@127.0.0.1:26257?sslmode=disable' \ > $out_file 2>&1 + if [[ $? -ne 0 ]]; then + echo "Warning: workload run for $workload_name at concurrency $conc exited non-zero, see $out_file" >&2 + fi dur_num="${duration%s}" ops_sec=$(grep -E "^\s*${dur_num}(\.[0-9]+)?s" "$out_file" | tail -1 | awk '{print $4}') if [[ -z $ops_sec ]]; then + echo "Warning: failed to parse ops/sec for $workload_name at concurrency $conc from $out_file, recording 0" >&2 ops_sec=0 fi echo $ops_sec @@ -277,11 +310,12 @@ execute_cockroachdb() for conc in $run_concurrency; do + start_cockroachdb + if [[ $to_use_pcp -eq 1 ]]; then start_pcp_subset fi - start_cockroachdb start_time=$(retrieve_time_stamp) echo "Running: $workload_name at concurrency $conc (iteration $iter)" @@ -354,7 +388,7 @@ generate_combine_report() file="cockroachdb_iter_${iter}.csv" rval=$(grep "^${workload_name},${conc}," $file | cut -d, -f3 | cut -d. -f1) if [[ -z $rval ]]; then - exit_out "Error: failed to parse ${workload_name},${conc} from cockroachdb_iter_${iter}.csv" 104 + exit_out "Error: failed to parse ${workload_name},${conc} from cockroachdb_iter_${iter}.csv" $E_INVAL_DATA fi let "sum=${sum}+${rval}" values="${values}${val_sep}${rval}"