From f26fe3c1ed97c0a4f2884513d9b298a2a0e4c1dd Mon Sep 17 00:00:00 2001 From: baydakov-georgiy Date: Thu, 16 Apr 2026 14:07:02 +0300 Subject: [PATCH 01/13] building test stand --- controller/Makefile | 10 +- test/virtual_load_network/.env | 28 ++- test/virtual_load_network/Dockerfile.dpdk | 55 +++++ test/virtual_load_network/Makefile | 30 +++ test/virtual_load_network/README.md | 86 ++++--- .../cloud-init/controller-network-config | 6 + .../configs/cloud-init/controller-user-data | 23 ++ .../configs/cloud-init/filter-network-config | 10 + .../configs/cloud-init/filter-user-data | 10 + .../configs/cloud-init/meta-data | 2 + test/virtual_load_network/configs/policy.toml | 15 ++ .../configs/riscv64-cross.txt | 20 ++ test/virtual_load_network/create_image.sh | 62 ++++++ test/virtual_load_network/docker-compose.yml | 38 +++- test/virtual_load_network/scripts/start-vm.sh | 37 ++++ test/virtual_load_network/scripts/start.sh | 209 +++++++++++++++--- test/virtual_load_network/scripts/stop.sh | 31 ++- worker/src/dpdk_filter/main.c | 4 +- 18 files changed, 593 insertions(+), 83 deletions(-) create mode 100644 test/virtual_load_network/Dockerfile.dpdk create mode 100644 test/virtual_load_network/Makefile create mode 100644 test/virtual_load_network/configs/cloud-init/controller-network-config create mode 100644 test/virtual_load_network/configs/cloud-init/controller-user-data create mode 100644 test/virtual_load_network/configs/cloud-init/filter-network-config create mode 100644 test/virtual_load_network/configs/cloud-init/filter-user-data create mode 100644 test/virtual_load_network/configs/cloud-init/meta-data create mode 100644 test/virtual_load_network/configs/policy.toml create mode 100644 test/virtual_load_network/configs/riscv64-cross.txt create mode 100755 test/virtual_load_network/create_image.sh create mode 100755 test/virtual_load_network/scripts/start-vm.sh diff --git a/controller/Makefile b/controller/Makefile index 9609a76..80d123b 100644 --- a/controller/Makefile +++ b/controller/Makefile @@ -8,7 +8,7 @@ all: generate build generate: protoc --go_out=. --go_opt=paths=source_relative \ --go-grpc_out=. --go-grpc_opt=paths=source_relative \ - $(PROTO_DIR)/file_service/file_service.proto \ + $(PROTO_DIR)/admin_service/admin_service.proto \ $(PROTO_DIR)/communication/communication.proto build: generate @@ -16,14 +16,18 @@ build: generate go build -o $(BIN_DIR)/grpc_server ./$(GRPC_SERVER) go build -o $(BIN_DIR)/manager ./$(MANAGER) +build-riscv: + bazel build --platforms=@rules_go//go/toolchain:linux_riscv64 //cmd/grpc_server:grpc_server + mkdir -p $(BIN_DIR) + cp -f bazel-bin/cmd/grpc_server/grpc_server_/grpc_server $(BIN_DIR)/grpc_server 2>/dev/null || cp -f bazel-bin/cmd/grpc_server/grpc_server $(BIN_DIR)/grpc_server + clean: rm -rf $(BIN_DIR) rm -f $(PROTO_DIR)/communication/*.pb.go - rm -f $(PROTO_DIR)/file_service/*.pb.go + rm -f $(PROTO_DIR)/admin_service/*.pb.go run-server: build ./$(BIN_DIR)/grpc_server run-manager: build ./$(BIN_DIR)/manager - diff --git a/test/virtual_load_network/.env b/test/virtual_load_network/.env index a4d98db..699bf5b 100644 --- a/test/virtual_load_network/.env +++ b/test/virtual_load_network/.env @@ -1,7 +1,25 @@ -NUM_HOSTS=3 -SUBNET=10.0.0 -GATEWAY=10.0.0.254 +NUM_HOSTS_NET1=2 +NUM_HOSTS_NET2=2 + +SUBNET1=10.0.0 +SUBNET2=10.0.1 +GATEWAY1=10.0.0.254 +GATEWAY2=10.0.1.254 +BRIDGE1=br-testnet1 +BRIDGE2=br-testnet2 DNS=8.8.8.8 -FILTER_PATH=../../worker/main-x86-virt -BRIDGE_IFACE=br-testnet + +INET_SUBNET=10.0.3 +INET_BRIDGE=br-inet + +MGMT_SUBNET=10.0.2 +MGMT_BRIDGE=br-mgmt + +QEMU_IMAGE=ubuntu-24.04.4-preinstalled-server-riscv64.img +QEMU_MEMORY=4G +QEMU_CPUS=2 + +FILTER_RISCV_BIN=../../worker/main-riscv +CONTROLLER_BIN=../../controller/bin/grpc_server + HUGEPAGES=1024 diff --git a/test/virtual_load_network/Dockerfile.dpdk b/test/virtual_load_network/Dockerfile.dpdk new file mode 100644 index 0000000..373ed60 --- /dev/null +++ b/test/virtual_load_network/Dockerfile.dpdk @@ -0,0 +1,55 @@ +FROM --platform=linux/riscv64 ubuntu:24.04 + +RUN apt update && apt install -y \ + build-essential meson ninja-build python3-pyelftools \ + libelf-dev libnuma-dev pkg-config git wget \ + clang llvm m4 libpcap-dev libatomic1 + +WORKDIR /tmp + +RUN git clone --depth 1 --branch v1.4.6 https://github.com/libbpf/libbpf.git && \ + cd libbpf/src && make && \ + make install PREFIX=/usr LIBDIR=/usr/lib/riscv64-linux-gnu && \ + ldconfig && rm -rf /tmp/libbpf + +RUN git clone --depth 1 --branch v1.4.2 https://github.com/xdp-project/xdp-tools.git && \ + cd xdp-tools && ./configure && make && \ + make -C lib/libxdp install PREFIX=/usr LIBDIR=/usr/lib/riscv64-linux-gnu && \ + ln -sf /usr/include/xdp/xsk.h /usr/include/bpf/xsk.h && \ + ldconfig && rm -rf /tmp/xdp-tools + +RUN pkg-config --exists libbpf && echo "libbpf OK" && \ + pkg-config --exists libxdp && echo "libxdp OK" + +RUN wget https://fast.dpdk.org/rel/dpdk-23.11.6.tar.xz && \ + tar -xf dpdk-23.11.6.tar.xz && \ + cd dpdk-stable-23.11.6 && \ + meson setup -Denable_drivers=net/af_xdp,net/tap -Dmachine=generic build && \ + ninja -C build && ninja -C build install && ldconfig && \ + rm -rf /tmp/dpdk-* + +RUN mkdir -p /output/include /output/lib/pkgconfig /output/lib/bpf /output/runtime && \ + cp /usr/lib/riscv64-linux-gnu/bpf/*.o /output/lib/bpf/ 2>/dev/null; \ + cp -r /usr/local/include/* /output/include/ 2>/dev/null; \ + cp -r /usr/include/bpf /output/include/ 2>/dev/null; \ + cp -r /usr/include/xdp /output/include/ 2>/dev/null; \ + for d in /usr/local/lib/riscv64-linux-gnu /usr/local/lib; do \ + [ -f "$d/librte_eal.so" ] && { \ + cp "$d"/librte_*.so* /output/lib/; \ + cp "$d"/librte_*.a /output/lib/ 2>/dev/null; \ + cp "$d"/pkgconfig/*.pc /output/lib/pkgconfig/ 2>/dev/null; \ + cp "$d"/librte_*.so* /output/runtime/; \ + break; \ + }; \ + done; \ + for lib in libbpf libxdp libelf libnuma; do \ + for d in /usr/lib/riscv64-linux-gnu /usr/lib /usr/local/lib; do \ + ls "$d"/${lib}.so* &>/dev/null 2>&1 && { \ + cp "$d"/${lib}.so* /output/lib/; \ + cp "$d"/${lib}.a /output/lib/ 2>/dev/null; \ + cp "$d"/${lib}.so* /output/runtime/; \ + break; \ + }; \ + done; \ + done; \ + true diff --git a/test/virtual_load_network/Makefile b/test/virtual_load_network/Makefile new file mode 100644 index 0000000..8f39899 --- /dev/null +++ b/test/virtual_load_network/Makefile @@ -0,0 +1,30 @@ +SCRIPTS_DIR = scripts +WORKER_DIR = ../../worker +CONTROLLER_DIR = ../../controller +DPDK_RISCV_INSTALL = $(WORKER_DIR)/dpdk-riscv-install + +.PHONY: run stop logs build build-filter build-controller clean-vms + +run: build + sudo $(SCRIPTS_DIR)/start.sh + +stop: + sudo $(SCRIPTS_DIR)/stop.sh + +logs: + @sudo bash -c 'while true; do clear; cat shared/filter1.log 2>/dev/null | tail -10; echo "---"; cat shared/filter2.log 2>/dev/null | tail -10; echo "---"; cat shared/controller.log 2>/dev/null | tail -5; sleep 2; done' + +build: build-filter build-controller + +build-filter: + @if [ ! -d "$(DPDK_RISCV_INSTALL)" ]; then \ + echo "DPDK sysroot not found. Run ./create_image.sh first."; \ + exit 1; \ + fi + $(MAKE) -C $(WORKER_DIR) -f Makefile.main_riscv + +build-controller: + $(MAKE) -C $(CONTROLLER_DIR) build-riscv + +clean-vms: + rm -f filter1.qcow2 filter2.qcow2 controller.qcow2 cloud-init-*.iso diff --git a/test/virtual_load_network/README.md b/test/virtual_load_network/README.md index 35a4e7c..9238dee 100644 --- a/test/virtual_load_network/README.md +++ b/test/virtual_load_network/README.md @@ -1,50 +1,74 @@ -# Тестовый стенд виртуальной нагрузки +# Виртуальный тестовый стенд -## Установка +## Что необходимо сделать +- [x] Настроить топологию сети и виртуальные машины +- [x] Добавить кросс-компиляцию controller под RISC-V +- [x] Добaвить кросс-компиляцию dpdk библиотеки + - [x] На данный момент используется Docker контейнер внутри которого билдиться библиотека. Это медленно (минут 50, но делается один раз) + - [ ] (\*) По хорошему необходимо создать отдельный sysroot с помощью Yocto Project внутри которого будем билдить +- [x] Запуск фильтров и контроллера + - [x] Запуск фильтров происходит через ssh подключение и прописывание вручную комманды + - [ ] (\*) По хорошему необходимо сделать запуск через systemd +- [ ] Взаимодействие controller и worker, пока не сделано: + - [ ] Кросс компиляция worker вместе с dpdk-filter. На данный момент билдиться только сам dpdk-filter -### DPDK -```sh -sudo apt-get install -y meson ninja-build python3-pyelftools libbpf-dev -git clone https://github.com/DPDK/dpdk.git -cd dpdk -meson setup -Denable_drivers=net/af_xdp,net/tap build -ninja -C build -sudo ninja -C build install -``` +## Подготовка -### Компиляция worker +### Создание RISC-V образа ```sh -cd worker -make -f Makefile.main_x86 virt +sudo scripts/create_image.sh ``` +Пока скрипт из test-image переписан только под arch linux. Надо переписать его под ubuntu и запускать внутри docker контейнера + +Подробности: `wiki/using_test_image.md` + ## Конфигурация -Параметры задаются в `.env`: +Параметры в `.env`: -| Переменная | По умолчанию | Описание | -|---------------|---------------------------|----------------------------------| -| NUM_HOSTS | 3 | Количество генераторов трафика | -| SUBNET | 10.0.0 | Подсеть (первые 3 октета) | -| GATEWAY | 10.0.0.254 | Адрес шлюза | -| DNS | 8.8.8.8 | DNS для контейнеров | -| FILTER_PATH | ../../worker/main-x86-virt| Путь до бинаря фильтра | -| HUGEPAGES | 1024 | Количество hugepages | +| Переменная | По умолчанию | Описание | +|------------------|----------------------|-----------------------------------| +| NUM_HOSTS_NET1 | 2 | Генераторов в сети 1 | +| NUM_HOSTS_NET2 | 2 | Генераторов в сети 2 | +| SUBNET1 | 10.0.0 | Подсеть 1 | +| SUBNET2 | 10.0.1 | Подсеть 2 | +| MGMT_SUBNET | 10.0.2 | Управляющая подсеть | +| QEMU_IMAGE | ubuntu-...riscv64.img| Путь к RISC-V образу | +| QEMU_MEMORY | 4G | RAM на каждую VM | +| QEMU_CPUS | 2 | CPU на каждую VM | +| FILTER_RISCV_BIN | ../../worker/main-riscv-virt | Путь к бинарю фильтра | +| CONTROLLER_BIN | ../../controller/bin/grpc_server | Путь к контроллеру | +| HUGEPAGES | 1024 | Количество hugepages | ## Запуск + +Виртуальные машины и сеть: ```sh cd test/virtual_load_network -sudo scripts/start.sh +make run +``` + +Запуск фильтров осуществляется через подлкючение по ssh: ``` +# for filter1 +ssh-keygen -R 10.0.2.1 +ssh ubuntu@10.0.2.1 +# password ubuntu +# in VM: +sudo rm -rf /var/run/dpdk /dev/hugepages/rtemap_* +sudo LIBXDP_OBJECT_PATH=/usr/lib/riscv64-linux-gnu/bpf LD_LIBRARY_PATH=/mnt/lib /mnt/filter --no-pci --iova-mode=va -d /mnt/lib/librte_net_af_xdp.so -d /mnt/lib/librte_net_tap.so -- -Скрипт выполняет: -1. Настройку hugepages -2. Запуск контейнеров-генераторов через `docker compose` -3. Создание veth пары и tc mirroring на bridge compose-сети -4. Настройку NAT -5. Запуск DPDK фильтра +# Also for filter2 +ssh-keygen -R 10.0.2.2 +ssh ubuntu@10.0.2.2 +# password ubuntu +# in VM: +sudo rm -rf /var/run/dpdk /dev/hugepages/rtemap_* +sudo LIBXDP_OBJECT_PATH=/usr/lib/riscv64-linux-gnu/bpf LD_LIBRARY_PATH=/mnt/lib /mnt/filter --no-pci --iova-mode=va -d /mnt/lib/librte_net_af_xdp.so -d /mnt/lib/librte_net_tap.so -- +``` ## Остановка ```sh -sudo scripts/stop.sh +make stop ``` diff --git a/test/virtual_load_network/configs/cloud-init/controller-network-config b/test/virtual_load_network/configs/cloud-init/controller-network-config new file mode 100644 index 0000000..16b55fc --- /dev/null +++ b/test/virtual_load_network/configs/cloud-init/controller-network-config @@ -0,0 +1,6 @@ +version: 2 +ethernets: + eth0: + dhcp4: false + addresses: + - MGMT_IP/24 diff --git a/test/virtual_load_network/configs/cloud-init/controller-user-data b/test/virtual_load_network/configs/cloud-init/controller-user-data new file mode 100644 index 0000000..613233b --- /dev/null +++ b/test/virtual_load_network/configs/cloud-init/controller-user-data @@ -0,0 +1,23 @@ +#cloud-config +mounts: + - ["host_share", "/mnt", "9p", "trans=virtio,version=9p2000.L", "0", "0"] +runcmd: + - | + cat > /etc/systemd/system/controller.service < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages + - dpkg -i /mnt/libxdp1_1.4.2-1ubuntu4_riscv64.deb diff --git a/test/virtual_load_network/configs/cloud-init/meta-data b/test/virtual_load_network/configs/cloud-init/meta-data new file mode 100644 index 0000000..f8ea600 --- /dev/null +++ b/test/virtual_load_network/configs/cloud-init/meta-data @@ -0,0 +1,2 @@ +instance-id: VM_NAME +local-hostname: VM_NAME diff --git a/test/virtual_load_network/configs/policy.toml b/test/virtual_load_network/configs/policy.toml new file mode 100644 index 0000000..9164fda --- /dev/null +++ b/test/virtual_load_network/configs/policy.toml @@ -0,0 +1,15 @@ +[global.rules] +block_categories = ["adult", "malware"] +block_domains = ["example.com", "blocked-site.com"] +allow_domains = ["google.com", "github.com"] +min_trust_level = 50 + +[global.rules.block_by_trust] +dangerous = 10 +suspicious = 30 + +[filters.filter_1] +block_domains = ["extra-blocked.com"] + +[filters.filter_2] +allow_domains = ["extra-allowed.com"] diff --git a/test/virtual_load_network/configs/riscv64-cross.txt b/test/virtual_load_network/configs/riscv64-cross.txt new file mode 100644 index 0000000..6bbfa70 --- /dev/null +++ b/test/virtual_load_network/configs/riscv64-cross.txt @@ -0,0 +1,20 @@ +[binaries] +c = 'riscv64-linux-gnu-gcc' +cpp = 'riscv64-linux-gnu-g++' +ar = 'riscv64-linux-gnu-ar' +strip = 'riscv64-linux-gnu-strip' +pkgconfig = 'pkg-config' + +[built-in options] +c_args = ['-march=rv64gc'] +cpp_args = ['-march=rv64gc'] + +[properties] +vendor_id = 'generic' +arch_id = 'generic' + +[host_machine] +system = 'linux' +cpu_family = 'riscv64' +cpu = 'rv64gc' +endian = 'little' diff --git a/test/virtual_load_network/create_image.sh b/test/virtual_load_network/create_image.sh new file mode 100755 index 0000000..725c6f9 --- /dev/null +++ b/test/virtual_load_network/create_image.sh @@ -0,0 +1,62 @@ +#!/bin/bash +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +cd "$SCRIPT_DIR" + +IMAGE="ubuntu-24.04.4-preinstalled-server-riscv64.img" +IMAGE_XZ="${IMAGE}.xz" +IMAGE_URL="https://cdimage.ubuntu.com/releases/24.04/release/${IMAGE_XZ}" +DPDK_DEST="$SCRIPT_DIR/../../worker/dpdk-riscv-install" +RUNTIME_DEST="$SCRIPT_DIR/shared/lib" + +if ! docker buildx version &>/dev/null; then + echo "Docker buildx required. Install docker-buildx." + exit 1 +fi + +echo "Building DPDK for RISC-V in Docker..." +docker buildx build --platform linux/riscv64 -t dpdk-riscv-builder -f Dockerfile.dpdk --load . + +echo "Extracting sysroot for cross-compilation..." +rm -rf "$DPDK_DEST" +mkdir -p "$DPDK_DEST" +CONTAINER=$(docker create --platform linux/riscv64 dpdk-riscv-builder) +docker cp "$CONTAINER:/output/include" "$DPDK_DEST/include" +docker cp "$CONTAINER:/output/lib" "$DPDK_DEST/lib" + +echo "Extracting runtime libraries..." +rm -rf "$RUNTIME_DEST" +mkdir -p "$RUNTIME_DEST" +docker cp "$CONTAINER:/output/runtime/." "$RUNTIME_DEST/" +docker rm "$CONTAINER" + +echo "Sysroot: $DPDK_DEST" +echo "Runtime libs: $RUNTIME_DEST" + +if ! cat /proc/sys/fs/binfmt_misc/qemu-riscv64 &>/dev/null; then + echo ':qemu-riscv64:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xf3\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/bin/qemu-riscv64-static:FPC' | sudo tee /proc/sys/fs/binfmt_misc/register >/dev/null +fi + +if [ ! -f "$IMAGE" ]; then + if [ ! -f "$IMAGE_XZ" ]; then + echo "Downloading Ubuntu RISC-V image..." + wget "$IMAGE_URL" + fi + xz -dk "$IMAGE_XZ" +fi + +echo "Preparing VM image (installing libatomic)..." +sudo losetup -D 2>/dev/null || true +LOOP=$(sudo losetup -f) +sudo losetup -P "$LOOP" "$IMAGE" +sudo mkdir -p /mnt/riscv-rootfs +sudo mount "${LOOP}p1" /mnt/riscv-rootfs +sudo cp /usr/bin/qemu-riscv64-static /mnt/riscv-rootfs/usr/bin/ +sudo rm -f /mnt/riscv-rootfs/etc/resolv.conf +sudo cp /etc/resolv.conf /mnt/riscv-rootfs/etc/resolv.conf +sudo chroot /mnt/riscv-rootfs /bin/bash -c "apt update && apt install -y libatomic1 libnuma1" +sudo umount /mnt/riscv-rootfs +sudo losetup -d "$LOOP" + +echo "Done. Image ready: $IMAGE" diff --git a/test/virtual_load_network/docker-compose.yml b/test/virtual_load_network/docker-compose.yml index 5d23547..59914b4 100644 --- a/test/virtual_load_network/docker-compose.yml +++ b/test/virtual_load_network/docker-compose.yml @@ -1,25 +1,49 @@ services: - traffic-gen: + traffic-gen-1: build: context: . dockerfile: Dockerfile.traffic-gen networks: - - testnet + - testnet1 cap_add: - NET_ADMIN dns: - ${DNS:-8.8.8.8} deploy: - replicas: ${NUM_HOSTS:-3} + replicas: ${NUM_HOSTS_NET1:-2} + restart: unless-stopped + + traffic-gen-2: + build: + context: . + dockerfile: Dockerfile.traffic-gen + networks: + - testnet2 + cap_add: + - NET_ADMIN + dns: + - ${DNS:-8.8.8.8} + deploy: + replicas: ${NUM_HOSTS_NET2:-2} restart: unless-stopped networks: - testnet: + testnet1: + driver: bridge + driver_opts: + com.docker.network.bridge.name: "br-testnet1" + com.docker.network.bridge.enable_ip_masquerade: "false" + ipam: + config: + - subnet: ${SUBNET1:-10.0.0}.0/24 + gateway: ${SUBNET1:-10.0.0}.254 + + testnet2: driver: bridge driver_opts: - com.docker.network.bridge.name: "br-testnet" + com.docker.network.bridge.name: "br-testnet2" com.docker.network.bridge.enable_ip_masquerade: "false" ipam: config: - - subnet: ${SUBNET:-10.0.0}.0/24 - gateway: ${GATEWAY:-10.0.0.254} + - subnet: ${SUBNET2:-10.0.1}.0/24 + gateway: ${SUBNET2:-10.0.1}.254 diff --git a/test/virtual_load_network/scripts/start-vm.sh b/test/virtual_load_network/scripts/start-vm.sh new file mode 100755 index 0000000..de4ed08 --- /dev/null +++ b/test/virtual_load_network/scripts/start-vm.sh @@ -0,0 +1,37 @@ +#!/bin/bash +set -e + +VM_NAME="$1" +VM_IMAGE="$2" +SHARED_DIR="$3" +TAP_IFACE="$4" +MEMORY="${5:-4G}" +CPUS="${6:-2}" + +if [ -z "$VM_NAME" ] || [ -z "$VM_IMAGE" ] || [ -z "$SHARED_DIR" ] || [ -z "$TAP_IFACE" ]; then + echo "Usage: $0 [memory] [cpus]" + exit 1 +fi + +if [ ! -f "$VM_IMAGE" ]; then + echo "Image not found: $VM_IMAGE" + exit 1 +fi + +ip tuntap add dev "$TAP_IFACE" mode tap 2>/dev/null || true +ip link set "$TAP_IFACE" up + +qemu-system-riscv64 \ + -machine virt,acpi=off -m "$MEMORY" -smp cpus="$CPUS" \ + -display none -serial "file:/tmp/qemu-${VM_NAME}.log" \ + -name "$VM_NAME" \ + -pidfile "/tmp/qemu-${VM_NAME}.pid" \ + -bios default \ + -netdev tap,id=net0,ifname="$TAP_IFACE",script=no,downscript=no \ + -device virtio-net-device,netdev=net0 \ + -netdev user,id=net1,hostfwd=tcp::$(shuf -i 10000-20000 -n1)-:22 \ + -device virtio-net-device,netdev=net1 \ + -device virtio-rng-pci \ + -drive "file=$VM_IMAGE,format=raw,if=virtio" \ + -virtfs "local,path=$SHARED_DIR,mount_tag=host_share,security_model=mapped-xattr" \ + -daemonize diff --git a/test/virtual_load_network/scripts/start.sh b/test/virtual_load_network/scripts/start.sh index 81c0f70..150d757 100755 --- a/test/virtual_load_network/scripts/start.sh +++ b/test/virtual_load_network/scripts/start.sh @@ -12,48 +12,199 @@ if [ -f .env ]; then set +a fi -NUM_HOSTS="${NUM_HOSTS:-3}" -SUBNET="${SUBNET:-10.0.0}" -GATEWAY="${GATEWAY:-10.0.0.254}" +SUBNET1="${SUBNET1:-10.0.0}" +SUBNET2="${SUBNET2:-10.0.1}" +INET_SUBNET="${INET_SUBNET:-10.0.3}" +BRIDGE1="${BRIDGE1:-br-testnet1}" +BRIDGE2="${BRIDGE2:-br-testnet2}" +INET_BRIDGE="${INET_BRIDGE:-br-inet}" +MGMT_SUBNET="${MGMT_SUBNET:-10.0.2}" +MGMT_BRIDGE="${MGMT_BRIDGE:-br-mgmt}" HUGEPAGES="${HUGEPAGES:-1024}" -FILTER_PATH="${FILTER_PATH:-../../worker/main-x86-virt}" -BRIDGE_IFACE="${BRIDGE_IFACE:-br-testnet}" +QEMU_IMAGE="${QEMU_IMAGE:-ubuntu-24.04.4-preinstalled-server-riscv64.img}" +QEMU_MEMORY="${QEMU_MEMORY:-4G}" +QEMU_CPUS="${QEMU_CPUS:-2}" +FILTER_RISCV_BIN="${FILTER_RISCV_BIN:-../../worker/main-riscv}" +CONTROLLER_BIN="${CONTROLLER_BIN:-../../controller/bin/grpc_server}" +FILTER1_MAC="52:54:00:f1:00:01" +FILTER2_MAC="52:54:00:f2:00:01" "$SCRIPT_DIR/stop.sh" 2>/dev/null || true -sysctl -w vm.drop_caches=3 > /dev/null -echo "$HUGEPAGES" > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages +sysctl -w vm.drop_caches=3 >/dev/null +echo "$HUGEPAGES" >/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages docker compose up -d --build -for i in $(seq 1 10); do - ip link show "$BRIDGE_IFACE" &>/dev/null && break - sleep 0.5 +for BRIDGE in "$BRIDGE1" "$BRIDGE2"; do + for i in $(seq 1 10); do + ip link show "$BRIDGE" &>/dev/null && break + sleep 0.5 + done + if ! ip link show "$BRIDGE" &>/dev/null; then + echo "Bridge $BRIDGE not found" + exit 1 + fi done -if ! ip link show "$BRIDGE_IFACE" &>/dev/null; then - echo "Bridge interface $BRIDGE_IFACE not found" - exit 1 +sysctl -w net.ipv4.ip_forward=1 >/dev/null + +ip link add "$INET_BRIDGE" type bridge 2>/dev/null || true +ip link set "$INET_BRIDGE" up +ip addr add "${INET_SUBNET}.254/24" dev "$INET_BRIDGE" 2>/dev/null || true + +iptables -t nat -A POSTROUTING -s ${INET_SUBNET}.0/24 ! -d ${INET_SUBNET}.0/24 -j MASQUERADE +iptables -I FORWARD -i "$INET_BRIDGE" -j ACCEPT +iptables -I FORWARD -o "$INET_BRIDGE" -j ACCEPT + +ip link add "$MGMT_BRIDGE" type bridge 2>/dev/null || true +ip link set "$MGMT_BRIDGE" up +ip addr add "${MGMT_SUBNET}.254/24" dev "$MGMT_BRIDGE" 2>/dev/null || true + +SHARED_DIR="$PROJECT_DIR/shared" +mkdir -p "$SHARED_DIR" + +if [ -f "$FILTER_RISCV_BIN" ]; then + cp "$FILTER_RISCV_BIN" "$SHARED_DIR/filter" +fi +if [ -f "$CONTROLLER_BIN" ]; then + cp "$CONTROLLER_BIN" "$SHARED_DIR/controller" fi -ip link delete veth0 2>/dev/null || true -ip link add veth0 type veth peer name veth1 -ip link set veth0 up -ip link set veth1 up +DPDK_SYSROOT="$PROJECT_DIR/../../worker/dpdk-riscv-install" +if [ -d "$DPDK_SYSROOT/lib" ]; then + mkdir -p "$SHARED_DIR/lib/bpf" + cp "$DPDK_SYSROOT"/lib/*.so* "$SHARED_DIR/lib/" + cp "$DPDK_SYSROOT"/lib/bpf/*.o "$SHARED_DIR/lib/bpf/" 2>/dev/null || true +fi +LIBXDP_DEB="libxdp1_1.4.2-1ubuntu4_riscv64.deb" +if [ ! -f "$SHARED_DIR/$LIBXDP_DEB" ]; then + wget -q -P "$SHARED_DIR" "http://ports.ubuntu.com/ubuntu-ports/pool/main/x/xdp-tools/$LIBXDP_DEB" +fi + +mkdir -p "$SHARED_DIR/internal/service/config" +cp ../../controller/internal/service/config/categories.json "$SHARED_DIR/internal/service/config/" +cp ../../controller/internal/service/config/providers.json "$SHARED_DIR/internal/service/config/" +if [ -d "$PROJECT_DIR/configs" ]; then + cp -r "$PROJECT_DIR/configs" "$SHARED_DIR/" +fi + +gen_cloud_init_iso() { + local VM_NAME="$1" + local VM_TYPE="$2" + local MGMT_IP="$3" + local GATEWAY_IP="${4:-}" + local CI_DIR="$PROJECT_DIR/cloud-init-${VM_NAME}" + local CI_TMPL="$PROJECT_DIR/configs/cloud-init" + + mkdir -p "$CI_DIR" + sed "s/VM_NAME/${VM_NAME}/g" "$CI_TMPL/meta-data" > "$CI_DIR/meta-data" + sed "s/VM_NAME/${VM_NAME}/g" "$CI_TMPL/${VM_TYPE}-user-data" > "$CI_DIR/user-data" + sed -e "s/MGMT_IP/${MGMT_IP}/g" -e "s/GATEWAY_IP/${GATEWAY_IP}/g" \ + "$CI_TMPL/${VM_TYPE}-network-config" > "$CI_DIR/network-config" + + mkisofs -o "$PROJECT_DIR/cloud-init-${VM_NAME}.iso" \ + -V cidata -J -r "$CI_DIR" > /dev/null 2>&1 -tc qdisc add dev "$BRIDGE_IFACE" clsact -tc filter add dev "$BRIDGE_IFACE" egress matchall action mirred egress mirror dev veth1 -tc filter add dev "$BRIDGE_IFACE" ingress matchall action mirred egress mirror dev veth1 + rm -rf "$CI_DIR" +} -sysctl -w net.ipv4.ip_forward=1 > /dev/null -iptables -t nat -A POSTROUTING -s ${SUBNET}.0/24 ! -d ${SUBNET}.0/24 -j MASQUERADE -iptables -I FORWARD -i "$BRIDGE_IFACE" -j ACCEPT -iptables -I FORWARD -o "$BRIDGE_IFACE" -j ACCEPT +start_controller() { + qemu-system-riscv64 \ + -machine virt,acpi=off -m "$QEMU_MEMORY" -smp cpus="$QEMU_CPUS" \ + -display none -serial "file:/tmp/qemu-controller.log" \ + -name "controller" \ + -pidfile "/tmp/qemu-controller.pid" \ + -kernel /usr/lib/u-boot/qemu-riscv64_smode/uboot.elf \ + -netdev tap,id=net0,ifname="tap-ctrl",script=no,downscript=no \ + -device virtio-net-device,netdev=net0 \ + -device virtio-rng-pci \ + -drive "file=${CTRL_OVERLAY},format=qcow2,if=virtio" \ + -drive "file=$PROJECT_DIR/cloud-init-controller.iso,format=raw,if=virtio" \ + -virtfs "local,path=$SHARED_DIR,mount_tag=host_share,security_model=mapped-xattr" \ + -daemonize || + { + echo "Failed to start controller" + exit 1 + } +} -if [ ! -x "$FILTER_PATH" ]; then - echo "Filter binary not found: $FILTER_PATH" - echo "Build: cd worker && make -f Makefile.main_x86 virt" - exit 1 +start_filter_vm() { + local VM_NAME="$1" + local TAP_IN="$2" + local TAP_OUT="$3" + local TAP_MGMT="$4" + local BRIDGE_IN="$5" + local MGMT_IP="$6" + local ETH0_MAC="$7" + + local OVERLAY="$PROJECT_DIR/${VM_NAME}.qcow2" + if [ ! -f "$OVERLAY" ]; then + qemu-img create -f qcow2 -b "$(realpath "$QEMU_IMAGE")" -F raw "$OVERLAY" >/dev/null + gen_cloud_init_iso "$VM_NAME" "filter" "$MGMT_IP" + fi + + ip tuntap add dev "$TAP_IN" mode tap 2>/dev/null || true + ip link set "$TAP_IN" master "$BRIDGE_IN" + ip link set "$TAP_IN" up + + ip tuntap add dev "$TAP_OUT" mode tap 2>/dev/null || true + ip link set "$TAP_OUT" master "$INET_BRIDGE" + ip link set "$TAP_OUT" up + + ip tuntap add dev "$TAP_MGMT" mode tap 2>/dev/null || true + ip link set "$TAP_MGMT" master "$MGMT_BRIDGE" + ip link set "$TAP_MGMT" up + + qemu-system-riscv64 \ + -machine virt,acpi=off -m "$QEMU_MEMORY" -smp cpus="$QEMU_CPUS" \ + -display none -serial "file:/tmp/qemu-${VM_NAME}.log" \ + -name "$VM_NAME" \ + -pidfile "/tmp/qemu-${VM_NAME}.pid" \ + -kernel /usr/lib/u-boot/qemu-riscv64_smode/uboot.elf \ + -netdev tap,id=net0,ifname="$TAP_IN",script=no,downscript=no \ + -device virtio-net-device,netdev=net0,mac="$ETH0_MAC" \ + -netdev tap,id=net1,ifname="$TAP_OUT",script=no,downscript=no \ + -device virtio-net-device,netdev=net1 \ + -netdev tap,id=net2,ifname="$TAP_MGMT",script=no,downscript=no \ + -device virtio-net-device,netdev=net2 \ + -device virtio-rng-pci \ + -drive "file=${OVERLAY},format=qcow2,if=virtio" \ + -drive "file=$PROJECT_DIR/cloud-init-${VM_NAME}.iso,format=raw,if=virtio" \ + -virtfs "local,path=$SHARED_DIR,mount_tag=host_share,security_model=mapped-xattr" \ + -daemonize || + { + echo "Failed to start $VM_NAME" + exit 1 + } +} + +start_filter_vm "filter1" "tap-f1-in" "tap-f1-out" "tap-f1-mgmt" "$BRIDGE1" "${MGMT_SUBNET}.1" "$FILTER1_MAC" +start_filter_vm "filter2" "tap-f2-in" "tap-f2-out" "tap-f2-mgmt" "$BRIDGE2" "${MGMT_SUBNET}.2" "$FILTER2_MAC" + +for SVC in $(docker compose ps -q 2>/dev/null); do + docker exec "$SVC" ip neigh flush all 2>/dev/null || true +done +for SVC in $(docker compose -p "$(basename "$PROJECT_DIR")" ps --format '{{.Name}}' 2>/dev/null | grep "gen-1"); do + docker exec "$SVC" arp -s "${SUBNET1}.254" "$FILTER1_MAC" 2>/dev/null || true +done +for SVC in $(docker compose -p "$(basename "$PROJECT_DIR")" ps --format '{{.Name}}' 2>/dev/null | grep "gen-2"); do + docker exec "$SVC" arp -s "${SUBNET2}.254" "$FILTER2_MAC" 2>/dev/null || true +done + +CTRL_OVERLAY="$PROJECT_DIR/controller.qcow2" +if [ ! -f "$CTRL_OVERLAY" ]; then + qemu-img create -f qcow2 -b "$(realpath "$QEMU_IMAGE")" -F raw "$CTRL_OVERLAY" >/dev/null + gen_cloud_init_iso "controller" "controller" "${MGMT_SUBNET}.3" fi -LD_LIBRARY_PATH=/usr/local/lib exec "$FILTER_PATH" --no-pci -- +ip tuntap add dev "tap-ctrl" mode tap 2>/dev/null || true +ip link set "tap-ctrl" master "$MGMT_BRIDGE" +ip link set "tap-ctrl" up + +start_controller + +echo "Test stand is running" +echo " Filter VMs: filter1 (pid $(cat /tmp/qemu-filter1.pid 2>/dev/null)), filter2 (pid $(cat /tmp/qemu-filter2.pid 2>/dev/null))" +echo " Controller: pid $(cat /tmp/qemu-controller.pid 2>/dev/null)" +echo " Traffic generators: $(docker compose ps --format '{{.Name}}' | wc -l) containers" diff --git a/test/virtual_load_network/scripts/stop.sh b/test/virtual_load_network/scripts/stop.sh index 6abeaf5..b022e66 100755 --- a/test/virtual_load_network/scripts/stop.sh +++ b/test/virtual_load_network/scripts/stop.sh @@ -11,14 +11,31 @@ if [ -f .env ]; then set +a fi -SUBNET="${SUBNET:-10.0.0}" -BRIDGE_IFACE="${BRIDGE_IFACE:-br-testnet}" +INET_SUBNET="${INET_SUBNET:-10.0.3}" +INET_BRIDGE="${INET_BRIDGE:-br-inet}" +MGMT_BRIDGE="${MGMT_BRIDGE:-br-mgmt}" -tc qdisc del dev "$BRIDGE_IFACE" clsact 2>/dev/null -iptables -D FORWARD -i "$BRIDGE_IFACE" -j ACCEPT 2>/dev/null -iptables -D FORWARD -o "$BRIDGE_IFACE" -j ACCEPT 2>/dev/null +for VM in filter1 filter2 controller; do + PID_FILE="/tmp/qemu-${VM}.pid" + if [ -f "$PID_FILE" ]; then + kill "$(cat "$PID_FILE")" 2>/dev/null + rm -f "$PID_FILE" + fi +done docker compose down 2>/dev/null -iptables -t nat -D POSTROUTING -s ${SUBNET}.0/24 ! -d ${SUBNET}.0/24 -j MASQUERADE 2>/dev/null -ip link delete veth0 2>/dev/null +iptables -t nat -D POSTROUTING -s ${INET_SUBNET}.0/24 ! -d ${INET_SUBNET}.0/24 -j MASQUERADE 2>/dev/null +iptables -D FORWARD -i "$INET_BRIDGE" -j ACCEPT 2>/dev/null +iptables -D FORWARD -o "$INET_BRIDGE" -j ACCEPT 2>/dev/null + +for TAP in tap-f1-in tap-f1-out tap-f2-in tap-f2-out tap-f1-mgmt tap-f2-mgmt tap-ctrl; do + ip tuntap del dev "$TAP" mode tap 2>/dev/null +done + +ip link delete "$INET_BRIDGE" 2>/dev/null +ip link delete "$MGMT_BRIDGE" 2>/dev/null + +rm -rf "$PROJECT_DIR/shared" +rm -f "$PROJECT_DIR"/filter1.qcow2 "$PROJECT_DIR"/filter2.qcow2 "$PROJECT_DIR"/controller.qcow2 +rm -f "$PROJECT_DIR"/cloud-init-*.iso diff --git a/worker/src/dpdk_filter/main.c b/worker/src/dpdk_filter/main.c index ea54728..95eacff 100644 --- a/worker/src/dpdk_filter/main.c +++ b/worker/src/dpdk_filter/main.c @@ -36,6 +36,8 @@ void forward_to_out(struct net_port *incoming_port, } int main(int argc, char **argv) { + // for test_stand disable bufferization + setvbuf(stdout, NULL, _IOLBF, 0); // since BASE_POLICY is filled when initializing worker, let’s initialize here struct BASE_POLICY policy; if (signal(SIGINT, signal_handler) == SIG_ERR) { @@ -134,4 +136,4 @@ int main(int argc, char **argv) { net_port_destroy(port_out); net_port_destroy(port_exception); return 0; -} \ No newline at end of file +} From 8595f23eaca0a85bc3941c7fcdc5866dab10e5d0 Mon Sep 17 00:00:00 2001 From: baydakov-georgiy Date: Mon, 20 Apr 2026 20:07:05 +0300 Subject: [PATCH 02/13] refactor(test_stand): replace Ubuntu image with Yocto for QEMU VMs --- test/virtual_load_network/.env | 5 +- test/virtual_load_network/Dockerfile.dpdk | 55 -------------- test/virtual_load_network/Makefile | 15 ++-- .../cloud-init/controller-network-config | 6 -- .../configs/cloud-init/controller-user-data | 23 ------ .../configs/cloud-init/filter-network-config | 10 --- .../configs/cloud-init/filter-user-data | 10 --- .../configs/cloud-init/meta-data | 2 - .../configs/riscv64-cross.txt | 20 ----- test/virtual_load_network/create_image.sh | 62 --------------- test/virtual_load_network/scripts/start-vm.sh | 45 +++++++---- test/virtual_load_network/scripts/start.sh | 75 +++++++------------ 12 files changed, 64 insertions(+), 264 deletions(-) delete mode 100644 test/virtual_load_network/Dockerfile.dpdk delete mode 100644 test/virtual_load_network/configs/cloud-init/controller-network-config delete mode 100644 test/virtual_load_network/configs/cloud-init/controller-user-data delete mode 100644 test/virtual_load_network/configs/cloud-init/filter-network-config delete mode 100644 test/virtual_load_network/configs/cloud-init/filter-user-data delete mode 100644 test/virtual_load_network/configs/cloud-init/meta-data delete mode 100644 test/virtual_load_network/configs/riscv64-cross.txt delete mode 100755 test/virtual_load_network/create_image.sh diff --git a/test/virtual_load_network/.env b/test/virtual_load_network/.env index 699bf5b..17bc607 100644 --- a/test/virtual_load_network/.env +++ b/test/virtual_load_network/.env @@ -15,7 +15,10 @@ INET_BRIDGE=br-inet MGMT_SUBNET=10.0.2 MGMT_BRIDGE=br-mgmt -QEMU_IMAGE=ubuntu-24.04.4-preinstalled-server-riscv64.img +YOCTO_DEPLOY_DIR=/home/lespend/program/yadro/vm_build_risc_v/qemu/poky/build/tmp/deploy/images/qemuriscv64 +QEMU_ROOTFS=${YOCTO_DEPLOY_DIR}/cluster-image-qemuriscv64.rootfs.ext4 +QEMU_KERNEL=${YOCTO_DEPLOY_DIR}/Image +QEMU_BIOS=${YOCTO_DEPLOY_DIR}/fw_jump.elf QEMU_MEMORY=4G QEMU_CPUS=2 diff --git a/test/virtual_load_network/Dockerfile.dpdk b/test/virtual_load_network/Dockerfile.dpdk deleted file mode 100644 index 373ed60..0000000 --- a/test/virtual_load_network/Dockerfile.dpdk +++ /dev/null @@ -1,55 +0,0 @@ -FROM --platform=linux/riscv64 ubuntu:24.04 - -RUN apt update && apt install -y \ - build-essential meson ninja-build python3-pyelftools \ - libelf-dev libnuma-dev pkg-config git wget \ - clang llvm m4 libpcap-dev libatomic1 - -WORKDIR /tmp - -RUN git clone --depth 1 --branch v1.4.6 https://github.com/libbpf/libbpf.git && \ - cd libbpf/src && make && \ - make install PREFIX=/usr LIBDIR=/usr/lib/riscv64-linux-gnu && \ - ldconfig && rm -rf /tmp/libbpf - -RUN git clone --depth 1 --branch v1.4.2 https://github.com/xdp-project/xdp-tools.git && \ - cd xdp-tools && ./configure && make && \ - make -C lib/libxdp install PREFIX=/usr LIBDIR=/usr/lib/riscv64-linux-gnu && \ - ln -sf /usr/include/xdp/xsk.h /usr/include/bpf/xsk.h && \ - ldconfig && rm -rf /tmp/xdp-tools - -RUN pkg-config --exists libbpf && echo "libbpf OK" && \ - pkg-config --exists libxdp && echo "libxdp OK" - -RUN wget https://fast.dpdk.org/rel/dpdk-23.11.6.tar.xz && \ - tar -xf dpdk-23.11.6.tar.xz && \ - cd dpdk-stable-23.11.6 && \ - meson setup -Denable_drivers=net/af_xdp,net/tap -Dmachine=generic build && \ - ninja -C build && ninja -C build install && ldconfig && \ - rm -rf /tmp/dpdk-* - -RUN mkdir -p /output/include /output/lib/pkgconfig /output/lib/bpf /output/runtime && \ - cp /usr/lib/riscv64-linux-gnu/bpf/*.o /output/lib/bpf/ 2>/dev/null; \ - cp -r /usr/local/include/* /output/include/ 2>/dev/null; \ - cp -r /usr/include/bpf /output/include/ 2>/dev/null; \ - cp -r /usr/include/xdp /output/include/ 2>/dev/null; \ - for d in /usr/local/lib/riscv64-linux-gnu /usr/local/lib; do \ - [ -f "$d/librte_eal.so" ] && { \ - cp "$d"/librte_*.so* /output/lib/; \ - cp "$d"/librte_*.a /output/lib/ 2>/dev/null; \ - cp "$d"/pkgconfig/*.pc /output/lib/pkgconfig/ 2>/dev/null; \ - cp "$d"/librte_*.so* /output/runtime/; \ - break; \ - }; \ - done; \ - for lib in libbpf libxdp libelf libnuma; do \ - for d in /usr/lib/riscv64-linux-gnu /usr/lib /usr/local/lib; do \ - ls "$d"/${lib}.so* &>/dev/null 2>&1 && { \ - cp "$d"/${lib}.so* /output/lib/; \ - cp "$d"/${lib}.a /output/lib/ 2>/dev/null; \ - cp "$d"/${lib}.so* /output/runtime/; \ - break; \ - }; \ - done; \ - done; \ - true diff --git a/test/virtual_load_network/Makefile b/test/virtual_load_network/Makefile index 8f39899..cf4f97e 100644 --- a/test/virtual_load_network/Makefile +++ b/test/virtual_load_network/Makefile @@ -1,9 +1,8 @@ SCRIPTS_DIR = scripts WORKER_DIR = ../../worker CONTROLLER_DIR = ../../controller -DPDK_RISCV_INSTALL = $(WORKER_DIR)/dpdk-riscv-install -.PHONY: run stop logs build build-filter build-controller clean-vms +.PHONY: run stop logs build build-worker build-controller clean-vms run: build sudo $(SCRIPTS_DIR)/start.sh @@ -14,17 +13,13 @@ stop: logs: @sudo bash -c 'while true; do clear; cat shared/filter1.log 2>/dev/null | tail -10; echo "---"; cat shared/filter2.log 2>/dev/null | tail -10; echo "---"; cat shared/controller.log 2>/dev/null | tail -5; sleep 2; done' -build: build-filter build-controller +build: build-worker build-controller -build-filter: - @if [ ! -d "$(DPDK_RISCV_INSTALL)" ]; then \ - echo "DPDK sysroot not found. Run ./create_image.sh first."; \ - exit 1; \ - fi - $(MAKE) -C $(WORKER_DIR) -f Makefile.main_riscv +build-worker: + cd $(WORKER_DIR) && bazel build --config=riscv64 //:worker build-controller: $(MAKE) -C $(CONTROLLER_DIR) build-riscv clean-vms: - rm -f filter1.qcow2 filter2.qcow2 controller.qcow2 cloud-init-*.iso + rm -f filter1.qcow2 filter2.qcow2 controller.qcow2 diff --git a/test/virtual_load_network/configs/cloud-init/controller-network-config b/test/virtual_load_network/configs/cloud-init/controller-network-config deleted file mode 100644 index 16b55fc..0000000 --- a/test/virtual_load_network/configs/cloud-init/controller-network-config +++ /dev/null @@ -1,6 +0,0 @@ -version: 2 -ethernets: - eth0: - dhcp4: false - addresses: - - MGMT_IP/24 diff --git a/test/virtual_load_network/configs/cloud-init/controller-user-data b/test/virtual_load_network/configs/cloud-init/controller-user-data deleted file mode 100644 index 613233b..0000000 --- a/test/virtual_load_network/configs/cloud-init/controller-user-data +++ /dev/null @@ -1,23 +0,0 @@ -#cloud-config -mounts: - - ["host_share", "/mnt", "9p", "trans=virtio,version=9p2000.L", "0", "0"] -runcmd: - - | - cat > /etc/systemd/system/controller.service < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages - - dpkg -i /mnt/libxdp1_1.4.2-1ubuntu4_riscv64.deb diff --git a/test/virtual_load_network/configs/cloud-init/meta-data b/test/virtual_load_network/configs/cloud-init/meta-data deleted file mode 100644 index f8ea600..0000000 --- a/test/virtual_load_network/configs/cloud-init/meta-data +++ /dev/null @@ -1,2 +0,0 @@ -instance-id: VM_NAME -local-hostname: VM_NAME diff --git a/test/virtual_load_network/configs/riscv64-cross.txt b/test/virtual_load_network/configs/riscv64-cross.txt deleted file mode 100644 index 6bbfa70..0000000 --- a/test/virtual_load_network/configs/riscv64-cross.txt +++ /dev/null @@ -1,20 +0,0 @@ -[binaries] -c = 'riscv64-linux-gnu-gcc' -cpp = 'riscv64-linux-gnu-g++' -ar = 'riscv64-linux-gnu-ar' -strip = 'riscv64-linux-gnu-strip' -pkgconfig = 'pkg-config' - -[built-in options] -c_args = ['-march=rv64gc'] -cpp_args = ['-march=rv64gc'] - -[properties] -vendor_id = 'generic' -arch_id = 'generic' - -[host_machine] -system = 'linux' -cpu_family = 'riscv64' -cpu = 'rv64gc' -endian = 'little' diff --git a/test/virtual_load_network/create_image.sh b/test/virtual_load_network/create_image.sh deleted file mode 100755 index 725c6f9..0000000 --- a/test/virtual_load_network/create_image.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -set -e - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -cd "$SCRIPT_DIR" - -IMAGE="ubuntu-24.04.4-preinstalled-server-riscv64.img" -IMAGE_XZ="${IMAGE}.xz" -IMAGE_URL="https://cdimage.ubuntu.com/releases/24.04/release/${IMAGE_XZ}" -DPDK_DEST="$SCRIPT_DIR/../../worker/dpdk-riscv-install" -RUNTIME_DEST="$SCRIPT_DIR/shared/lib" - -if ! docker buildx version &>/dev/null; then - echo "Docker buildx required. Install docker-buildx." - exit 1 -fi - -echo "Building DPDK for RISC-V in Docker..." -docker buildx build --platform linux/riscv64 -t dpdk-riscv-builder -f Dockerfile.dpdk --load . - -echo "Extracting sysroot for cross-compilation..." -rm -rf "$DPDK_DEST" -mkdir -p "$DPDK_DEST" -CONTAINER=$(docker create --platform linux/riscv64 dpdk-riscv-builder) -docker cp "$CONTAINER:/output/include" "$DPDK_DEST/include" -docker cp "$CONTAINER:/output/lib" "$DPDK_DEST/lib" - -echo "Extracting runtime libraries..." -rm -rf "$RUNTIME_DEST" -mkdir -p "$RUNTIME_DEST" -docker cp "$CONTAINER:/output/runtime/." "$RUNTIME_DEST/" -docker rm "$CONTAINER" - -echo "Sysroot: $DPDK_DEST" -echo "Runtime libs: $RUNTIME_DEST" - -if ! cat /proc/sys/fs/binfmt_misc/qemu-riscv64 &>/dev/null; then - echo ':qemu-riscv64:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xf3\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/bin/qemu-riscv64-static:FPC' | sudo tee /proc/sys/fs/binfmt_misc/register >/dev/null -fi - -if [ ! -f "$IMAGE" ]; then - if [ ! -f "$IMAGE_XZ" ]; then - echo "Downloading Ubuntu RISC-V image..." - wget "$IMAGE_URL" - fi - xz -dk "$IMAGE_XZ" -fi - -echo "Preparing VM image (installing libatomic)..." -sudo losetup -D 2>/dev/null || true -LOOP=$(sudo losetup -f) -sudo losetup -P "$LOOP" "$IMAGE" -sudo mkdir -p /mnt/riscv-rootfs -sudo mount "${LOOP}p1" /mnt/riscv-rootfs -sudo cp /usr/bin/qemu-riscv64-static /mnt/riscv-rootfs/usr/bin/ -sudo rm -f /mnt/riscv-rootfs/etc/resolv.conf -sudo cp /etc/resolv.conf /mnt/riscv-rootfs/etc/resolv.conf -sudo chroot /mnt/riscv-rootfs /bin/bash -c "apt update && apt install -y libatomic1 libnuma1" -sudo umount /mnt/riscv-rootfs -sudo losetup -d "$LOOP" - -echo "Done. Image ready: $IMAGE" diff --git a/test/virtual_load_network/scripts/start-vm.sh b/test/virtual_load_network/scripts/start-vm.sh index de4ed08..d19d1ba 100755 --- a/test/virtual_load_network/scripts/start-vm.sh +++ b/test/virtual_load_network/scripts/start-vm.sh @@ -1,37 +1,52 @@ #!/bin/bash set -e +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +if [ -f "$PROJECT_DIR/.env" ]; then + set -a + source "$PROJECT_DIR/.env" + set +a +fi + VM_NAME="$1" -VM_IMAGE="$2" -SHARED_DIR="$3" -TAP_IFACE="$4" -MEMORY="${5:-4G}" -CPUS="${6:-2}" - -if [ -z "$VM_NAME" ] || [ -z "$VM_IMAGE" ] || [ -z "$SHARED_DIR" ] || [ -z "$TAP_IFACE" ]; then - echo "Usage: $0 [memory] [cpus]" +SHARED_DIR="$2" +TAP_IFACE="$3" +MEMORY="${4:-${QEMU_MEMORY:-4G}}" +CPUS="${5:-${QEMU_CPUS:-2}}" + +YOCTO_DEPLOY_DIR="${YOCTO_DEPLOY_DIR:-/home/lespend/program/yadro/vm_build_risc_v/qemu/poky/build/tmp/deploy/images/qemuriscv64}" +QEMU_ROOTFS="${QEMU_ROOTFS:-${YOCTO_DEPLOY_DIR}/cluster-image-qemuriscv64.rootfs.ext4}" +QEMU_KERNEL="${QEMU_KERNEL:-${YOCTO_DEPLOY_DIR}/Image}" +QEMU_BIOS="${QEMU_BIOS:-${YOCTO_DEPLOY_DIR}/fw_jump.elf}" + +if [ -z "$VM_NAME" ] || [ -z "$SHARED_DIR" ] || [ -z "$TAP_IFACE" ]; then + echo "Usage: $0 [memory] [cpus]" exit 1 fi -if [ ! -f "$VM_IMAGE" ]; then - echo "Image not found: $VM_IMAGE" - exit 1 +OVERLAY="$PROJECT_DIR/${VM_NAME}.qcow2" +if [ ! -f "$OVERLAY" ]; then + qemu-img create -f qcow2 -b "$QEMU_ROOTFS" -F raw "$OVERLAY" >/dev/null fi ip tuntap add dev "$TAP_IFACE" mode tap 2>/dev/null || true ip link set "$TAP_IFACE" up qemu-system-riscv64 \ - -machine virt,acpi=off -m "$MEMORY" -smp cpus="$CPUS" \ + -machine virt -m "$MEMORY" -smp "$CPUS" \ -display none -serial "file:/tmp/qemu-${VM_NAME}.log" \ -name "$VM_NAME" \ -pidfile "/tmp/qemu-${VM_NAME}.pid" \ - -bios default \ + -bios "$QEMU_BIOS" \ + -kernel "$QEMU_KERNEL" \ + -append "root=/dev/vda rw earlycon=sbi console=ttyS0" \ -netdev tap,id=net0,ifname="$TAP_IFACE",script=no,downscript=no \ -device virtio-net-device,netdev=net0 \ -netdev user,id=net1,hostfwd=tcp::$(shuf -i 10000-20000 -n1)-:22 \ -device virtio-net-device,netdev=net1 \ - -device virtio-rng-pci \ - -drive "file=$VM_IMAGE,format=raw,if=virtio" \ + -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \ + -drive "id=disk0,file=${OVERLAY},format=qcow2,if=none" -device virtio-blk-device,drive=disk0 \ -virtfs "local,path=$SHARED_DIR,mount_tag=host_share,security_model=mapped-xattr" \ -daemonize diff --git a/test/virtual_load_network/scripts/start.sh b/test/virtual_load_network/scripts/start.sh index 150d757..f69b96e 100755 --- a/test/virtual_load_network/scripts/start.sh +++ b/test/virtual_load_network/scripts/start.sh @@ -21,7 +21,10 @@ INET_BRIDGE="${INET_BRIDGE:-br-inet}" MGMT_SUBNET="${MGMT_SUBNET:-10.0.2}" MGMT_BRIDGE="${MGMT_BRIDGE:-br-mgmt}" HUGEPAGES="${HUGEPAGES:-1024}" -QEMU_IMAGE="${QEMU_IMAGE:-ubuntu-24.04.4-preinstalled-server-riscv64.img}" +YOCTO_DEPLOY_DIR="${YOCTO_DEPLOY_DIR:-/home/lespend/program/yadro/vm_build_risc_v/qemu/poky/build/tmp/deploy/images/qemuriscv64}" +QEMU_ROOTFS="${QEMU_ROOTFS:-${YOCTO_DEPLOY_DIR}/cluster-image-qemuriscv64.rootfs.ext4}" +QEMU_KERNEL="${QEMU_KERNEL:-${YOCTO_DEPLOY_DIR}/Image}" +QEMU_BIOS="${QEMU_BIOS:-${YOCTO_DEPLOY_DIR}/fw_jump.elf}" QEMU_MEMORY="${QEMU_MEMORY:-4G}" QEMU_CPUS="${QEMU_CPUS:-2}" FILTER_RISCV_BIN="${FILTER_RISCV_BIN:-../../worker/main-riscv}" @@ -61,6 +64,9 @@ ip link add "$MGMT_BRIDGE" type bridge 2>/dev/null || true ip link set "$MGMT_BRIDGE" up ip addr add "${MGMT_SUBNET}.254/24" dev "$MGMT_BRIDGE" 2>/dev/null || true +iptables -I FORWARD -i "$MGMT_BRIDGE" -j ACCEPT +iptables -I FORWARD -o "$MGMT_BRIDGE" -j ACCEPT + SHARED_DIR="$PROJECT_DIR/shared" mkdir -p "$SHARED_DIR" @@ -71,17 +77,6 @@ if [ -f "$CONTROLLER_BIN" ]; then cp "$CONTROLLER_BIN" "$SHARED_DIR/controller" fi -DPDK_SYSROOT="$PROJECT_DIR/../../worker/dpdk-riscv-install" -if [ -d "$DPDK_SYSROOT/lib" ]; then - mkdir -p "$SHARED_DIR/lib/bpf" - cp "$DPDK_SYSROOT"/lib/*.so* "$SHARED_DIR/lib/" - cp "$DPDK_SYSROOT"/lib/bpf/*.o "$SHARED_DIR/lib/bpf/" 2>/dev/null || true -fi -LIBXDP_DEB="libxdp1_1.4.2-1ubuntu4_riscv64.deb" -if [ ! -f "$SHARED_DIR/$LIBXDP_DEB" ]; then - wget -q -P "$SHARED_DIR" "http://ports.ubuntu.com/ubuntu-ports/pool/main/x/xdp-tools/$LIBXDP_DEB" -fi - mkdir -p "$SHARED_DIR/internal/service/config" cp ../../controller/internal/service/config/categories.json "$SHARED_DIR/internal/service/config/" cp ../../controller/internal/service/config/providers.json "$SHARED_DIR/internal/service/config/" @@ -89,38 +84,24 @@ if [ -d "$PROJECT_DIR/configs" ]; then cp -r "$PROJECT_DIR/configs" "$SHARED_DIR/" fi -gen_cloud_init_iso() { - local VM_NAME="$1" - local VM_TYPE="$2" - local MGMT_IP="$3" - local GATEWAY_IP="${4:-}" - local CI_DIR="$PROJECT_DIR/cloud-init-${VM_NAME}" - local CI_TMPL="$PROJECT_DIR/configs/cloud-init" - - mkdir -p "$CI_DIR" - sed "s/VM_NAME/${VM_NAME}/g" "$CI_TMPL/meta-data" > "$CI_DIR/meta-data" - sed "s/VM_NAME/${VM_NAME}/g" "$CI_TMPL/${VM_TYPE}-user-data" > "$CI_DIR/user-data" - sed -e "s/MGMT_IP/${MGMT_IP}/g" -e "s/GATEWAY_IP/${GATEWAY_IP}/g" \ - "$CI_TMPL/${VM_TYPE}-network-config" > "$CI_DIR/network-config" - - mkisofs -o "$PROJECT_DIR/cloud-init-${VM_NAME}.iso" \ - -V cidata -J -r "$CI_DIR" > /dev/null 2>&1 - - rm -rf "$CI_DIR" -} - start_controller() { + local CTRL_OVERLAY="$PROJECT_DIR/controller.qcow2" + if [ ! -f "$CTRL_OVERLAY" ]; then + qemu-img create -f qcow2 -b "$QEMU_ROOTFS" -F raw "$CTRL_OVERLAY" >/dev/null + fi + qemu-system-riscv64 \ - -machine virt,acpi=off -m "$QEMU_MEMORY" -smp cpus="$QEMU_CPUS" \ + -machine virt -m "$QEMU_MEMORY" -smp "$QEMU_CPUS" \ -display none -serial "file:/tmp/qemu-controller.log" \ -name "controller" \ -pidfile "/tmp/qemu-controller.pid" \ - -kernel /usr/lib/u-boot/qemu-riscv64_smode/uboot.elf \ + -bios "$QEMU_BIOS" \ + -kernel "$QEMU_KERNEL" \ + -append "root=/dev/vda rw earlycon=sbi console=ttyS0 ip=${MGMT_SUBNET}.3::${MGMT_SUBNET}.254:255.255.255.0::eth0:off" \ -netdev tap,id=net0,ifname="tap-ctrl",script=no,downscript=no \ -device virtio-net-device,netdev=net0 \ - -device virtio-rng-pci \ - -drive "file=${CTRL_OVERLAY},format=qcow2,if=virtio" \ - -drive "file=$PROJECT_DIR/cloud-init-controller.iso,format=raw,if=virtio" \ + -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \ + -drive "id=disk0,file=${CTRL_OVERLAY},format=qcow2,if=none" -device virtio-blk-device,drive=disk0 \ -virtfs "local,path=$SHARED_DIR,mount_tag=host_share,security_model=mapped-xattr" \ -daemonize || { @@ -140,8 +121,7 @@ start_filter_vm() { local OVERLAY="$PROJECT_DIR/${VM_NAME}.qcow2" if [ ! -f "$OVERLAY" ]; then - qemu-img create -f qcow2 -b "$(realpath "$QEMU_IMAGE")" -F raw "$OVERLAY" >/dev/null - gen_cloud_init_iso "$VM_NAME" "filter" "$MGMT_IP" + qemu-img create -f qcow2 -b "$QEMU_ROOTFS" -F raw "$OVERLAY" >/dev/null fi ip tuntap add dev "$TAP_IN" mode tap 2>/dev/null || true @@ -157,20 +137,21 @@ start_filter_vm() { ip link set "$TAP_MGMT" up qemu-system-riscv64 \ - -machine virt,acpi=off -m "$QEMU_MEMORY" -smp cpus="$QEMU_CPUS" \ + -machine virt -m "$QEMU_MEMORY" -smp "$QEMU_CPUS" \ -display none -serial "file:/tmp/qemu-${VM_NAME}.log" \ -name "$VM_NAME" \ -pidfile "/tmp/qemu-${VM_NAME}.pid" \ - -kernel /usr/lib/u-boot/qemu-riscv64_smode/uboot.elf \ + -bios "$QEMU_BIOS" \ + -kernel "$QEMU_KERNEL" \ + -append "root=/dev/vda rw earlycon=sbi console=ttyS0 ip=${MGMT_IP}::${MGMT_SUBNET}.254:255.255.255.0::eth2:off" \ -netdev tap,id=net0,ifname="$TAP_IN",script=no,downscript=no \ -device virtio-net-device,netdev=net0,mac="$ETH0_MAC" \ -netdev tap,id=net1,ifname="$TAP_OUT",script=no,downscript=no \ -device virtio-net-device,netdev=net1 \ -netdev tap,id=net2,ifname="$TAP_MGMT",script=no,downscript=no \ -device virtio-net-device,netdev=net2 \ - -device virtio-rng-pci \ - -drive "file=${OVERLAY},format=qcow2,if=virtio" \ - -drive "file=$PROJECT_DIR/cloud-init-${VM_NAME}.iso,format=raw,if=virtio" \ + -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \ + -drive "id=disk0,file=${OVERLAY},format=qcow2,if=none" -device virtio-blk-device,drive=disk0 \ -virtfs "local,path=$SHARED_DIR,mount_tag=host_share,security_model=mapped-xattr" \ -daemonize || { @@ -192,12 +173,6 @@ for SVC in $(docker compose -p "$(basename "$PROJECT_DIR")" ps --format '{{.Name docker exec "$SVC" arp -s "${SUBNET2}.254" "$FILTER2_MAC" 2>/dev/null || true done -CTRL_OVERLAY="$PROJECT_DIR/controller.qcow2" -if [ ! -f "$CTRL_OVERLAY" ]; then - qemu-img create -f qcow2 -b "$(realpath "$QEMU_IMAGE")" -F raw "$CTRL_OVERLAY" >/dev/null - gen_cloud_init_iso "controller" "controller" "${MGMT_SUBNET}.3" -fi - ip tuntap add dev "tap-ctrl" mode tap 2>/dev/null || true ip link set "tap-ctrl" master "$MGMT_BRIDGE" ip link set "tap-ctrl" up From c3e276e7a89840476efc41e9abec55051fc0bfc5 Mon Sep 17 00:00:00 2001 From: baydakov-georgiy Date: Wed, 22 Apr 2026 18:14:46 +0300 Subject: [PATCH 03/13] fix(controller): incorrect resp.Body.Close() --- controller/internal/service/service/service.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/controller/internal/service/service/service.go b/controller/internal/service/service/service.go index 303edb6..5b72c10 100644 --- a/controller/internal/service/service/service.go +++ b/controller/internal/service/service/service.go @@ -77,18 +77,19 @@ func (s *Service) Check(checkValue string, endpointName string) ([]int, error) { continue } - resp.Body.Close() - if resp.StatusCode != http.StatusOK { + resp.Body.Close() log.Printf("Provider %s returned %s", providerName, resp.Status) continue } var data map[string]interface{} if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { + resp.Body.Close() log.Printf("Error parsing response from %s: %v", providerName, err) continue } + resp.Body.Close() categoryPath := provider.Endpoints[endpointName].Categories actual, err := s.jsonParser.ExtractCategories(data, categoryPath) From b64a3a7d8b7979260636830374a73c9e6f949b15 Mon Sep 17 00:00:00 2001 From: baydakov-georgiy Date: Wed, 22 Apr 2026 18:20:31 +0300 Subject: [PATCH 04/13] chore(test_stand): add more urls for traffic-gen --- test/virtual_load_network/scripts/traffic-gen.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/virtual_load_network/scripts/traffic-gen.sh b/test/virtual_load_network/scripts/traffic-gen.sh index d8a1e54..5821ab1 100644 --- a/test/virtual_load_network/scripts/traffic-gen.sh +++ b/test/virtual_load_network/scripts/traffic-gen.sh @@ -2,7 +2,10 @@ URLS=( "http://www.google.com" - "https://github.com" + "https://youtube.com" + "https://www.tiktok.com" + "https://1xbet.com/" + "https://betboom.ru/" ) while true; do From 8fdb3ed47bc6e4f71189ee996d8e1b4b3d472326 Mon Sep 17 00:00:00 2001 From: baydakov-georgiy Date: Thu, 23 Apr 2026 21:41:14 +0300 Subject: [PATCH 05/13] feat(test_stand): add interaction with controller and worker --- test/virtual_load_network/scripts/start.sh | 23 ++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/test/virtual_load_network/scripts/start.sh b/test/virtual_load_network/scripts/start.sh index f69b96e..2c5de21 100755 --- a/test/virtual_load_network/scripts/start.sh +++ b/test/virtual_load_network/scripts/start.sh @@ -28,7 +28,9 @@ QEMU_BIOS="${QEMU_BIOS:-${YOCTO_DEPLOY_DIR}/fw_jump.elf}" QEMU_MEMORY="${QEMU_MEMORY:-4G}" QEMU_CPUS="${QEMU_CPUS:-2}" FILTER_RISCV_BIN="${FILTER_RISCV_BIN:-../../worker/main-riscv}" +WORKER_RISCV_BIN="${WORKER_RISCV_BIN:-../../worker/bazel-bin/worker}" CONTROLLER_BIN="${CONTROLLER_BIN:-../../controller/bin/grpc_server}" +VM_HUGEPAGES="${VM_HUGEPAGES:-64}" FILTER1_MAC="52:54:00:f1:00:01" FILTER2_MAC="52:54:00:f2:00:01" @@ -73,10 +75,21 @@ mkdir -p "$SHARED_DIR" if [ -f "$FILTER_RISCV_BIN" ]; then cp "$FILTER_RISCV_BIN" "$SHARED_DIR/filter" fi +if [ -f "$WORKER_RISCV_BIN" ]; then + cp "$WORKER_RISCV_BIN" "$SHARED_DIR/worker" +fi if [ -f "$CONTROLLER_BIN" ]; then cp "$CONTROLLER_BIN" "$SHARED_DIR/controller" fi +cat > "$SHARED_DIR/setup-inet.sh" << EOF +#!/bin/sh +ip addr add ${INET_SUBNET}.1/24 dev eth1 2>/dev/null || true +ip route replace default via ${INET_SUBNET}.254 dev eth1 +echo "nameserver 8.8.8.8" > /etc/resolv.conf +EOF +chmod +x "$SHARED_DIR/setup-inet.sh" + mkdir -p "$SHARED_DIR/internal/service/config" cp ../../controller/internal/service/config/categories.json "$SHARED_DIR/internal/service/config/" cp ../../controller/internal/service/config/providers.json "$SHARED_DIR/internal/service/config/" @@ -97,9 +110,11 @@ start_controller() { -pidfile "/tmp/qemu-controller.pid" \ -bios "$QEMU_BIOS" \ -kernel "$QEMU_KERNEL" \ - -append "root=/dev/vda rw earlycon=sbi console=ttyS0 ip=${MGMT_SUBNET}.3::${MGMT_SUBNET}.254:255.255.255.0::eth0:off" \ + -append "root=/dev/vda rw earlycon=sbi console=ttyS0 hugepages=${VM_HUGEPAGES} ip=${MGMT_SUBNET}.3::${MGMT_SUBNET}.254:255.255.255.0::eth0:off" \ -netdev tap,id=net0,ifname="tap-ctrl",script=no,downscript=no \ -device virtio-net-device,netdev=net0 \ + -netdev tap,id=net1,ifname="tap-ctrl-inet",script=no,downscript=no \ + -device virtio-net-device,netdev=net1 \ -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \ -drive "id=disk0,file=${CTRL_OVERLAY},format=qcow2,if=none" -device virtio-blk-device,drive=disk0 \ -virtfs "local,path=$SHARED_DIR,mount_tag=host_share,security_model=mapped-xattr" \ @@ -143,7 +158,7 @@ start_filter_vm() { -pidfile "/tmp/qemu-${VM_NAME}.pid" \ -bios "$QEMU_BIOS" \ -kernel "$QEMU_KERNEL" \ - -append "root=/dev/vda rw earlycon=sbi console=ttyS0 ip=${MGMT_IP}::${MGMT_SUBNET}.254:255.255.255.0::eth2:off" \ + -append "root=/dev/vda rw earlycon=sbi console=ttyS0 hugepages=${VM_HUGEPAGES} ip=${MGMT_IP}::${MGMT_SUBNET}.254:255.255.255.0::eth2:off" \ -netdev tap,id=net0,ifname="$TAP_IN",script=no,downscript=no \ -device virtio-net-device,netdev=net0,mac="$ETH0_MAC" \ -netdev tap,id=net1,ifname="$TAP_OUT",script=no,downscript=no \ @@ -177,6 +192,10 @@ ip tuntap add dev "tap-ctrl" mode tap 2>/dev/null || true ip link set "tap-ctrl" master "$MGMT_BRIDGE" ip link set "tap-ctrl" up +ip tuntap add dev "tap-ctrl-inet" mode tap 2>/dev/null || true +ip link set "tap-ctrl-inet" master "$INET_BRIDGE" +ip link set "tap-ctrl-inet" up + start_controller echo "Test stand is running" From 9e90f1d85ebb177996eca77efe768417804bf4a7 Mon Sep 17 00:00:00 2001 From: baydakov-georgiy Date: Thu, 23 Apr 2026 21:43:44 +0300 Subject: [PATCH 06/13] docs(test_stand): instruction for using test stand --- wiki/using_test_stand.md | 297 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100644 wiki/using_test_stand.md diff --git a/wiki/using_test_stand.md b/wiki/using_test_stand.md new file mode 100644 index 0000000..fe7220a --- /dev/null +++ b/wiki/using_test_stand.md @@ -0,0 +1,297 @@ +# Using Test Stand + +The test stand runs QEMU RISC-V virtual machines (controller + filter workers) with traffic generators in Docker containers. It allows testing the full filtering pipeline: traffic generation, DNS interception, domain classification via Kaspersky API, and policy-based blocking. + +## Prerequisites + +- `qemu-system-riscv64` +- Docker and Docker Compose +- Bazel + +## Network Layout + +Bridges: +* br-mgmt (10.0.2.0/24) - SSH access, gRPC communication +* br-inet (10.0.3.0/24) - Internet access (controller) +* br-testnet1 (10.0.0.0/24) - Traffic to filter1 +* br-testnet2 (10.0.1.0/24) - Traffic to filter2 + +VMs: +* filter1 (10.0.2.1) - DPDK packet filter and gRPC worker +* filter2 (10.0.2.2) - DPDK packet filter and gRPC worker +* controller (10.0.2.3) - gRPC server and Kaspersky API classifier + +## 0. Yocto Image + +The test stand requires a RISC-V image `cluster-image` and SDK from `vm_build_risc_v` + +**Image** +```bash +cd vm_build_risc_v +make build +make qemu +``` + +`make build` creates the Docker container with the build environment. + +`make qemu` runs the full Yocto build for `qemuriscv64` inside it. + +**SDK** +```bash +make qemu-sdk +``` + +The SDK installer script will appear in `vm_build_risc_v/qemu/poky/build/tmp/deploy/sdk/`. Install it in `/opt/riscv-sdk`: + +```bash +sudo vm_build_risc_v/qemu/poky/build/tmp/deploy/sdk/poky-glibc-x86_64-cluster-image-riscv64-qemuriscv64-toolchain-*.sh -d /opt/riscv-sdk +``` + +Edit `test/virtual_load_network/.env`: +```env +YOCTO_DEPLOY_DIR=/path/to/deploy/images/qemuriscv64 +``` + + +## 1. Build Controller + +```bash +cd controller +make build-riscv +``` + +The binary is placed at `controller/bin/grpc_server`. + +## 2. Build Worker + +```bash +cd worker +bazel build --config=riscv64 //:worker +``` + +The binary is placed at `worker/bazel-bin/worker`. + +## 3. Start Test Stand + +```bash +cd test/virtual_load_network +make run +``` + +This will: +- Start traffic generator containers +- Create network bridges and TAP interfaces +- Launch 3 QEMU VMs: `filter1`, `filter2`, and `controller` +- Copy binaries and configs to `shared/` directory + +Expected output: +``` +➜ virtual_load_network git:(test_stand) ✗ sudo scripts/start.sh +[sudo] password for lespend: +[+] Building 2.9s (13/13) FINISHED + => [internal] load local bake definitions 0.0s + => => reading from stdin 1.21kB 0.0s + => [traffic-gen-1 internal] load build definition from Dockerfile.traffic-gen 0.0s + => => transferring dockerfile: 208B 0.0s + => [traffic-gen-2 internal] load metadata for docker.io/library/alpine:3.20 1.9s + => [traffic-gen-1 internal] load .dockerignore 0.0s + => => transferring context: 2B 0.0s + => [traffic-gen-1 internal] load build context 0.0s + => => transferring context: 500B 0.0s + => [traffic-gen-2 1/4] FROM docker.io/library/alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc 0.0s + => CACHED [traffic-gen-2 2/4] RUN apk add --no-cache curl bash 0.0s + => CACHED [traffic-gen-2 3/4] COPY ./scripts/traffic-gen.sh /entrypoint.sh 0.0s + => CACHED [traffic-gen-1 4/4] RUN chmod +x /entrypoint.sh 0.0s + => [traffic-gen-2] exporting to image 0.0s + => => exporting layers 0.0s + => => writing image sha256:e99e2c8bdbe3ed141026d06d3ececcadbc769171b650ac5359ff79940eb87889 0.0s + => => naming to docker.io/library/virtual_load_network-traffic-gen-2 0.0s + => [traffic-gen-1] exporting to image 0.0s + => => exporting layers 0.0s + => => writing image sha256:8528a890010cfd50b8c21c85de488da8e22070021e37709b7a7c9926c16629b2 0.0s + => => naming to docker.io/library/virtual_load_network-traffic-gen-1 0.0s + => [traffic-gen-2] resolving provenance for metadata file 0.0s + => [traffic-gen-1] resolving provenance for metadata file 0.0s +[+] up 8/8 + ✔ Image virtual_load_network-traffic-gen-2 Built 3.0s + ✔ Image virtual_load_network-traffic-gen-1 Built 3.0s + ✔ Network virtual_load_network_testnet1 Created 0.1s + ✔ Network virtual_load_network_testnet2 Created 0.1s + ✔ Container virtual_load_network-traffic-gen-2-2 Started 0.7s + ✔ Container virtual_load_network-traffic-gen-1-2 Started 0.7s + ✔ Container virtual_load_network-traffic-gen-2-1 Started 0.5s + ✔ Container virtual_load_network-traffic-gen-1-1 Started 0.5s +10.0.0.254 dev eth0 lladdr 1e:68:2a:65:96:99 ref 1 used 0/0/0 probes 4 REACHABLE + +*** Round 1, deleting 1 entries *** +*** Flush is complete after 1 round(s) *** +10.0.0.254 dev eth0 lladdr 1e:68:2a:65:96:99 ref 1 used 0/0/0 probes 4 REACHABLE + +*** Round 1, deleting 1 entries *** +*** Flush is complete after 1 round(s) *** +10.0.1.254 dev eth0 lladdr 36:e3:51:79:b3:3c ref 1 used 0/0/0 probes 4 REACHABLE + +*** Round 1, deleting 1 entries *** +*** Flush is complete after 1 round(s) *** +10.0.1.254 dev eth0 lladdr 36:e3:51:79:b3:3c ref 1 used 0/0/0 probes 4 REACHABLE + +*** Round 1, deleting 1 entries *** +*** Flush is complete after 1 round(s) *** +Test stand is running + Filter VMs: filter1 (pid 37833), filter2 (pid 37857) + Controller: pid 38215 + Traffic generators: 4 containers +``` +``` +``` + +### 1. Start Controller + +Connect to the controller VM via SSH: + +```bash +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@10.0.2.3 +``` + +Set up internet access (required for Kaspersky API): + +```bash +sh /mnt/shared/setup-inet.sh +``` + +Start the controller: + +```bash +cd /mnt/shared +KASPERSKY_API_KEY= ./controller +KASPERSKY_API_KEY=wyo915OXTCe5stpLCtc5Ww== ./controller +``` + +### 2. Load Filtering Policy + +On the host, install dependencies and load a policy: + +```bash +cd admin +uv venv --python 3.12 .venv +source .venv/bin/activate +uv pip install -r requirements.txt +uv pip install "setuptools<75" +python -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. admin_service.proto +``` + +Edit `admin/.env` to point to the controller: + +``` +SERVER_HOST=10.0.2.3 +SERVER_PORT=50051 +``` + +Edit `config.toml` with desired rules, then load: + +```bash +python admin.py --file config.toml +``` + +Expected output: +``` +(.venv) ➜ admin git:(test_stand) ✗ python admin.py --file config.toml +Config loaded +``` +``` +``` + +### 3. Start Worker + +Connect to the filter VM via SSH: + +```bash +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@10.0.2.1 # filter1 +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@10.0.2.2 # filter2 +``` + +Start the worker: + +```bash +cd /mnt/shared +WORKER_ID=1 METRICS_GATEWAY_ADDRESS=10.0.2.254 METRICS_GATEWAY_PORT=9091 CONTROLLER_GRPC_ADDR=10.0.2.3:50051 DPDK_PORT_IN=eth0 DPDK_PORT_OUT=eth1 ./worker +``` + +For `filter2`, use `WORKER_ID=2`. + +To redirect logs to a shared file (viewable from host): + +```bash +WORKER_ID=1 METRICS_GATEWAY_ADDRESS=10.0.2.254 METRICS_GATEWAY_PORT=9091 CONTROLLER_GRPC_ADDR=10.0.2.3:50051 DPDK_PORT_IN=eth0 DPDK_PORT_OUT=eth1 ./worker > /mnt/shared/filter1.log 2>&1 & +``` + +Expected output: +``` +root@qemuriscv64:~# cd /mnt/shared +root@qemuriscv64:/mnt/shared# WORKER_ID=1 METRICS_GATEWAY_ADDRESS=10.0.2.254 METRICS_GATEWAY_PORT=9091 CONTROLLER_GRPC_ADDR=10.0.2.3:50051 DPDK_PORT_IN=eth0 +DPDK_PORT_OUT=eth1 ./worker +[2026-04-23 16:46:28.534] [info] Initialize MetricsCollector with 10.0.2.254:9091 +[2026-04-23 16:46:28.823] [info] gRPC channel created to 10.0.2.3:50051 +[2026-04-23 16:46:28.826] [info] Signal handlers registered +[2026-04-23 16:46:28.828] [info] Worker 1 requests policy +[2026-04-23 16:46:29.321] [info] Policy received +EAL: Detected CPU lcores: 2 +EAL: Detected NUMA nodes: 1 +EAL: Detected shared linkage of DPDK +EAL: Multi-process socket /var/run/dpdk/rte/mp_socket +EAL: Selected IOVA mode 'PA' +EAL: TSC using RISC-V rdtime. +TELEMETRY: No legacy callbacks, legacy socket not created +libbpf: Attribute of type 0x2a found multiple times in message, previous attribute is being ignored. +libbpf: Attribute of type 0x2a found multiple times in message, previous attribute is being ignored. +libbpf: elf: skipping unrecognized data section(8) .xdp_run_config +libbpf: elf: skipping unrecognized data section(9) xdp_metadata +libbpf: Attribute of type 0x2a found multiple times in message, previous attribute is being ignored. +libbpf: Attribute of type 0x2a found multiple times in message, previous attribute is being ignored. +libbpf: elf: skipping unrecognized data section(7) xdp_metadata +libbpf: elf: skipping unrecognized data section(7) xdp_metadata +libxdp: No bpffs found at /sys/fs/bpf +libxdp: Can't use dispatcher without a working bpffs +libxdp: Falling back to loading single prog without dispatcher +libbpf: elf: skipping unrecognized data section(7) xdp_metadata +Port 0 initialized +libbpf: Attribute of type 0x2a found multiple times in message, previous attribute is being ignored. +libbpf: Attribute of type 0x2a found multiple times in message, previous attribute is being ignored. +libbpf: elf: skipping unrecognized data section(8) .xdp_run_config +libbpf: elf: skipping unrecognized data section(9) xdp_metadata +libbpf: Attribute of type 0x2a found multiple times in message, previous attribute is being ignored. +libbpf: Attribute of type 0x2a found multiple times in message, previous attribute is being ignored. +libbpf: elf: skipping unrecognized data section(7) xdp_metadata +libbpf: elf: skipping unrecognized data section(7) xdp_metadata +libxdp: No bpffs found at /sys/fs/bpf +libxdp: Can't use dispatcher without a working bpffs +libxdp: Falling back to loading single prog without dispatcher +libbpf: elf: skipping unrecognized data section(7) xdp_metadata +Port 1 initialized +Port 2 initialized +Port 0 started +Port 1 started +Port 2 started +[INFO] Loaded 0 records from SQLite, 0 expired skipped +[2026-04-23 16:46:30.684] [info] DPDK initialized: in_port=0, out_port=1 +[2026-04-23 16:46:31.204] [info] Worker 1 classifying domain 'betboom.ru' +[2026-04-23 16:46:33.307] [info] Domain 'betboom.ru' classified as category 'Gambling' with trust level 0 +This site has a locked category[INFO] Packet without dns request +``` +``` +``` + + +## 5. Monitoring +```sh +sudo tcpdump -i tap-f1-out port 53 # output packets +sudo tcpdump -i tap-f1-in port 53 # input packets +``` + +## 4. Stop Test Stand + +```bash +cd test/virtual_load_network +make stop +``` + From d4a83e74f179e2954f579b68a7ffea2ad0275c30 Mon Sep 17 00:00:00 2001 From: baydakov-georgiy Date: Thu, 30 Apr 2026 16:14:57 +0300 Subject: [PATCH 07/13] feat(test_stand): add autostart in vm for controller and worker --- test/virtual_load_network/scripts/start.sh | 32 ++++++++++++++-------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/test/virtual_load_network/scripts/start.sh b/test/virtual_load_network/scripts/start.sh index 2c5de21..5084709 100755 --- a/test/virtual_load_network/scripts/start.sh +++ b/test/virtual_load_network/scripts/start.sh @@ -31,8 +31,12 @@ FILTER_RISCV_BIN="${FILTER_RISCV_BIN:-../../worker/main-riscv}" WORKER_RISCV_BIN="${WORKER_RISCV_BIN:-../../worker/bazel-bin/worker}" CONTROLLER_BIN="${CONTROLLER_BIN:-../../controller/bin/grpc_server}" VM_HUGEPAGES="${VM_HUGEPAGES:-64}" -FILTER1_MAC="52:54:00:f1:00:01" -FILTER2_MAC="52:54:00:f2:00:01" +FILTER1_MAC_IN="52:54:00:f1:00:01" +FILTER1_MAC_OUT="52:54:00:f1:00:02" +FILTER1_MAC_MGMT="52:54:00:f1:00:03" +FILTER2_MAC_IN="52:54:00:f2:00:01" +FILTER2_MAC_OUT="52:54:00:f2:00:02" +FILTER2_MAC_MGMT="52:54:00:f2:00:03" "$SCRIPT_DIR/stop.sh" 2>/dev/null || true @@ -71,6 +75,7 @@ iptables -I FORWARD -o "$MGMT_BRIDGE" -j ACCEPT SHARED_DIR="$PROJECT_DIR/shared" mkdir -p "$SHARED_DIR" +mkdir -p "$SHARED_DIR/logs" if [ -f "$FILTER_RISCV_BIN" ]; then cp "$FILTER_RISCV_BIN" "$SHARED_DIR/filter" @@ -110,7 +115,7 @@ start_controller() { -pidfile "/tmp/qemu-controller.pid" \ -bios "$QEMU_BIOS" \ -kernel "$QEMU_KERNEL" \ - -append "root=/dev/vda rw earlycon=sbi console=ttyS0 hugepages=${VM_HUGEPAGES} ip=${MGMT_SUBNET}.3::${MGMT_SUBNET}.254:255.255.255.0::eth0:off" \ + -append "root=/dev/vda rw earlycon=sbi console=ttyS0 hugepages=${VM_HUGEPAGES} role=controller ip=${MGMT_SUBNET}.3::${MGMT_SUBNET}.254:255.255.255.0::eth0:off" \ -netdev tap,id=net0,ifname="tap-ctrl",script=no,downscript=no \ -device virtio-net-device,netdev=net0 \ -netdev tap,id=net1,ifname="tap-ctrl-inet",script=no,downscript=no \ @@ -132,7 +137,10 @@ start_filter_vm() { local TAP_MGMT="$4" local BRIDGE_IN="$5" local MGMT_IP="$6" - local ETH0_MAC="$7" + local MAC_IN="$7" + local MAC_OUT="$8" + local MAC_MGMT="$9" + local WORKER_ID="${10}" local OVERLAY="$PROJECT_DIR/${VM_NAME}.qcow2" if [ ! -f "$OVERLAY" ]; then @@ -158,13 +166,13 @@ start_filter_vm() { -pidfile "/tmp/qemu-${VM_NAME}.pid" \ -bios "$QEMU_BIOS" \ -kernel "$QEMU_KERNEL" \ - -append "root=/dev/vda rw earlycon=sbi console=ttyS0 hugepages=${VM_HUGEPAGES} ip=${MGMT_IP}::${MGMT_SUBNET}.254:255.255.255.0::eth2:off" \ + -append "root=/dev/vda rw earlycon=sbi console=ttyS0 hugepages=${VM_HUGEPAGES} role=worker worker_id=${WORKER_ID} ip=${MGMT_IP}::${MGMT_SUBNET}.254:255.255.255.0::eth2:off" \ -netdev tap,id=net0,ifname="$TAP_IN",script=no,downscript=no \ - -device virtio-net-device,netdev=net0,mac="$ETH0_MAC" \ + -device virtio-net-device,netdev=net0,mac="$MAC_IN" \ -netdev tap,id=net1,ifname="$TAP_OUT",script=no,downscript=no \ - -device virtio-net-device,netdev=net1 \ + -device virtio-net-device,netdev=net1,mac="$MAC_OUT" \ -netdev tap,id=net2,ifname="$TAP_MGMT",script=no,downscript=no \ - -device virtio-net-device,netdev=net2 \ + -device virtio-net-device,netdev=net2,mac="$MAC_MGMT" \ -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \ -drive "id=disk0,file=${OVERLAY},format=qcow2,if=none" -device virtio-blk-device,drive=disk0 \ -virtfs "local,path=$SHARED_DIR,mount_tag=host_share,security_model=mapped-xattr" \ @@ -175,17 +183,17 @@ start_filter_vm() { } } -start_filter_vm "filter1" "tap-f1-in" "tap-f1-out" "tap-f1-mgmt" "$BRIDGE1" "${MGMT_SUBNET}.1" "$FILTER1_MAC" -start_filter_vm "filter2" "tap-f2-in" "tap-f2-out" "tap-f2-mgmt" "$BRIDGE2" "${MGMT_SUBNET}.2" "$FILTER2_MAC" +start_filter_vm "filter1" "tap-f1-in" "tap-f1-out" "tap-f1-mgmt" "$BRIDGE1" "${MGMT_SUBNET}.1" "$FILTER1_MAC_IN" "$FILTER1_MAC_OUT" "$FILTER1_MAC_MGMT" 1 +start_filter_vm "filter2" "tap-f2-in" "tap-f2-out" "tap-f2-mgmt" "$BRIDGE2" "${MGMT_SUBNET}.2" "$FILTER2_MAC_IN" "$FILTER2_MAC_OUT" "$FILTER2_MAC_MGMT" 2 for SVC in $(docker compose ps -q 2>/dev/null); do docker exec "$SVC" ip neigh flush all 2>/dev/null || true done for SVC in $(docker compose -p "$(basename "$PROJECT_DIR")" ps --format '{{.Name}}' 2>/dev/null | grep "gen-1"); do - docker exec "$SVC" arp -s "${SUBNET1}.254" "$FILTER1_MAC" 2>/dev/null || true + docker exec "$SVC" arp -s "${SUBNET1}.254" "$FILTER1_MAC_IN" 2>/dev/null || true done for SVC in $(docker compose -p "$(basename "$PROJECT_DIR")" ps --format '{{.Name}}' 2>/dev/null | grep "gen-2"); do - docker exec "$SVC" arp -s "${SUBNET2}.254" "$FILTER2_MAC" 2>/dev/null || true + docker exec "$SVC" arp -s "${SUBNET2}.254" "$FILTER2_MAC_IN" 2>/dev/null || true done ip tuntap add dev "tap-ctrl" mode tap 2>/dev/null || true From 75c99dbf5ec6402a415d39ead69b748a0d3d3130 Mon Sep 17 00:00:00 2001 From: baydakov-georgiy Date: Fri, 15 May 2026 16:01:40 +0300 Subject: [PATCH 08/13] feat(test_stand): add IPv6 support --- .../Dockerfile.traffic-gen | 2 +- test/virtual_load_network/docker-compose.yml | 8 ++++++ test/virtual_load_network/scripts/start.sh | 25 ++++++++++++++++++- test/virtual_load_network/scripts/stop.sh | 7 +++++- 4 files changed, 39 insertions(+), 3 deletions(-) diff --git a/test/virtual_load_network/Dockerfile.traffic-gen b/test/virtual_load_network/Dockerfile.traffic-gen index a6e9a82..a4daa1c 100644 --- a/test/virtual_load_network/Dockerfile.traffic-gen +++ b/test/virtual_load_network/Dockerfile.traffic-gen @@ -1,6 +1,6 @@ FROM alpine:3.20 -RUN apk add --no-cache curl bash +RUN apk add --no-cache curl bash iproute2 COPY ./scripts/traffic-gen.sh /entrypoint.sh RUN chmod +x /entrypoint.sh diff --git a/test/virtual_load_network/docker-compose.yml b/test/virtual_load_network/docker-compose.yml index 59914b4..bc92d5d 100644 --- a/test/virtual_load_network/docker-compose.yml +++ b/test/virtual_load_network/docker-compose.yml @@ -9,6 +9,7 @@ services: - NET_ADMIN dns: - ${DNS:-8.8.8.8} + - ${DNS_V6:-2001:4860:4860::8888} deploy: replicas: ${NUM_HOSTS_NET1:-2} restart: unless-stopped @@ -23,6 +24,7 @@ services: - NET_ADMIN dns: - ${DNS:-8.8.8.8} + - ${DNS_V6:-2001:4860:4860::8888} deploy: replicas: ${NUM_HOSTS_NET2:-2} restart: unless-stopped @@ -30,6 +32,7 @@ services: networks: testnet1: driver: bridge + enable_ipv6: true driver_opts: com.docker.network.bridge.name: "br-testnet1" com.docker.network.bridge.enable_ip_masquerade: "false" @@ -37,9 +40,12 @@ networks: config: - subnet: ${SUBNET1:-10.0.0}.0/24 gateway: ${SUBNET1:-10.0.0}.254 + - subnet: ${SUBNET1_V6:-fd00:1::}0/64 + gateway: ${SUBNET1_V6:-fd00:1::}ff testnet2: driver: bridge + enable_ipv6: true driver_opts: com.docker.network.bridge.name: "br-testnet2" com.docker.network.bridge.enable_ip_masquerade: "false" @@ -47,3 +53,5 @@ networks: config: - subnet: ${SUBNET2:-10.0.1}.0/24 gateway: ${SUBNET2:-10.0.1}.254 + - subnet: ${SUBNET2_V6:-fd00:2::}0/64 + gateway: ${SUBNET2_V6:-fd00:2::}ff diff --git a/test/virtual_load_network/scripts/start.sh b/test/virtual_load_network/scripts/start.sh index 5084709..33c9af3 100755 --- a/test/virtual_load_network/scripts/start.sh +++ b/test/virtual_load_network/scripts/start.sh @@ -21,6 +21,12 @@ INET_BRIDGE="${INET_BRIDGE:-br-inet}" MGMT_SUBNET="${MGMT_SUBNET:-10.0.2}" MGMT_BRIDGE="${MGMT_BRIDGE:-br-mgmt}" HUGEPAGES="${HUGEPAGES:-1024}" + +SUBNET1_V6="${SUBNET1_V6:-fd00:1::}" +SUBNET2_V6="${SUBNET2_V6:-fd00:2::}" +INET_SUBNET_V6="${INET_SUBNET_V6:-fd00:b::}" +MGMT_SUBNET_V6="${MGMT_SUBNET_V6:-fd00:a::}" + YOCTO_DEPLOY_DIR="${YOCTO_DEPLOY_DIR:-/home/lespend/program/yadro/vm_build_risc_v/qemu/poky/build/tmp/deploy/images/qemuriscv64}" QEMU_ROOTFS="${QEMU_ROOTFS:-${YOCTO_DEPLOY_DIR}/cluster-image-qemuriscv64.rootfs.ext4}" QEMU_KERNEL="${QEMU_KERNEL:-${YOCTO_DEPLOY_DIR}/Image}" @@ -29,7 +35,7 @@ QEMU_MEMORY="${QEMU_MEMORY:-4G}" QEMU_CPUS="${QEMU_CPUS:-2}" FILTER_RISCV_BIN="${FILTER_RISCV_BIN:-../../worker/main-riscv}" WORKER_RISCV_BIN="${WORKER_RISCV_BIN:-../../worker/bazel-bin/worker}" -CONTROLLER_BIN="${CONTROLLER_BIN:-../../controller/bin/grpc_server}" +CONTROLLER_BIN="${CONTROLLER_BIN:-$(readlink -f ../../controller/bazel-bin/cmd/grpc_server/grpc_server_/grpc_server 2>/dev/null)}" VM_HUGEPAGES="${VM_HUGEPAGES:-64}" FILTER1_MAC_IN="52:54:00:f1:00:01" FILTER1_MAC_OUT="52:54:00:f1:00:02" @@ -57,18 +63,25 @@ for BRIDGE in "$BRIDGE1" "$BRIDGE2"; do done sysctl -w net.ipv4.ip_forward=1 >/dev/null +sysctl -w net.ipv6.conf.all.forwarding=1 >/dev/null ip link add "$INET_BRIDGE" type bridge 2>/dev/null || true ip link set "$INET_BRIDGE" up ip addr add "${INET_SUBNET}.254/24" dev "$INET_BRIDGE" 2>/dev/null || true +ip -6 addr add "${INET_SUBNET_V6}fe/64" dev "$INET_BRIDGE" 2>/dev/null || true iptables -t nat -A POSTROUTING -s ${INET_SUBNET}.0/24 ! -d ${INET_SUBNET}.0/24 -j MASQUERADE iptables -I FORWARD -i "$INET_BRIDGE" -j ACCEPT iptables -I FORWARD -o "$INET_BRIDGE" -j ACCEPT +ip6tables -t nat -A POSTROUTING -s ${INET_SUBNET_V6}/64 ! -d ${INET_SUBNET_V6}/64 -j MASQUERADE +ip6tables -I FORWARD -i "$INET_BRIDGE" -j ACCEPT +ip6tables -I FORWARD -o "$INET_BRIDGE" -j ACCEPT + ip link add "$MGMT_BRIDGE" type bridge 2>/dev/null || true ip link set "$MGMT_BRIDGE" up ip addr add "${MGMT_SUBNET}.254/24" dev "$MGMT_BRIDGE" 2>/dev/null || true +ip -6 addr add "${MGMT_SUBNET_V6}fe/64" dev "$MGMT_BRIDGE" 2>/dev/null || true iptables -I FORWARD -i "$MGMT_BRIDGE" -j ACCEPT iptables -I FORWARD -o "$MGMT_BRIDGE" -j ACCEPT @@ -90,8 +103,11 @@ fi cat > "$SHARED_DIR/setup-inet.sh" << EOF #!/bin/sh ip addr add ${INET_SUBNET}.1/24 dev eth1 2>/dev/null || true +ip -6 addr add ${INET_SUBNET_V6}1/64 dev eth1 2>/dev/null || true ip route replace default via ${INET_SUBNET}.254 dev eth1 +ip -6 route replace default via ${INET_SUBNET_V6}fe dev eth1 2>/dev/null || true echo "nameserver 8.8.8.8" > /etc/resolv.conf +echo "nameserver 2001:4860:4860::8888" >> /etc/resolv.conf EOF chmod +x "$SHARED_DIR/setup-inet.sh" @@ -188,12 +204,19 @@ start_filter_vm "filter2" "tap-f2-in" "tap-f2-out" "tap-f2-mgmt" "$BRIDGE2" "${M for SVC in $(docker compose ps -q 2>/dev/null); do docker exec "$SVC" ip neigh flush all 2>/dev/null || true + docker exec "$SVC" ip -6 neigh flush all 2>/dev/null || true done for SVC in $(docker compose -p "$(basename "$PROJECT_DIR")" ps --format '{{.Name}}' 2>/dev/null | grep "gen-1"); do docker exec "$SVC" arp -s "${SUBNET1}.254" "$FILTER1_MAC_IN" 2>/dev/null || true + docker exec "$SVC" ip -6 neigh del "${SUBNET1_V6}fe" dev eth0 2>/dev/null || true + docker exec "$SVC" ip -6 neigh add "${SUBNET1_V6}fe" lladdr "$FILTER1_MAC_IN" nud permanent dev eth0 2>/dev/null || true + docker exec "$SVC" ip -6 route replace default via "${SUBNET1_V6}fe" dev eth0 2>/dev/null || true done for SVC in $(docker compose -p "$(basename "$PROJECT_DIR")" ps --format '{{.Name}}' 2>/dev/null | grep "gen-2"); do docker exec "$SVC" arp -s "${SUBNET2}.254" "$FILTER2_MAC_IN" 2>/dev/null || true + docker exec "$SVC" ip -6 neigh del "${SUBNET2_V6}fe" dev eth0 2>/dev/null || true + docker exec "$SVC" ip -6 neigh add "${SUBNET2_V6}fe" lladdr "$FILTER2_MAC_IN" nud permanent dev eth0 2>/dev/null || true + docker exec "$SVC" ip -6 route replace default via "${SUBNET2_V6}fe" dev eth0 2>/dev/null || true done ip tuntap add dev "tap-ctrl" mode tap 2>/dev/null || true diff --git a/test/virtual_load_network/scripts/stop.sh b/test/virtual_load_network/scripts/stop.sh index b022e66..a8712b5 100755 --- a/test/virtual_load_network/scripts/stop.sh +++ b/test/virtual_load_network/scripts/stop.sh @@ -14,6 +14,7 @@ fi INET_SUBNET="${INET_SUBNET:-10.0.3}" INET_BRIDGE="${INET_BRIDGE:-br-inet}" MGMT_BRIDGE="${MGMT_BRIDGE:-br-mgmt}" +INET_SUBNET_V6="${INET_SUBNET_V6:-fd00:b::}" for VM in filter1 filter2 controller; do PID_FILE="/tmp/qemu-${VM}.pid" @@ -29,7 +30,11 @@ iptables -t nat -D POSTROUTING -s ${INET_SUBNET}.0/24 ! -d ${INET_SUBNET}.0/24 - iptables -D FORWARD -i "$INET_BRIDGE" -j ACCEPT 2>/dev/null iptables -D FORWARD -o "$INET_BRIDGE" -j ACCEPT 2>/dev/null -for TAP in tap-f1-in tap-f1-out tap-f2-in tap-f2-out tap-f1-mgmt tap-f2-mgmt tap-ctrl; do +ip6tables -t nat -D POSTROUTING -s ${INET_SUBNET_V6}/64 ! -d ${INET_SUBNET_V6}/64 -j MASQUERADE 2>/dev/null +ip6tables -D FORWARD -i "$INET_BRIDGE" -j ACCEPT 2>/dev/null +ip6tables -D FORWARD -o "$INET_BRIDGE" -j ACCEPT 2>/dev/null + +for TAP in tap-f1-in tap-f1-out tap-f2-in tap-f2-out tap-f1-mgmt tap-f2-mgmt tap-ctrl tap-ctrl-inet; do ip tuntap del dev "$TAP" mode tap 2>/dev/null done From 383705001e9d9949060311d9cfce48948ca3d838 Mon Sep 17 00:00:00 2001 From: baydakov-georgiy Date: Thu, 21 May 2026 19:18:00 +0300 Subject: [PATCH 09/13] feat(test_stand): add grafana and prometheus for metrics --- test/virtual_load_network/.env | 8 +- .../configs/controller.env | 1 + test/virtual_load_network/configs/worker.env | 5 + test/virtual_load_network/docker-compose.yml | 33 ++++++ .../provisioning/dashboards/cluster.json | 105 ++++++++++++++++++ .../provisioning/dashboards/dashboards.yml | 8 ++ .../provisioning/datasources/prometheus.yml | 8 ++ .../monitoring/prometheus.yml | 9 ++ .../scripts/traffic-gen.sh | 2 +- 9 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 test/virtual_load_network/configs/controller.env create mode 100644 test/virtual_load_network/configs/worker.env create mode 100644 test/virtual_load_network/monitoring/grafana/provisioning/dashboards/cluster.json create mode 100644 test/virtual_load_network/monitoring/grafana/provisioning/dashboards/dashboards.yml create mode 100644 test/virtual_load_network/monitoring/grafana/provisioning/datasources/prometheus.yml create mode 100644 test/virtual_load_network/monitoring/prometheus.yml diff --git a/test/virtual_load_network/.env b/test/virtual_load_network/.env index 17bc607..87fa869 100644 --- a/test/virtual_load_network/.env +++ b/test/virtual_load_network/.env @@ -9,11 +9,17 @@ BRIDGE1=br-testnet1 BRIDGE2=br-testnet2 DNS=8.8.8.8 +SUBNET1_V6=fd00:1:: +SUBNET2_V6=fd00:2:: +DNS_V6=2001:4860:4860::8888 + INET_SUBNET=10.0.3 INET_BRIDGE=br-inet +INET_SUBNET_V6=fd00:b:: MGMT_SUBNET=10.0.2 MGMT_BRIDGE=br-mgmt +MGMT_SUBNET_V6=fd00:a:: YOCTO_DEPLOY_DIR=/home/lespend/program/yadro/vm_build_risc_v/qemu/poky/build/tmp/deploy/images/qemuriscv64 QEMU_ROOTFS=${YOCTO_DEPLOY_DIR}/cluster-image-qemuriscv64.rootfs.ext4 @@ -23,6 +29,6 @@ QEMU_MEMORY=4G QEMU_CPUS=2 FILTER_RISCV_BIN=../../worker/main-riscv -CONTROLLER_BIN=../../controller/bin/grpc_server +CONTROLLER_BIN=../../controller/bazel-bin/cmd/grpc_server/grpc_server_/grpc_server HUGEPAGES=1024 diff --git a/test/virtual_load_network/configs/controller.env b/test/virtual_load_network/configs/controller.env new file mode 100644 index 0000000..07ee673 --- /dev/null +++ b/test/virtual_load_network/configs/controller.env @@ -0,0 +1 @@ +KASPERSKY_API_KEY=wyo915OXTCe5stpLCtc5Ww== diff --git a/test/virtual_load_network/configs/worker.env b/test/virtual_load_network/configs/worker.env new file mode 100644 index 0000000..2dff044 --- /dev/null +++ b/test/virtual_load_network/configs/worker.env @@ -0,0 +1,5 @@ +CONTROLLER_GRPC_ADDR=10.0.2.3:50051 +METRICS_GATEWAY_ADDRESS=10.0.2.254 +METRICS_GATEWAY_PORT=9091 +DPDK_PORT_IN=eth0 +DPDK_PORT_OUT=eth1 diff --git a/test/virtual_load_network/docker-compose.yml b/test/virtual_load_network/docker-compose.yml index bc92d5d..f1a8fff 100644 --- a/test/virtual_load_network/docker-compose.yml +++ b/test/virtual_load_network/docker-compose.yml @@ -29,6 +29,36 @@ services: replicas: ${NUM_HOSTS_NET2:-2} restart: unless-stopped + pushgateway: + image: prom/pushgateway:latest + ports: + - "9091:9091" + restart: unless-stopped + + prometheus: + image: prom/prometheus:latest + volumes: + - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro + ports: + - "9090:9090" + depends_on: + - pushgateway + restart: unless-stopped + + grafana: + image: grafana/grafana:latest + volumes: + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - grafana-data:/var/lib/grafana + ports: + - "3000:3000" + environment: + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin" + depends_on: + - prometheus + restart: unless-stopped + networks: testnet1: driver: bridge @@ -55,3 +85,6 @@ networks: gateway: ${SUBNET2:-10.0.1}.254 - subnet: ${SUBNET2_V6:-fd00:2::}0/64 gateway: ${SUBNET2_V6:-fd00:2::}ff + +volumes: + grafana-data: diff --git a/test/virtual_load_network/monitoring/grafana/provisioning/dashboards/cluster.json b/test/virtual_load_network/monitoring/grafana/provisioning/dashboards/cluster.json new file mode 100644 index 0000000..8c31c54 --- /dev/null +++ b/test/virtual_load_network/monitoring/grafana/provisioning/dashboards/cluster.json @@ -0,0 +1,105 @@ +{ + "title": "Cluster Workers", + "uid": "cluster-workers", + "panels": [ + { + "title": "Packets Received", + "type": "timeseries", + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 0}, + "targets": [ + { + "expr": "rate(packets_received_total[1m])", + "legendFormat": "{{job}}" + } + ], + "datasource": "Prometheus" + }, + { + "title": "Packets Passed", + "type": "timeseries", + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 0}, + "targets": [ + { + "expr": "rate(packets_passed_total[1m])", + "legendFormat": "{{job}}" + } + ], + "datasource": "Prometheus" + }, + { + "title": "Packets Dropped", + "type": "timeseries", + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 0}, + "targets": [ + { + "expr": "rate(packets_dropped_total[1m])", + "legendFormat": "{{job}}" + } + ], + "datasource": "Prometheus" + }, + { + "title": "Blocked Domains", + "type": "timeseries", + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 8}, + "targets": [ + { + "expr": "rate(blocked_domains_total[1m])", + "legendFormat": "{{job}}" + } + ], + "datasource": "Prometheus" + }, + { + "title": "CPU Usage %", + "type": "timeseries", + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 8}, + "targets": [ + { + "expr": "cpu_usage{cpu='cpu'}", + "legendFormat": "{{job}}" + } + ], + "datasource": "Prometheus" + }, + { + "title": "Memory Used (MB)", + "type": "timeseries", + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 8}, + "targets": [ + { + "expr": "memory_used / 1024 / 1024", + "legendFormat": "{{job}}" + } + ], + "datasource": "Prometheus" + }, + { + "title": "Task Duration (s)", + "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 16}, + "targets": [ + { + "expr": "task_duration_seconds", + "legendFormat": "{{job}}" + } + ], + "datasource": "Prometheus" + }, + { + "title": "Push Errors", + "type": "stat", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 16}, + "targets": [ + { + "expr": "push_errors_total", + "legendFormat": "{{job}}" + } + ], + "datasource": "Prometheus" + } + ], + "time": {"from": "now-15m", "to": "now"}, + "refresh": "5s", + "schemaVersion": 39 +} diff --git a/test/virtual_load_network/monitoring/grafana/provisioning/dashboards/dashboards.yml b/test/virtual_load_network/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..24910df --- /dev/null +++ b/test/virtual_load_network/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,8 @@ +apiVersion: 1 + +providers: + - name: default + type: file + options: + path: /etc/grafana/provisioning/dashboards + foldersFromFilesStructure: false diff --git a/test/virtual_load_network/monitoring/grafana/provisioning/datasources/prometheus.yml b/test/virtual_load_network/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..86fd346 --- /dev/null +++ b/test/virtual_load_network/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,8 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true diff --git a/test/virtual_load_network/monitoring/prometheus.yml b/test/virtual_load_network/monitoring/prometheus.yml new file mode 100644 index 0000000..c7dae34 --- /dev/null +++ b/test/virtual_load_network/monitoring/prometheus.yml @@ -0,0 +1,9 @@ +global: + scrape_interval: 5s + +scrape_configs: + - job_name: pushgateway + honor_labels: true + static_configs: + - targets: + - pushgateway:9091 diff --git a/test/virtual_load_network/scripts/traffic-gen.sh b/test/virtual_load_network/scripts/traffic-gen.sh index 5821ab1..0e61af0 100644 --- a/test/virtual_load_network/scripts/traffic-gen.sh +++ b/test/virtual_load_network/scripts/traffic-gen.sh @@ -11,7 +11,7 @@ URLS=( while true; do url=${URLS[$RANDOM % ${#URLS[@]}]} echo "[$(date '+%H:%M:%S')] -> request $url" - status=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "$url") + #status=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "$url") echo "[$(date '+%H:%M:%S')] <- response $url, status = $status" sleep 5 done From 2852c755c42100c95f8a0a6cf2d55be29920da5b Mon Sep 17 00:00:00 2001 From: LapshinAE0 Date: Wed, 27 May 2026 23:40:33 +0300 Subject: [PATCH 10/13] docs: update wiki-using_test_stand --- wiki/using_test_stand.md | 321 ++++++++++++++++++++------------------- 1 file changed, 164 insertions(+), 157 deletions(-) diff --git a/wiki/using_test_stand.md b/wiki/using_test_stand.md index fe7220a..ae4ba80 100644 --- a/wiki/using_test_stand.md +++ b/wiki/using_test_stand.md @@ -64,18 +64,31 @@ The binary is placed at `controller/bin/grpc_server`. ## 2. Build Worker +standard assembly: ```bash cd worker bazel build --config=riscv64 //:worker ``` +build with debug output: +```bash +cd worker +bazel build --config=riscv64 //:worker --copt="-DDEBUG=1" +``` + +If you experience a hang during assembly, limit the memory with a flag, for example: +```bash +cd worker +bazel build --config=riscv64 //:worker --copt="-DDEBUG=1" --local_ram_resources=4096 +``` + The binary is placed at `worker/bazel-bin/worker`. ## 3. Start Test Stand ```bash -cd test/virtual_load_network -make run +cd test/virtual_load_network/scripts +sudo ./start.sh ``` This will: @@ -86,212 +99,206 @@ This will: Expected output: ``` -➜ virtual_load_network git:(test_stand) ✗ sudo scripts/start.sh -[sudo] password for lespend: -[+] Building 2.9s (13/13) FINISHED - => [internal] load local bake definitions 0.0s - => => reading from stdin 1.21kB 0.0s - => [traffic-gen-1 internal] load build definition from Dockerfile.traffic-gen 0.0s - => => transferring dockerfile: 208B 0.0s - => [traffic-gen-2 internal] load metadata for docker.io/library/alpine:3.20 1.9s - => [traffic-gen-1 internal] load .dockerignore 0.0s - => => transferring context: 2B 0.0s - => [traffic-gen-1 internal] load build context 0.0s - => => transferring context: 500B 0.0s - => [traffic-gen-2 1/4] FROM docker.io/library/alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc 0.0s - => CACHED [traffic-gen-2 2/4] RUN apk add --no-cache curl bash 0.0s - => CACHED [traffic-gen-2 3/4] COPY ./scripts/traffic-gen.sh /entrypoint.sh 0.0s - => CACHED [traffic-gen-1 4/4] RUN chmod +x /entrypoint.sh 0.0s - => [traffic-gen-2] exporting to image 0.0s - => => exporting layers 0.0s - => => writing image sha256:e99e2c8bdbe3ed141026d06d3ececcadbc769171b650ac5359ff79940eb87889 0.0s - => => naming to docker.io/library/virtual_load_network-traffic-gen-2 0.0s - => [traffic-gen-1] exporting to image 0.0s - => => exporting layers 0.0s - => => writing image sha256:8528a890010cfd50b8c21c85de488da8e22070021e37709b7a7c9926c16629b2 0.0s - => => naming to docker.io/library/virtual_load_network-traffic-gen-1 0.0s - => [traffic-gen-2] resolving provenance for metadata file 0.0s - => [traffic-gen-1] resolving provenance for metadata file 0.0s -[+] up 8/8 - ✔ Image virtual_load_network-traffic-gen-2 Built 3.0s - ✔ Image virtual_load_network-traffic-gen-1 Built 3.0s - ✔ Network virtual_load_network_testnet1 Created 0.1s - ✔ Network virtual_load_network_testnet2 Created 0.1s - ✔ Container virtual_load_network-traffic-gen-2-2 Started 0.7s - ✔ Container virtual_load_network-traffic-gen-1-2 Started 0.7s - ✔ Container virtual_load_network-traffic-gen-2-1 Started 0.5s - ✔ Container virtual_load_network-traffic-gen-1-1 Started 0.5s -10.0.0.254 dev eth0 lladdr 1e:68:2a:65:96:99 ref 1 used 0/0/0 probes 4 REACHABLE - -*** Round 1, deleting 1 entries *** -*** Flush is complete after 1 round(s) *** -10.0.0.254 dev eth0 lladdr 1e:68:2a:65:96:99 ref 1 used 0/0/0 probes 4 REACHABLE - -*** Round 1, deleting 1 entries *** -*** Flush is complete after 1 round(s) *** -10.0.1.254 dev eth0 lladdr 36:e3:51:79:b3:3c ref 1 used 0/0/0 probes 4 REACHABLE - -*** Round 1, deleting 1 entries *** -*** Flush is complete after 1 round(s) *** -10.0.1.254 dev eth0 lladdr 36:e3:51:79:b3:3c ref 1 used 0/0/0 probes 4 REACHABLE - -*** Round 1, deleting 1 entries *** -*** Flush is complete after 1 round(s) *** + => [traffic-gen-1 internal] load build definition from Dockerfile.traffic-gen 0.0s + => => transferring dockerfile: 217B 0.0s + => [traffic-gen-2 internal] load build definition from Dockerfile.traffic-gen 0.0s + => => transferring dockerfile: 217B 0.0s + => [traffic-gen-1 internal] load metadata for docker.io/library/alpine:3.20 0.7s + => [traffic-gen-2 internal] load .dockerignore 0.0s + => => transferring context: 2B 0.0s + => [traffic-gen-1 internal] load .dockerignore 0.0s + => => transferring context: 2B 0.0s + => [traffic-gen-2 1/4] FROM docker.io/library/alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc 0.0s + => => resolve docker.io/library/alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc 0.0s + => [traffic-gen-2 internal] load build context 0.0s + => => transferring context: 71B 0.0s + => [traffic-gen-1 internal] load build context 0.0s + => => transferring context: 71B 0.0s + => CACHED [traffic-gen-1 2/4] RUN apk add --no-cache curl bash iproute2 0.0s + => CACHED [traffic-gen-1 3/4] COPY ./scripts/traffic-gen.sh /entrypoint.sh 0.0s + => CACHED [traffic-gen-1 4/4] RUN chmod +x /entrypoint.sh 0.0s + => [traffic-gen-2] exporting to image 0.0s + => => exporting layers 0.0s + => => exporting manifest sha256:2a5b7514893477d98546e00304bab0c5d3df5e691685413fdcd4c5d7f8a44b8e 0.0s + => => exporting config sha256:0820b0d32a4e01e75aa6f7fb86776dcb475af2f7513593360acbecaf13110a4c 0.0s + => => exporting attestation manifest sha256:47fe7dece22aee82abbc0e0fbad91b19cf38bea269f9c2a480c8d063a4b4391d 0.0s + => => exporting manifest list sha256:26a7229340f15481c403acf5125032cb86a70fcd48358256764d944d2a97a3c9 0.0s + => => naming to docker.io/library/virtual_load_network-traffic-gen-2:latest 0.0s + => => unpacking to docker.io/library/virtual_load_network-traffic-gen-2:latest 0.0s + => [traffic-gen-1] exporting to image 0.0s + => => exporting layers 0.0s + => => exporting manifest sha256:ba6645395e7cabb7c7babca4f5ae9e76586e524ee00b385e28b34ef6741d9964 0.0s + => => exporting config sha256:ab641cf5526440f2ce75212d98a7ce2c485b1a51d4215adfe0fc720d836c5774 0.0s + => => exporting attestation manifest sha256:4fd13be17b47ec34c6b497e30cbf496c9ebd542db2d7d1d03bee83e9dc8a4148 0.0s + => => exporting manifest list sha256:3f4c8d5955f564bcbf2b33634da0edab10ca65e99e21042d2ed671601e2078e1 0.0s + => => naming to docker.io/library/virtual_load_network-traffic-gen-1:latest 0.0s + => => unpacking to docker.io/library/virtual_load_network-traffic-gen-1:latest 0.0s + => [traffic-gen-1] resolving provenance for metadata file 0.0s + => [traffic-gen-2] resolving provenance for metadata file 0.0s +[+] Running 12/12 + ✔ traffic-gen-1 Built 0.0s + ✔ traffic-gen-2 Built 0.0s + ✔ Network virtual_load_network_testnet2 Created 0.0s + ✔ Network virtual_load_network_default Created 0.0s + ✔ Network virtual_load_network_testnet1 Created 0.0s + ✔ Container virtual_load_network-traffic-gen-2-2 Started 0.3s + ✔ Container virtual_load_network-pushgateway-1 Started 0.3s + ✔ Container virtual_load_network-traffic-gen-2-1 Started 0.4s + ✔ Container virtual_load_network-traffic-gen-1-2 Started 0.3s + ✔ Container virtual_load_network-traffic-gen-1-1 Started 0.4s + ✔ Container virtual_load_network-prometheus-1 Started 0.3s + ✔ Container virtual_load_network-grafana-1 Started 0.4s +Nothing to flush +Nothing to flush +Nothing to flush +Nothing to flush +Nothing to flush +Nothing to flush Test stand is running - Filter VMs: filter1 (pid 37833), filter2 (pid 37857) - Controller: pid 38215 - Traffic generators: 4 containers -``` -``` + Filter VMs: filter1 (pid 176929), filter2 (pid 177022) + Controller: pid 178189 + Traffic generators: 7 containers ``` -### 1. Start Controller +### 1. If you need to connect via SSH Connect to the controller VM via SSH: - ```bash ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@10.0.2.3 ``` -Set up internet access (required for Kaspersky API): - +Connect to the filter1 VM via SSH: ```bash -sh /mnt/shared/setup-inet.sh +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@10.0.2.1 ``` -Start the controller: - +Connect to the filter2 VM via SSH: ```bash -cd /mnt/shared -KASPERSKY_API_KEY= ./controller -KASPERSKY_API_KEY=wyo915OXTCe5stpLCtc5Ww== ./controller +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@10.0.2.2 ``` + ### 2. Load Filtering Policy -On the host, install dependencies and load a policy: +Detailed instructions for administering the policy are provided in grpc_server/admin/README.md +## Installing dependencies ```bash -cd admin -uv venv --python 3.12 .venv -source .venv/bin/activate -uv pip install -r requirements.txt -uv pip install "setuptools<75" -python -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. admin_service.proto +pip install -r requirements.txt ``` -Edit `admin/.env` to point to the controller: - -``` -SERVER_HOST=10.0.2.3 -SERVER_PORT=50051 +## Generating proto files +```bash +python -m grpc_tools.protoc \ + --python_out=. \ + --grpc_python_out=. \ + -I . \ + admin_service.proto ``` -Edit `config.toml` with desired rules, then load: +## Example of a toml file with policy +```toml +[global.rules] # Глобальные правила для всех воркеров +block_categories = ["Gambling", "Weapons"] # Категории для блокировки +block_domains = ["youtube.com", "tiktok.com"] # Домены для блокировки +allow_domains = ["github.com", "stackoverflow.com"] # Разрешённые домены (приоритет над блокировкой) +block_ips = ["192.168.0.1", "2001:0db8:85a3:0000:0000:8a2e:0370:7334"] # IP для блокировки (IPv4 и IPv6) +allow_ips = ["8.8.8.8"] # Разрешённые IP +ttl_ip = 604800 # TTL кэша IP +ttl_domain = 604800 # TTL кэша доменов +min_trust_level = 5 # Мин. уровень доверия + +[global.rules.block_by_trust] # Блокировка по уровню доверия +ENTERTAINMENT = 6 +NEWS = 4 + +[filters.filter_1] # Правила для воркера #1 +block_categories = ["Weapons", "Malware"] # Доп. категории к глобальным +block_domains = ["instagram.com"] # Доп. домены к глобальным +allow_domains = ["vk.com"] # Доп. разрешённые домены +min_trust_level = 0 # Переопределяет глобальный + +[filters.filter_1.block_by_trust] # Уровни доверия для воркера #1 +SOCIAL = 8 +ENTERTAINMENT = 7 + +[filters.filter_2] # Правила для воркера #2 +block_categories = ["Malware"] # Доп. категории к глобальным +allow_domains = ["github.com", "gitlab.com"] # Доп. разрешённые домены -```bash -python admin.py --file config.toml ``` -Expected output: -``` -(.venv) ➜ admin git:(test_stand) ✗ python admin.py --file config.toml -Config loaded -``` -``` -``` +### Send a new policy to the controller -### 3. Start Worker +```bash +python admin.py load --file .toml +``` -Connect to the filter VM via SSH: +### Get the current policy from the controller ```bash -ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@10.0.2.1 # filter1 -ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@10.0.2.2 # filter2 +python admin.py get --file .toml ``` -Start the worker: +### Enable/disable filtering on the worker ```bash -cd /mnt/shared -WORKER_ID=1 METRICS_GATEWAY_ADDRESS=10.0.2.254 METRICS_GATEWAY_PORT=9091 CONTROLLER_GRPC_ADDR=10.0.2.3:50051 DPDK_PORT_IN=eth0 DPDK_PORT_OUT=eth1 ./worker +python admin.py toogle --id 1 --on #id - worker id +python admin.py toogle --id 1 --off #id - worker id ``` -For `filter2`, use `WORKER_ID=2`. -To redirect logs to a shared file (viewable from host): +## 5. Monitoring + +### Conroller +Log monitoring ```bash -WORKER_ID=1 METRICS_GATEWAY_ADDRESS=10.0.2.254 METRICS_GATEWAY_PORT=9091 CONTROLLER_GRPC_ADDR=10.0.2.3:50051 DPDK_PORT_IN=eth0 DPDK_PORT_OUT=eth1 ./worker > /mnt/shared/filter1.log 2>&1 & +cd test/virtual_load_network/shared/logs +sudo tail -f controller.log ``` -Expected output: +### Worker 1 + +Log monitoring +```bash +cd test/virtual_load_network/shared/logs +sudo tail -f worker1.log ``` -root@qemuriscv64:~# cd /mnt/shared -root@qemuriscv64:/mnt/shared# WORKER_ID=1 METRICS_GATEWAY_ADDRESS=10.0.2.254 METRICS_GATEWAY_PORT=9091 CONTROLLER_GRPC_ADDR=10.0.2.3:50051 DPDK_PORT_IN=eth0 -DPDK_PORT_OUT=eth1 ./worker -[2026-04-23 16:46:28.534] [info] Initialize MetricsCollector with 10.0.2.254:9091 -[2026-04-23 16:46:28.823] [info] gRPC channel created to 10.0.2.3:50051 -[2026-04-23 16:46:28.826] [info] Signal handlers registered -[2026-04-23 16:46:28.828] [info] Worker 1 requests policy -[2026-04-23 16:46:29.321] [info] Policy received -EAL: Detected CPU lcores: 2 -EAL: Detected NUMA nodes: 1 -EAL: Detected shared linkage of DPDK -EAL: Multi-process socket /var/run/dpdk/rte/mp_socket -EAL: Selected IOVA mode 'PA' -EAL: TSC using RISC-V rdtime. -TELEMETRY: No legacy callbacks, legacy socket not created -libbpf: Attribute of type 0x2a found multiple times in message, previous attribute is being ignored. -libbpf: Attribute of type 0x2a found multiple times in message, previous attribute is being ignored. -libbpf: elf: skipping unrecognized data section(8) .xdp_run_config -libbpf: elf: skipping unrecognized data section(9) xdp_metadata -libbpf: Attribute of type 0x2a found multiple times in message, previous attribute is being ignored. -libbpf: Attribute of type 0x2a found multiple times in message, previous attribute is being ignored. -libbpf: elf: skipping unrecognized data section(7) xdp_metadata -libbpf: elf: skipping unrecognized data section(7) xdp_metadata -libxdp: No bpffs found at /sys/fs/bpf -libxdp: Can't use dispatcher without a working bpffs -libxdp: Falling back to loading single prog without dispatcher -libbpf: elf: skipping unrecognized data section(7) xdp_metadata -Port 0 initialized -libbpf: Attribute of type 0x2a found multiple times in message, previous attribute is being ignored. -libbpf: Attribute of type 0x2a found multiple times in message, previous attribute is being ignored. -libbpf: elf: skipping unrecognized data section(8) .xdp_run_config -libbpf: elf: skipping unrecognized data section(9) xdp_metadata -libbpf: Attribute of type 0x2a found multiple times in message, previous attribute is being ignored. -libbpf: Attribute of type 0x2a found multiple times in message, previous attribute is being ignored. -libbpf: elf: skipping unrecognized data section(7) xdp_metadata -libbpf: elf: skipping unrecognized data section(7) xdp_metadata -libxdp: No bpffs found at /sys/fs/bpf -libxdp: Can't use dispatcher without a working bpffs -libxdp: Falling back to loading single prog without dispatcher -libbpf: elf: skipping unrecognized data section(7) xdp_metadata -Port 1 initialized -Port 2 initialized -Port 0 started -Port 1 started -Port 2 started -[INFO] Loaded 0 records from SQLite, 0 expired skipped -[2026-04-23 16:46:30.684] [info] DPDK initialized: in_port=0, out_port=1 -[2026-04-23 16:46:31.204] [info] Worker 1 classifying domain 'betboom.ru' -[2026-04-23 16:46:33.307] [info] Domain 'betboom.ru' classified as category 'Gambling' with trust level 0 -This site has a locked category[INFO] Packet without dns request + +Monitoring the port on which packets are received +```bash +sudo tcpdump -i tap-f1-in port 53 ``` + +Monitoring the port from which packets are emitted +```bash +sudo tcpdump -i tap-f1-out port 53 ``` + +### Worker 2 + +Log monitoring +```bash +cd test/virtual_load_network/shared/logs +sudo tail -f worker2.log ``` +Monitoring the port on which packets are received +```bash +sudo tcpdump -i tap-f2-in port 53 +``` -## 5. Monitoring -```sh -sudo tcpdump -i tap-f1-out port 53 # output packets -sudo tcpdump -i tap-f1-in port 53 # input packets +Monitoring the port from which packets are emitted +```bash +sudo tcpdump -i tap-f2-out port 53 ``` + ## 4. Stop Test Stand ```bash -cd test/virtual_load_network -make stop +cd test/virtual_load_network/scripts +sudo ./stop.sh ``` From c3a79b3fa8641dc192e8492e662f1dc3dab1889b Mon Sep 17 00:00:00 2001 From: LapshinAE0 Date: Thu, 28 May 2026 01:38:04 +0300 Subject: [PATCH 11/13] docs: fix api python commands --- wiki/using_test_stand.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wiki/using_test_stand.md b/wiki/using_test_stand.md index ae4ba80..9264bed 100644 --- a/wiki/using_test_stand.md +++ b/wiki/using_test_stand.md @@ -237,14 +237,14 @@ python admin.py load --file .toml ### Get the current policy from the controller ```bash -python admin.py get --file .toml +python admin.py get --save .toml ``` ### Enable/disable filtering on the worker ```bash -python admin.py toogle --id 1 --on #id - worker id -python admin.py toogle --id 1 --off #id - worker id +python admin.py toggle --id 1 --on #id - worker id +python admin.py toggle --id 1 --off #id - worker id ``` From a0c11f99e58df46fe73cb41bf6aaa1313e453eff Mon Sep 17 00:00:00 2001 From: baydakov-georgiy Date: Sat, 13 Jun 2026 23:49:08 +0300 Subject: [PATCH 12/13] feat(test_stand): add DHCP server --- test/virtual_load_network/scripts/start.sh | 39 +++++++++++----------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/test/virtual_load_network/scripts/start.sh b/test/virtual_load_network/scripts/start.sh index 33c9af3..79b6bc0 100755 --- a/test/virtual_load_network/scripts/start.sh +++ b/test/virtual_load_network/scripts/start.sh @@ -131,7 +131,7 @@ start_controller() { -pidfile "/tmp/qemu-controller.pid" \ -bios "$QEMU_BIOS" \ -kernel "$QEMU_KERNEL" \ - -append "root=/dev/vda rw earlycon=sbi console=ttyS0 hugepages=${VM_HUGEPAGES} role=controller ip=${MGMT_SUBNET}.3::${MGMT_SUBNET}.254:255.255.255.0::eth0:off" \ + -append "root=/dev/vda rw earlycon=sbi console=ttyS0 hugepages=${VM_HUGEPAGES} role=controller mgmt_if=eth0" \ -netdev tap,id=net0,ifname="tap-ctrl",script=no,downscript=no \ -device virtio-net-device,netdev=net0 \ -netdev tap,id=net1,ifname="tap-ctrl-inet",script=no,downscript=no \ @@ -152,11 +152,10 @@ start_filter_vm() { local TAP_OUT="$3" local TAP_MGMT="$4" local BRIDGE_IN="$5" - local MGMT_IP="$6" - local MAC_IN="$7" - local MAC_OUT="$8" - local MAC_MGMT="$9" - local WORKER_ID="${10}" + local MAC_IN="$6" + local MAC_OUT="$7" + local MAC_MGMT="$8" + local WORKER_ID="$9" local OVERLAY="$PROJECT_DIR/${VM_NAME}.qcow2" if [ ! -f "$OVERLAY" ]; then @@ -182,7 +181,7 @@ start_filter_vm() { -pidfile "/tmp/qemu-${VM_NAME}.pid" \ -bios "$QEMU_BIOS" \ -kernel "$QEMU_KERNEL" \ - -append "root=/dev/vda rw earlycon=sbi console=ttyS0 hugepages=${VM_HUGEPAGES} role=worker worker_id=${WORKER_ID} ip=${MGMT_IP}::${MGMT_SUBNET}.254:255.255.255.0::eth2:off" \ + -append "root=/dev/vda rw earlycon=sbi console=ttyS0 hugepages=${VM_HUGEPAGES} role=worker worker_id=${WORKER_ID} mgmt_if=eth2" \ -netdev tap,id=net0,ifname="$TAP_IN",script=no,downscript=no \ -device virtio-net-device,netdev=net0,mac="$MAC_IN" \ -netdev tap,id=net1,ifname="$TAP_OUT",script=no,downscript=no \ @@ -199,8 +198,20 @@ start_filter_vm() { } } -start_filter_vm "filter1" "tap-f1-in" "tap-f1-out" "tap-f1-mgmt" "$BRIDGE1" "${MGMT_SUBNET}.1" "$FILTER1_MAC_IN" "$FILTER1_MAC_OUT" "$FILTER1_MAC_MGMT" 1 -start_filter_vm "filter2" "tap-f2-in" "tap-f2-out" "tap-f2-mgmt" "$BRIDGE2" "${MGMT_SUBNET}.2" "$FILTER2_MAC_IN" "$FILTER2_MAC_OUT" "$FILTER2_MAC_MGMT" 2 +ip tuntap add dev "tap-ctrl" mode tap 2>/dev/null || true +ip link set "tap-ctrl" master "$MGMT_BRIDGE" +ip link set "tap-ctrl" up + +ip tuntap add dev "tap-ctrl-inet" mode tap 2>/dev/null || true +ip link set "tap-ctrl-inet" master "$INET_BRIDGE" +ip link set "tap-ctrl-inet" up + +start_controller + +sleep 3 + +start_filter_vm "filter1" "tap-f1-in" "tap-f1-out" "tap-f1-mgmt" "$BRIDGE1" "$FILTER1_MAC_IN" "$FILTER1_MAC_OUT" "$FILTER1_MAC_MGMT" 1 +start_filter_vm "filter2" "tap-f2-in" "tap-f2-out" "tap-f2-mgmt" "$BRIDGE2" "$FILTER2_MAC_IN" "$FILTER2_MAC_OUT" "$FILTER2_MAC_MGMT" 2 for SVC in $(docker compose ps -q 2>/dev/null); do docker exec "$SVC" ip neigh flush all 2>/dev/null || true @@ -219,16 +230,6 @@ for SVC in $(docker compose -p "$(basename "$PROJECT_DIR")" ps --format '{{.Name docker exec "$SVC" ip -6 route replace default via "${SUBNET2_V6}fe" dev eth0 2>/dev/null || true done -ip tuntap add dev "tap-ctrl" mode tap 2>/dev/null || true -ip link set "tap-ctrl" master "$MGMT_BRIDGE" -ip link set "tap-ctrl" up - -ip tuntap add dev "tap-ctrl-inet" mode tap 2>/dev/null || true -ip link set "tap-ctrl-inet" master "$INET_BRIDGE" -ip link set "tap-ctrl-inet" up - -start_controller - echo "Test stand is running" echo " Filter VMs: filter1 (pid $(cat /tmp/qemu-filter1.pid 2>/dev/null)), filter2 (pid $(cat /tmp/qemu-filter2.pid 2>/dev/null))" echo " Controller: pid $(cat /tmp/qemu-controller.pid 2>/dev/null)" From d61c2fc78c11dee8f58f55f1ae470e7c8cb89eb9 Mon Sep 17 00:00:00 2001 From: baydakov-georgiy Date: Sun, 14 Jun 2026 14:36:32 +0300 Subject: [PATCH 13/13] feat(test_stand): add redis-server for controller cash --- test/virtual_load_network/scripts/start.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/virtual_load_network/scripts/start.sh b/test/virtual_load_network/scripts/start.sh index 79b6bc0..0a4eb76 100755 --- a/test/virtual_load_network/scripts/start.sh +++ b/test/virtual_load_network/scripts/start.sh @@ -116,6 +116,7 @@ cp ../../controller/internal/service/config/categories.json "$SHARED_DIR/interna cp ../../controller/internal/service/config/providers.json "$SHARED_DIR/internal/service/config/" if [ -d "$PROJECT_DIR/configs" ]; then cp -r "$PROJECT_DIR/configs" "$SHARED_DIR/" + cp -r "$PROJECT_DIR/configs/controller.env" "$SHARED_DIR/.env" fi start_controller() {