diff --git a/controller/Makefile b/controller/Makefile index 9609a76..80d123b 100644 --- a/controller/Makefile +++ b/controller/Makefile @@ -8,7 +8,7 @@ all: generate build generate: protoc --go_out=. --go_opt=paths=source_relative \ --go-grpc_out=. --go-grpc_opt=paths=source_relative \ - $(PROTO_DIR)/file_service/file_service.proto \ + $(PROTO_DIR)/admin_service/admin_service.proto \ $(PROTO_DIR)/communication/communication.proto build: generate @@ -16,14 +16,18 @@ build: generate go build -o $(BIN_DIR)/grpc_server ./$(GRPC_SERVER) go build -o $(BIN_DIR)/manager ./$(MANAGER) +build-riscv: + bazel build --platforms=@rules_go//go/toolchain:linux_riscv64 //cmd/grpc_server:grpc_server + mkdir -p $(BIN_DIR) + cp -f bazel-bin/cmd/grpc_server/grpc_server_/grpc_server $(BIN_DIR)/grpc_server 2>/dev/null || cp -f bazel-bin/cmd/grpc_server/grpc_server $(BIN_DIR)/grpc_server + clean: rm -rf $(BIN_DIR) rm -f $(PROTO_DIR)/communication/*.pb.go - rm -f $(PROTO_DIR)/file_service/*.pb.go + rm -f $(PROTO_DIR)/admin_service/*.pb.go run-server: build ./$(BIN_DIR)/grpc_server run-manager: build ./$(BIN_DIR)/manager - diff --git a/controller/internal/service/service/service.go b/controller/internal/service/service/service.go index 303edb6..5b72c10 100644 --- a/controller/internal/service/service/service.go +++ b/controller/internal/service/service/service.go @@ -77,18 +77,19 @@ func (s *Service) Check(checkValue string, endpointName string) ([]int, error) { continue } - resp.Body.Close() - if resp.StatusCode != http.StatusOK { + resp.Body.Close() log.Printf("Provider %s returned %s", providerName, resp.Status) continue } var data map[string]interface{} if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { + resp.Body.Close() log.Printf("Error parsing response from %s: %v", providerName, err) continue } + resp.Body.Close() categoryPath := provider.Endpoints[endpointName].Categories actual, err := s.jsonParser.ExtractCategories(data, categoryPath) diff --git a/test/virtual_load_network/.env b/test/virtual_load_network/.env index a4d98db..87fa869 100644 --- a/test/virtual_load_network/.env +++ b/test/virtual_load_network/.env @@ -1,7 +1,34 @@ -NUM_HOSTS=3 -SUBNET=10.0.0 -GATEWAY=10.0.0.254 +NUM_HOSTS_NET1=2 +NUM_HOSTS_NET2=2 + +SUBNET1=10.0.0 +SUBNET2=10.0.1 +GATEWAY1=10.0.0.254 +GATEWAY2=10.0.1.254 +BRIDGE1=br-testnet1 +BRIDGE2=br-testnet2 DNS=8.8.8.8 -FILTER_PATH=../../worker/main-x86-virt -BRIDGE_IFACE=br-testnet + +SUBNET1_V6=fd00:1:: +SUBNET2_V6=fd00:2:: +DNS_V6=2001:4860:4860::8888 + +INET_SUBNET=10.0.3 +INET_BRIDGE=br-inet +INET_SUBNET_V6=fd00:b:: + +MGMT_SUBNET=10.0.2 +MGMT_BRIDGE=br-mgmt +MGMT_SUBNET_V6=fd00:a:: + +YOCTO_DEPLOY_DIR=/home/lespend/program/yadro/vm_build_risc_v/qemu/poky/build/tmp/deploy/images/qemuriscv64 +QEMU_ROOTFS=${YOCTO_DEPLOY_DIR}/cluster-image-qemuriscv64.rootfs.ext4 +QEMU_KERNEL=${YOCTO_DEPLOY_DIR}/Image +QEMU_BIOS=${YOCTO_DEPLOY_DIR}/fw_jump.elf +QEMU_MEMORY=4G +QEMU_CPUS=2 + +FILTER_RISCV_BIN=../../worker/main-riscv +CONTROLLER_BIN=../../controller/bazel-bin/cmd/grpc_server/grpc_server_/grpc_server + HUGEPAGES=1024 diff --git a/test/virtual_load_network/Dockerfile.traffic-gen b/test/virtual_load_network/Dockerfile.traffic-gen index a6e9a82..a4daa1c 100644 --- a/test/virtual_load_network/Dockerfile.traffic-gen +++ b/test/virtual_load_network/Dockerfile.traffic-gen @@ -1,6 +1,6 @@ FROM alpine:3.20 -RUN apk add --no-cache curl bash +RUN apk add --no-cache curl bash iproute2 COPY ./scripts/traffic-gen.sh /entrypoint.sh RUN chmod +x /entrypoint.sh diff --git a/test/virtual_load_network/Makefile b/test/virtual_load_network/Makefile new file mode 100644 index 0000000..cf4f97e --- /dev/null +++ b/test/virtual_load_network/Makefile @@ -0,0 +1,25 @@ +SCRIPTS_DIR = scripts +WORKER_DIR = ../../worker +CONTROLLER_DIR = ../../controller + +.PHONY: run stop logs build build-worker build-controller clean-vms + +run: build + sudo $(SCRIPTS_DIR)/start.sh + +stop: + sudo $(SCRIPTS_DIR)/stop.sh + +logs: + @sudo bash -c 'while true; do clear; cat shared/filter1.log 2>/dev/null | tail -10; echo "---"; cat shared/filter2.log 2>/dev/null | tail -10; echo "---"; cat shared/controller.log 2>/dev/null | tail -5; sleep 2; done' + +build: build-worker build-controller + +build-worker: + cd $(WORKER_DIR) && bazel build --config=riscv64 //:worker + +build-controller: + $(MAKE) -C $(CONTROLLER_DIR) build-riscv + +clean-vms: + rm -f filter1.qcow2 filter2.qcow2 controller.qcow2 diff --git a/test/virtual_load_network/README.md b/test/virtual_load_network/README.md index 35a4e7c..9238dee 100644 --- a/test/virtual_load_network/README.md +++ b/test/virtual_load_network/README.md @@ -1,50 +1,74 @@ -# Тестовый стенд виртуальной нагрузки +# Виртуальный тестовый стенд -## Установка +## Что необходимо сделать +- [x] Настроить топологию сети и виртуальные машины +- [x] Добавить кросс-компиляцию controller под RISC-V +- [x] Добaвить кросс-компиляцию dpdk библиотеки + - [x] На данный момент используется Docker контейнер внутри которого билдиться библиотека. Это медленно (минут 50, но делается один раз) + - [ ] (\*) По хорошему необходимо создать отдельный sysroot с помощью Yocto Project внутри которого будем билдить +- [x] Запуск фильтров и контроллера + - [x] Запуск фильтров происходит через ssh подключение и прописывание вручную комманды + - [ ] (\*) По хорошему необходимо сделать запуск через systemd +- [ ] Взаимодействие controller и worker, пока не сделано: + - [ ] Кросс компиляция worker вместе с dpdk-filter. На данный момент билдиться только сам dpdk-filter -### DPDK -```sh -sudo apt-get install -y meson ninja-build python3-pyelftools libbpf-dev -git clone https://github.com/DPDK/dpdk.git -cd dpdk -meson setup -Denable_drivers=net/af_xdp,net/tap build -ninja -C build -sudo ninja -C build install -``` +## Подготовка -### Компиляция worker +### Создание RISC-V образа ```sh -cd worker -make -f Makefile.main_x86 virt +sudo scripts/create_image.sh ``` +Пока скрипт из test-image переписан только под arch linux. Надо переписать его под ubuntu и запускать внутри docker контейнера + +Подробности: `wiki/using_test_image.md` + ## Конфигурация -Параметры задаются в `.env`: +Параметры в `.env`: -| Переменная | По умолчанию | Описание | -|---------------|---------------------------|----------------------------------| -| NUM_HOSTS | 3 | Количество генераторов трафика | -| SUBNET | 10.0.0 | Подсеть (первые 3 октета) | -| GATEWAY | 10.0.0.254 | Адрес шлюза | -| DNS | 8.8.8.8 | DNS для контейнеров | -| FILTER_PATH | ../../worker/main-x86-virt| Путь до бинаря фильтра | -| HUGEPAGES | 1024 | Количество hugepages | +| Переменная | По умолчанию | Описание | +|------------------|----------------------|-----------------------------------| +| NUM_HOSTS_NET1 | 2 | Генераторов в сети 1 | +| NUM_HOSTS_NET2 | 2 | Генераторов в сети 2 | +| SUBNET1 | 10.0.0 | Подсеть 1 | +| SUBNET2 | 10.0.1 | Подсеть 2 | +| MGMT_SUBNET | 10.0.2 | Управляющая подсеть | +| QEMU_IMAGE | ubuntu-...riscv64.img| Путь к RISC-V образу | +| QEMU_MEMORY | 4G | RAM на каждую VM | +| QEMU_CPUS | 2 | CPU на каждую VM | +| FILTER_RISCV_BIN | ../../worker/main-riscv-virt | Путь к бинарю фильтра | +| CONTROLLER_BIN | ../../controller/bin/grpc_server | Путь к контроллеру | +| HUGEPAGES | 1024 | Количество hugepages | ## Запуск + +Виртуальные машины и сеть: ```sh cd test/virtual_load_network -sudo scripts/start.sh +make run +``` + +Запуск фильтров осуществляется через подлкючение по ssh: ``` +# for filter1 +ssh-keygen -R 10.0.2.1 +ssh ubuntu@10.0.2.1 +# password ubuntu +# in VM: +sudo rm -rf /var/run/dpdk /dev/hugepages/rtemap_* +sudo LIBXDP_OBJECT_PATH=/usr/lib/riscv64-linux-gnu/bpf LD_LIBRARY_PATH=/mnt/lib /mnt/filter --no-pci --iova-mode=va -d /mnt/lib/librte_net_af_xdp.so -d /mnt/lib/librte_net_tap.so -- -Скрипт выполняет: -1. Настройку hugepages -2. Запуск контейнеров-генераторов через `docker compose` -3. Создание veth пары и tc mirroring на bridge compose-сети -4. Настройку NAT -5. Запуск DPDK фильтра +# Also for filter2 +ssh-keygen -R 10.0.2.2 +ssh ubuntu@10.0.2.2 +# password ubuntu +# in VM: +sudo rm -rf /var/run/dpdk /dev/hugepages/rtemap_* +sudo LIBXDP_OBJECT_PATH=/usr/lib/riscv64-linux-gnu/bpf LD_LIBRARY_PATH=/mnt/lib /mnt/filter --no-pci --iova-mode=va -d /mnt/lib/librte_net_af_xdp.so -d /mnt/lib/librte_net_tap.so -- +``` ## Остановка ```sh -sudo scripts/stop.sh +make stop ``` diff --git a/test/virtual_load_network/configs/controller.env b/test/virtual_load_network/configs/controller.env new file mode 100644 index 0000000..07ee673 --- /dev/null +++ b/test/virtual_load_network/configs/controller.env @@ -0,0 +1 @@ +KASPERSKY_API_KEY=wyo915OXTCe5stpLCtc5Ww== diff --git a/test/virtual_load_network/configs/policy.toml b/test/virtual_load_network/configs/policy.toml new file mode 100644 index 0000000..9164fda --- /dev/null +++ b/test/virtual_load_network/configs/policy.toml @@ -0,0 +1,15 @@ +[global.rules] +block_categories = ["adult", "malware"] +block_domains = ["example.com", "blocked-site.com"] +allow_domains = ["google.com", "github.com"] +min_trust_level = 50 + +[global.rules.block_by_trust] +dangerous = 10 +suspicious = 30 + +[filters.filter_1] +block_domains = ["extra-blocked.com"] + +[filters.filter_2] +allow_domains = ["extra-allowed.com"] diff --git a/test/virtual_load_network/configs/worker.env b/test/virtual_load_network/configs/worker.env new file mode 100644 index 0000000..2dff044 --- /dev/null +++ b/test/virtual_load_network/configs/worker.env @@ -0,0 +1,5 @@ +CONTROLLER_GRPC_ADDR=10.0.2.3:50051 +METRICS_GATEWAY_ADDRESS=10.0.2.254 +METRICS_GATEWAY_PORT=9091 +DPDK_PORT_IN=eth0 +DPDK_PORT_OUT=eth1 diff --git a/test/virtual_load_network/docker-compose.yml b/test/virtual_load_network/docker-compose.yml index 5d23547..f1a8fff 100644 --- a/test/virtual_load_network/docker-compose.yml +++ b/test/virtual_load_network/docker-compose.yml @@ -1,25 +1,90 @@ services: - traffic-gen: + traffic-gen-1: build: context: . dockerfile: Dockerfile.traffic-gen networks: - - testnet + - testnet1 cap_add: - NET_ADMIN dns: - ${DNS:-8.8.8.8} + - ${DNS_V6:-2001:4860:4860::8888} deploy: - replicas: ${NUM_HOSTS:-3} + replicas: ${NUM_HOSTS_NET1:-2} + restart: unless-stopped + + traffic-gen-2: + build: + context: . + dockerfile: Dockerfile.traffic-gen + networks: + - testnet2 + cap_add: + - NET_ADMIN + dns: + - ${DNS:-8.8.8.8} + - ${DNS_V6:-2001:4860:4860::8888} + deploy: + replicas: ${NUM_HOSTS_NET2:-2} + restart: unless-stopped + + pushgateway: + image: prom/pushgateway:latest + ports: + - "9091:9091" + restart: unless-stopped + + prometheus: + image: prom/prometheus:latest + volumes: + - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro + ports: + - "9090:9090" + depends_on: + - pushgateway + restart: unless-stopped + + grafana: + image: grafana/grafana:latest + volumes: + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - grafana-data:/var/lib/grafana + ports: + - "3000:3000" + environment: + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin" + depends_on: + - prometheus restart: unless-stopped networks: - testnet: + testnet1: + driver: bridge + enable_ipv6: true + driver_opts: + com.docker.network.bridge.name: "br-testnet1" + com.docker.network.bridge.enable_ip_masquerade: "false" + ipam: + config: + - subnet: ${SUBNET1:-10.0.0}.0/24 + gateway: ${SUBNET1:-10.0.0}.254 + - subnet: ${SUBNET1_V6:-fd00:1::}0/64 + gateway: ${SUBNET1_V6:-fd00:1::}ff + + testnet2: driver: bridge + enable_ipv6: true driver_opts: - com.docker.network.bridge.name: "br-testnet" + com.docker.network.bridge.name: "br-testnet2" com.docker.network.bridge.enable_ip_masquerade: "false" ipam: config: - - subnet: ${SUBNET:-10.0.0}.0/24 - gateway: ${GATEWAY:-10.0.0.254} + - subnet: ${SUBNET2:-10.0.1}.0/24 + gateway: ${SUBNET2:-10.0.1}.254 + - subnet: ${SUBNET2_V6:-fd00:2::}0/64 + gateway: ${SUBNET2_V6:-fd00:2::}ff + +volumes: + grafana-data: diff --git a/test/virtual_load_network/monitoring/grafana/provisioning/dashboards/cluster.json b/test/virtual_load_network/monitoring/grafana/provisioning/dashboards/cluster.json new file mode 100644 index 0000000..8c31c54 --- /dev/null +++ b/test/virtual_load_network/monitoring/grafana/provisioning/dashboards/cluster.json @@ -0,0 +1,105 @@ +{ + "title": "Cluster Workers", + "uid": "cluster-workers", + "panels": [ + { + "title": "Packets Received", + "type": "timeseries", + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 0}, + "targets": [ + { + "expr": "rate(packets_received_total[1m])", + "legendFormat": "{{job}}" + } + ], + "datasource": "Prometheus" + }, + { + "title": "Packets Passed", + "type": "timeseries", + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 0}, + "targets": [ + { + "expr": "rate(packets_passed_total[1m])", + "legendFormat": "{{job}}" + } + ], + "datasource": "Prometheus" + }, + { + "title": "Packets Dropped", + "type": "timeseries", + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 0}, + "targets": [ + { + "expr": "rate(packets_dropped_total[1m])", + "legendFormat": "{{job}}" + } + ], + "datasource": "Prometheus" + }, + { + "title": "Blocked Domains", + "type": "timeseries", + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 8}, + "targets": [ + { + "expr": "rate(blocked_domains_total[1m])", + "legendFormat": "{{job}}" + } + ], + "datasource": "Prometheus" + }, + { + "title": "CPU Usage %", + "type": "timeseries", + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 8}, + "targets": [ + { + "expr": "cpu_usage{cpu='cpu'}", + "legendFormat": "{{job}}" + } + ], + "datasource": "Prometheus" + }, + { + "title": "Memory Used (MB)", + "type": "timeseries", + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 8}, + "targets": [ + { + "expr": "memory_used / 1024 / 1024", + "legendFormat": "{{job}}" + } + ], + "datasource": "Prometheus" + }, + { + "title": "Task Duration (s)", + "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 16}, + "targets": [ + { + "expr": "task_duration_seconds", + "legendFormat": "{{job}}" + } + ], + "datasource": "Prometheus" + }, + { + "title": "Push Errors", + "type": "stat", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 16}, + "targets": [ + { + "expr": "push_errors_total", + "legendFormat": "{{job}}" + } + ], + "datasource": "Prometheus" + } + ], + "time": {"from": "now-15m", "to": "now"}, + "refresh": "5s", + "schemaVersion": 39 +} diff --git a/test/virtual_load_network/monitoring/grafana/provisioning/dashboards/dashboards.yml b/test/virtual_load_network/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..24910df --- /dev/null +++ b/test/virtual_load_network/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,8 @@ +apiVersion: 1 + +providers: + - name: default + type: file + options: + path: /etc/grafana/provisioning/dashboards + foldersFromFilesStructure: false diff --git a/test/virtual_load_network/monitoring/grafana/provisioning/datasources/prometheus.yml b/test/virtual_load_network/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..86fd346 --- /dev/null +++ b/test/virtual_load_network/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,8 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true diff --git a/test/virtual_load_network/monitoring/prometheus.yml b/test/virtual_load_network/monitoring/prometheus.yml new file mode 100644 index 0000000..c7dae34 --- /dev/null +++ b/test/virtual_load_network/monitoring/prometheus.yml @@ -0,0 +1,9 @@ +global: + scrape_interval: 5s + +scrape_configs: + - job_name: pushgateway + honor_labels: true + static_configs: + - targets: + - pushgateway:9091 diff --git a/test/virtual_load_network/scripts/start-vm.sh b/test/virtual_load_network/scripts/start-vm.sh new file mode 100755 index 0000000..d19d1ba --- /dev/null +++ b/test/virtual_load_network/scripts/start-vm.sh @@ -0,0 +1,52 @@ +#!/bin/bash +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +if [ -f "$PROJECT_DIR/.env" ]; then + set -a + source "$PROJECT_DIR/.env" + set +a +fi + +VM_NAME="$1" +SHARED_DIR="$2" +TAP_IFACE="$3" +MEMORY="${4:-${QEMU_MEMORY:-4G}}" +CPUS="${5:-${QEMU_CPUS:-2}}" + +YOCTO_DEPLOY_DIR="${YOCTO_DEPLOY_DIR:-/home/lespend/program/yadro/vm_build_risc_v/qemu/poky/build/tmp/deploy/images/qemuriscv64}" +QEMU_ROOTFS="${QEMU_ROOTFS:-${YOCTO_DEPLOY_DIR}/cluster-image-qemuriscv64.rootfs.ext4}" +QEMU_KERNEL="${QEMU_KERNEL:-${YOCTO_DEPLOY_DIR}/Image}" +QEMU_BIOS="${QEMU_BIOS:-${YOCTO_DEPLOY_DIR}/fw_jump.elf}" + +if [ -z "$VM_NAME" ] || [ -z "$SHARED_DIR" ] || [ -z "$TAP_IFACE" ]; then + echo "Usage: $0 [memory] [cpus]" + exit 1 +fi + +OVERLAY="$PROJECT_DIR/${VM_NAME}.qcow2" +if [ ! -f "$OVERLAY" ]; then + qemu-img create -f qcow2 -b "$QEMU_ROOTFS" -F raw "$OVERLAY" >/dev/null +fi + +ip tuntap add dev "$TAP_IFACE" mode tap 2>/dev/null || true +ip link set "$TAP_IFACE" up + +qemu-system-riscv64 \ + -machine virt -m "$MEMORY" -smp "$CPUS" \ + -display none -serial "file:/tmp/qemu-${VM_NAME}.log" \ + -name "$VM_NAME" \ + -pidfile "/tmp/qemu-${VM_NAME}.pid" \ + -bios "$QEMU_BIOS" \ + -kernel "$QEMU_KERNEL" \ + -append "root=/dev/vda rw earlycon=sbi console=ttyS0" \ + -netdev tap,id=net0,ifname="$TAP_IFACE",script=no,downscript=no \ + -device virtio-net-device,netdev=net0 \ + -netdev user,id=net1,hostfwd=tcp::$(shuf -i 10000-20000 -n1)-:22 \ + -device virtio-net-device,netdev=net1 \ + -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \ + -drive "id=disk0,file=${OVERLAY},format=qcow2,if=none" -device virtio-blk-device,drive=disk0 \ + -virtfs "local,path=$SHARED_DIR,mount_tag=host_share,security_model=mapped-xattr" \ + -daemonize diff --git a/test/virtual_load_network/scripts/start.sh b/test/virtual_load_network/scripts/start.sh index 81c0f70..0a4eb76 100755 --- a/test/virtual_load_network/scripts/start.sh +++ b/test/virtual_load_network/scripts/start.sh @@ -12,48 +12,226 @@ if [ -f .env ]; then set +a fi -NUM_HOSTS="${NUM_HOSTS:-3}" -SUBNET="${SUBNET:-10.0.0}" -GATEWAY="${GATEWAY:-10.0.0.254}" +SUBNET1="${SUBNET1:-10.0.0}" +SUBNET2="${SUBNET2:-10.0.1}" +INET_SUBNET="${INET_SUBNET:-10.0.3}" +BRIDGE1="${BRIDGE1:-br-testnet1}" +BRIDGE2="${BRIDGE2:-br-testnet2}" +INET_BRIDGE="${INET_BRIDGE:-br-inet}" +MGMT_SUBNET="${MGMT_SUBNET:-10.0.2}" +MGMT_BRIDGE="${MGMT_BRIDGE:-br-mgmt}" HUGEPAGES="${HUGEPAGES:-1024}" -FILTER_PATH="${FILTER_PATH:-../../worker/main-x86-virt}" -BRIDGE_IFACE="${BRIDGE_IFACE:-br-testnet}" + +SUBNET1_V6="${SUBNET1_V6:-fd00:1::}" +SUBNET2_V6="${SUBNET2_V6:-fd00:2::}" +INET_SUBNET_V6="${INET_SUBNET_V6:-fd00:b::}" +MGMT_SUBNET_V6="${MGMT_SUBNET_V6:-fd00:a::}" + +YOCTO_DEPLOY_DIR="${YOCTO_DEPLOY_DIR:-/home/lespend/program/yadro/vm_build_risc_v/qemu/poky/build/tmp/deploy/images/qemuriscv64}" +QEMU_ROOTFS="${QEMU_ROOTFS:-${YOCTO_DEPLOY_DIR}/cluster-image-qemuriscv64.rootfs.ext4}" +QEMU_KERNEL="${QEMU_KERNEL:-${YOCTO_DEPLOY_DIR}/Image}" +QEMU_BIOS="${QEMU_BIOS:-${YOCTO_DEPLOY_DIR}/fw_jump.elf}" +QEMU_MEMORY="${QEMU_MEMORY:-4G}" +QEMU_CPUS="${QEMU_CPUS:-2}" +FILTER_RISCV_BIN="${FILTER_RISCV_BIN:-../../worker/main-riscv}" +WORKER_RISCV_BIN="${WORKER_RISCV_BIN:-../../worker/bazel-bin/worker}" +CONTROLLER_BIN="${CONTROLLER_BIN:-$(readlink -f ../../controller/bazel-bin/cmd/grpc_server/grpc_server_/grpc_server 2>/dev/null)}" +VM_HUGEPAGES="${VM_HUGEPAGES:-64}" +FILTER1_MAC_IN="52:54:00:f1:00:01" +FILTER1_MAC_OUT="52:54:00:f1:00:02" +FILTER1_MAC_MGMT="52:54:00:f1:00:03" +FILTER2_MAC_IN="52:54:00:f2:00:01" +FILTER2_MAC_OUT="52:54:00:f2:00:02" +FILTER2_MAC_MGMT="52:54:00:f2:00:03" "$SCRIPT_DIR/stop.sh" 2>/dev/null || true -sysctl -w vm.drop_caches=3 > /dev/null -echo "$HUGEPAGES" > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages +sysctl -w vm.drop_caches=3 >/dev/null +echo "$HUGEPAGES" >/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages docker compose up -d --build -for i in $(seq 1 10); do - ip link show "$BRIDGE_IFACE" &>/dev/null && break - sleep 0.5 +for BRIDGE in "$BRIDGE1" "$BRIDGE2"; do + for i in $(seq 1 10); do + ip link show "$BRIDGE" &>/dev/null && break + sleep 0.5 + done + if ! ip link show "$BRIDGE" &>/dev/null; then + echo "Bridge $BRIDGE not found" + exit 1 + fi done -if ! ip link show "$BRIDGE_IFACE" &>/dev/null; then - echo "Bridge interface $BRIDGE_IFACE not found" - exit 1 -fi +sysctl -w net.ipv4.ip_forward=1 >/dev/null +sysctl -w net.ipv6.conf.all.forwarding=1 >/dev/null + +ip link add "$INET_BRIDGE" type bridge 2>/dev/null || true +ip link set "$INET_BRIDGE" up +ip addr add "${INET_SUBNET}.254/24" dev "$INET_BRIDGE" 2>/dev/null || true +ip -6 addr add "${INET_SUBNET_V6}fe/64" dev "$INET_BRIDGE" 2>/dev/null || true + +iptables -t nat -A POSTROUTING -s ${INET_SUBNET}.0/24 ! -d ${INET_SUBNET}.0/24 -j MASQUERADE +iptables -I FORWARD -i "$INET_BRIDGE" -j ACCEPT +iptables -I FORWARD -o "$INET_BRIDGE" -j ACCEPT + +ip6tables -t nat -A POSTROUTING -s ${INET_SUBNET_V6}/64 ! -d ${INET_SUBNET_V6}/64 -j MASQUERADE +ip6tables -I FORWARD -i "$INET_BRIDGE" -j ACCEPT +ip6tables -I FORWARD -o "$INET_BRIDGE" -j ACCEPT + +ip link add "$MGMT_BRIDGE" type bridge 2>/dev/null || true +ip link set "$MGMT_BRIDGE" up +ip addr add "${MGMT_SUBNET}.254/24" dev "$MGMT_BRIDGE" 2>/dev/null || true +ip -6 addr add "${MGMT_SUBNET_V6}fe/64" dev "$MGMT_BRIDGE" 2>/dev/null || true -ip link delete veth0 2>/dev/null || true -ip link add veth0 type veth peer name veth1 -ip link set veth0 up -ip link set veth1 up +iptables -I FORWARD -i "$MGMT_BRIDGE" -j ACCEPT +iptables -I FORWARD -o "$MGMT_BRIDGE" -j ACCEPT -tc qdisc add dev "$BRIDGE_IFACE" clsact -tc filter add dev "$BRIDGE_IFACE" egress matchall action mirred egress mirror dev veth1 -tc filter add dev "$BRIDGE_IFACE" ingress matchall action mirred egress mirror dev veth1 +SHARED_DIR="$PROJECT_DIR/shared" +mkdir -p "$SHARED_DIR" +mkdir -p "$SHARED_DIR/logs" -sysctl -w net.ipv4.ip_forward=1 > /dev/null -iptables -t nat -A POSTROUTING -s ${SUBNET}.0/24 ! -d ${SUBNET}.0/24 -j MASQUERADE -iptables -I FORWARD -i "$BRIDGE_IFACE" -j ACCEPT -iptables -I FORWARD -o "$BRIDGE_IFACE" -j ACCEPT +if [ -f "$FILTER_RISCV_BIN" ]; then + cp "$FILTER_RISCV_BIN" "$SHARED_DIR/filter" +fi +if [ -f "$WORKER_RISCV_BIN" ]; then + cp "$WORKER_RISCV_BIN" "$SHARED_DIR/worker" +fi +if [ -f "$CONTROLLER_BIN" ]; then + cp "$CONTROLLER_BIN" "$SHARED_DIR/controller" +fi + +cat > "$SHARED_DIR/setup-inet.sh" << EOF +#!/bin/sh +ip addr add ${INET_SUBNET}.1/24 dev eth1 2>/dev/null || true +ip -6 addr add ${INET_SUBNET_V6}1/64 dev eth1 2>/dev/null || true +ip route replace default via ${INET_SUBNET}.254 dev eth1 +ip -6 route replace default via ${INET_SUBNET_V6}fe dev eth1 2>/dev/null || true +echo "nameserver 8.8.8.8" > /etc/resolv.conf +echo "nameserver 2001:4860:4860::8888" >> /etc/resolv.conf +EOF +chmod +x "$SHARED_DIR/setup-inet.sh" -if [ ! -x "$FILTER_PATH" ]; then - echo "Filter binary not found: $FILTER_PATH" - echo "Build: cd worker && make -f Makefile.main_x86 virt" - exit 1 +mkdir -p "$SHARED_DIR/internal/service/config" +cp ../../controller/internal/service/config/categories.json "$SHARED_DIR/internal/service/config/" +cp ../../controller/internal/service/config/providers.json "$SHARED_DIR/internal/service/config/" +if [ -d "$PROJECT_DIR/configs" ]; then + cp -r "$PROJECT_DIR/configs" "$SHARED_DIR/" + cp -r "$PROJECT_DIR/configs/controller.env" "$SHARED_DIR/.env" fi -LD_LIBRARY_PATH=/usr/local/lib exec "$FILTER_PATH" --no-pci -- +start_controller() { + local CTRL_OVERLAY="$PROJECT_DIR/controller.qcow2" + if [ ! -f "$CTRL_OVERLAY" ]; then + qemu-img create -f qcow2 -b "$QEMU_ROOTFS" -F raw "$CTRL_OVERLAY" >/dev/null + fi + + qemu-system-riscv64 \ + -machine virt -m "$QEMU_MEMORY" -smp "$QEMU_CPUS" \ + -display none -serial "file:/tmp/qemu-controller.log" \ + -name "controller" \ + -pidfile "/tmp/qemu-controller.pid" \ + -bios "$QEMU_BIOS" \ + -kernel "$QEMU_KERNEL" \ + -append "root=/dev/vda rw earlycon=sbi console=ttyS0 hugepages=${VM_HUGEPAGES} role=controller mgmt_if=eth0" \ + -netdev tap,id=net0,ifname="tap-ctrl",script=no,downscript=no \ + -device virtio-net-device,netdev=net0 \ + -netdev tap,id=net1,ifname="tap-ctrl-inet",script=no,downscript=no \ + -device virtio-net-device,netdev=net1 \ + -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \ + -drive "id=disk0,file=${CTRL_OVERLAY},format=qcow2,if=none" -device virtio-blk-device,drive=disk0 \ + -virtfs "local,path=$SHARED_DIR,mount_tag=host_share,security_model=mapped-xattr" \ + -daemonize || + { + echo "Failed to start controller" + exit 1 + } +} + +start_filter_vm() { + local VM_NAME="$1" + local TAP_IN="$2" + local TAP_OUT="$3" + local TAP_MGMT="$4" + local BRIDGE_IN="$5" + local MAC_IN="$6" + local MAC_OUT="$7" + local MAC_MGMT="$8" + local WORKER_ID="$9" + + local OVERLAY="$PROJECT_DIR/${VM_NAME}.qcow2" + if [ ! -f "$OVERLAY" ]; then + qemu-img create -f qcow2 -b "$QEMU_ROOTFS" -F raw "$OVERLAY" >/dev/null + fi + + ip tuntap add dev "$TAP_IN" mode tap 2>/dev/null || true + ip link set "$TAP_IN" master "$BRIDGE_IN" + ip link set "$TAP_IN" up + + ip tuntap add dev "$TAP_OUT" mode tap 2>/dev/null || true + ip link set "$TAP_OUT" master "$INET_BRIDGE" + ip link set "$TAP_OUT" up + + ip tuntap add dev "$TAP_MGMT" mode tap 2>/dev/null || true + ip link set "$TAP_MGMT" master "$MGMT_BRIDGE" + ip link set "$TAP_MGMT" up + + qemu-system-riscv64 \ + -machine virt -m "$QEMU_MEMORY" -smp "$QEMU_CPUS" \ + -display none -serial "file:/tmp/qemu-${VM_NAME}.log" \ + -name "$VM_NAME" \ + -pidfile "/tmp/qemu-${VM_NAME}.pid" \ + -bios "$QEMU_BIOS" \ + -kernel "$QEMU_KERNEL" \ + -append "root=/dev/vda rw earlycon=sbi console=ttyS0 hugepages=${VM_HUGEPAGES} role=worker worker_id=${WORKER_ID} mgmt_if=eth2" \ + -netdev tap,id=net0,ifname="$TAP_IN",script=no,downscript=no \ + -device virtio-net-device,netdev=net0,mac="$MAC_IN" \ + -netdev tap,id=net1,ifname="$TAP_OUT",script=no,downscript=no \ + -device virtio-net-device,netdev=net1,mac="$MAC_OUT" \ + -netdev tap,id=net2,ifname="$TAP_MGMT",script=no,downscript=no \ + -device virtio-net-device,netdev=net2,mac="$MAC_MGMT" \ + -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \ + -drive "id=disk0,file=${OVERLAY},format=qcow2,if=none" -device virtio-blk-device,drive=disk0 \ + -virtfs "local,path=$SHARED_DIR,mount_tag=host_share,security_model=mapped-xattr" \ + -daemonize || + { + echo "Failed to start $VM_NAME" + exit 1 + } +} + +ip tuntap add dev "tap-ctrl" mode tap 2>/dev/null || true +ip link set "tap-ctrl" master "$MGMT_BRIDGE" +ip link set "tap-ctrl" up + +ip tuntap add dev "tap-ctrl-inet" mode tap 2>/dev/null || true +ip link set "tap-ctrl-inet" master "$INET_BRIDGE" +ip link set "tap-ctrl-inet" up + +start_controller + +sleep 3 + +start_filter_vm "filter1" "tap-f1-in" "tap-f1-out" "tap-f1-mgmt" "$BRIDGE1" "$FILTER1_MAC_IN" "$FILTER1_MAC_OUT" "$FILTER1_MAC_MGMT" 1 +start_filter_vm "filter2" "tap-f2-in" "tap-f2-out" "tap-f2-mgmt" "$BRIDGE2" "$FILTER2_MAC_IN" "$FILTER2_MAC_OUT" "$FILTER2_MAC_MGMT" 2 + +for SVC in $(docker compose ps -q 2>/dev/null); do + docker exec "$SVC" ip neigh flush all 2>/dev/null || true + docker exec "$SVC" ip -6 neigh flush all 2>/dev/null || true +done +for SVC in $(docker compose -p "$(basename "$PROJECT_DIR")" ps --format '{{.Name}}' 2>/dev/null | grep "gen-1"); do + docker exec "$SVC" arp -s "${SUBNET1}.254" "$FILTER1_MAC_IN" 2>/dev/null || true + docker exec "$SVC" ip -6 neigh del "${SUBNET1_V6}fe" dev eth0 2>/dev/null || true + docker exec "$SVC" ip -6 neigh add "${SUBNET1_V6}fe" lladdr "$FILTER1_MAC_IN" nud permanent dev eth0 2>/dev/null || true + docker exec "$SVC" ip -6 route replace default via "${SUBNET1_V6}fe" dev eth0 2>/dev/null || true +done +for SVC in $(docker compose -p "$(basename "$PROJECT_DIR")" ps --format '{{.Name}}' 2>/dev/null | grep "gen-2"); do + docker exec "$SVC" arp -s "${SUBNET2}.254" "$FILTER2_MAC_IN" 2>/dev/null || true + docker exec "$SVC" ip -6 neigh del "${SUBNET2_V6}fe" dev eth0 2>/dev/null || true + docker exec "$SVC" ip -6 neigh add "${SUBNET2_V6}fe" lladdr "$FILTER2_MAC_IN" nud permanent dev eth0 2>/dev/null || true + docker exec "$SVC" ip -6 route replace default via "${SUBNET2_V6}fe" dev eth0 2>/dev/null || true +done + +echo "Test stand is running" +echo " Filter VMs: filter1 (pid $(cat /tmp/qemu-filter1.pid 2>/dev/null)), filter2 (pid $(cat /tmp/qemu-filter2.pid 2>/dev/null))" +echo " Controller: pid $(cat /tmp/qemu-controller.pid 2>/dev/null)" +echo " Traffic generators: $(docker compose ps --format '{{.Name}}' | wc -l) containers" diff --git a/test/virtual_load_network/scripts/stop.sh b/test/virtual_load_network/scripts/stop.sh index 6abeaf5..a8712b5 100755 --- a/test/virtual_load_network/scripts/stop.sh +++ b/test/virtual_load_network/scripts/stop.sh @@ -11,14 +11,36 @@ if [ -f .env ]; then set +a fi -SUBNET="${SUBNET:-10.0.0}" -BRIDGE_IFACE="${BRIDGE_IFACE:-br-testnet}" +INET_SUBNET="${INET_SUBNET:-10.0.3}" +INET_BRIDGE="${INET_BRIDGE:-br-inet}" +MGMT_BRIDGE="${MGMT_BRIDGE:-br-mgmt}" +INET_SUBNET_V6="${INET_SUBNET_V6:-fd00:b::}" -tc qdisc del dev "$BRIDGE_IFACE" clsact 2>/dev/null -iptables -D FORWARD -i "$BRIDGE_IFACE" -j ACCEPT 2>/dev/null -iptables -D FORWARD -o "$BRIDGE_IFACE" -j ACCEPT 2>/dev/null +for VM in filter1 filter2 controller; do + PID_FILE="/tmp/qemu-${VM}.pid" + if [ -f "$PID_FILE" ]; then + kill "$(cat "$PID_FILE")" 2>/dev/null + rm -f "$PID_FILE" + fi +done docker compose down 2>/dev/null -iptables -t nat -D POSTROUTING -s ${SUBNET}.0/24 ! -d ${SUBNET}.0/24 -j MASQUERADE 2>/dev/null -ip link delete veth0 2>/dev/null +iptables -t nat -D POSTROUTING -s ${INET_SUBNET}.0/24 ! -d ${INET_SUBNET}.0/24 -j MASQUERADE 2>/dev/null +iptables -D FORWARD -i "$INET_BRIDGE" -j ACCEPT 2>/dev/null +iptables -D FORWARD -o "$INET_BRIDGE" -j ACCEPT 2>/dev/null + +ip6tables -t nat -D POSTROUTING -s ${INET_SUBNET_V6}/64 ! -d ${INET_SUBNET_V6}/64 -j MASQUERADE 2>/dev/null +ip6tables -D FORWARD -i "$INET_BRIDGE" -j ACCEPT 2>/dev/null +ip6tables -D FORWARD -o "$INET_BRIDGE" -j ACCEPT 2>/dev/null + +for TAP in tap-f1-in tap-f1-out tap-f2-in tap-f2-out tap-f1-mgmt tap-f2-mgmt tap-ctrl tap-ctrl-inet; do + ip tuntap del dev "$TAP" mode tap 2>/dev/null +done + +ip link delete "$INET_BRIDGE" 2>/dev/null +ip link delete "$MGMT_BRIDGE" 2>/dev/null + +rm -rf "$PROJECT_DIR/shared" +rm -f "$PROJECT_DIR"/filter1.qcow2 "$PROJECT_DIR"/filter2.qcow2 "$PROJECT_DIR"/controller.qcow2 +rm -f "$PROJECT_DIR"/cloud-init-*.iso diff --git a/test/virtual_load_network/scripts/traffic-gen.sh b/test/virtual_load_network/scripts/traffic-gen.sh index d8a1e54..0e61af0 100644 --- a/test/virtual_load_network/scripts/traffic-gen.sh +++ b/test/virtual_load_network/scripts/traffic-gen.sh @@ -2,13 +2,16 @@ URLS=( "http://www.google.com" - "https://github.com" + "https://youtube.com" + "https://www.tiktok.com" + "https://1xbet.com/" + "https://betboom.ru/" ) while true; do url=${URLS[$RANDOM % ${#URLS[@]}]} echo "[$(date '+%H:%M:%S')] -> request $url" - status=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "$url") + #status=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "$url") echo "[$(date '+%H:%M:%S')] <- response $url, status = $status" sleep 5 done diff --git a/wiki/using_test_stand.md b/wiki/using_test_stand.md new file mode 100644 index 0000000..9264bed --- /dev/null +++ b/wiki/using_test_stand.md @@ -0,0 +1,304 @@ +# Using Test Stand + +The test stand runs QEMU RISC-V virtual machines (controller + filter workers) with traffic generators in Docker containers. It allows testing the full filtering pipeline: traffic generation, DNS interception, domain classification via Kaspersky API, and policy-based blocking. + +## Prerequisites + +- `qemu-system-riscv64` +- Docker and Docker Compose +- Bazel + +## Network Layout + +Bridges: +* br-mgmt (10.0.2.0/24) - SSH access, gRPC communication +* br-inet (10.0.3.0/24) - Internet access (controller) +* br-testnet1 (10.0.0.0/24) - Traffic to filter1 +* br-testnet2 (10.0.1.0/24) - Traffic to filter2 + +VMs: +* filter1 (10.0.2.1) - DPDK packet filter and gRPC worker +* filter2 (10.0.2.2) - DPDK packet filter and gRPC worker +* controller (10.0.2.3) - gRPC server and Kaspersky API classifier + +## 0. Yocto Image + +The test stand requires a RISC-V image `cluster-image` and SDK from `vm_build_risc_v` + +**Image** +```bash +cd vm_build_risc_v +make build +make qemu +``` + +`make build` creates the Docker container with the build environment. + +`make qemu` runs the full Yocto build for `qemuriscv64` inside it. + +**SDK** +```bash +make qemu-sdk +``` + +The SDK installer script will appear in `vm_build_risc_v/qemu/poky/build/tmp/deploy/sdk/`. Install it in `/opt/riscv-sdk`: + +```bash +sudo vm_build_risc_v/qemu/poky/build/tmp/deploy/sdk/poky-glibc-x86_64-cluster-image-riscv64-qemuriscv64-toolchain-*.sh -d /opt/riscv-sdk +``` + +Edit `test/virtual_load_network/.env`: +```env +YOCTO_DEPLOY_DIR=/path/to/deploy/images/qemuriscv64 +``` + + +## 1. Build Controller + +```bash +cd controller +make build-riscv +``` + +The binary is placed at `controller/bin/grpc_server`. + +## 2. Build Worker + +standard assembly: +```bash +cd worker +bazel build --config=riscv64 //:worker +``` + +build with debug output: +```bash +cd worker +bazel build --config=riscv64 //:worker --copt="-DDEBUG=1" +``` + +If you experience a hang during assembly, limit the memory with a flag, for example: +```bash +cd worker +bazel build --config=riscv64 //:worker --copt="-DDEBUG=1" --local_ram_resources=4096 +``` + +The binary is placed at `worker/bazel-bin/worker`. + +## 3. Start Test Stand + +```bash +cd test/virtual_load_network/scripts +sudo ./start.sh +``` + +This will: +- Start traffic generator containers +- Create network bridges and TAP interfaces +- Launch 3 QEMU VMs: `filter1`, `filter2`, and `controller` +- Copy binaries and configs to `shared/` directory + +Expected output: +``` + => [traffic-gen-1 internal] load build definition from Dockerfile.traffic-gen 0.0s + => => transferring dockerfile: 217B 0.0s + => [traffic-gen-2 internal] load build definition from Dockerfile.traffic-gen 0.0s + => => transferring dockerfile: 217B 0.0s + => [traffic-gen-1 internal] load metadata for docker.io/library/alpine:3.20 0.7s + => [traffic-gen-2 internal] load .dockerignore 0.0s + => => transferring context: 2B 0.0s + => [traffic-gen-1 internal] load .dockerignore 0.0s + => => transferring context: 2B 0.0s + => [traffic-gen-2 1/4] FROM docker.io/library/alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc 0.0s + => => resolve docker.io/library/alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc 0.0s + => [traffic-gen-2 internal] load build context 0.0s + => => transferring context: 71B 0.0s + => [traffic-gen-1 internal] load build context 0.0s + => => transferring context: 71B 0.0s + => CACHED [traffic-gen-1 2/4] RUN apk add --no-cache curl bash iproute2 0.0s + => CACHED [traffic-gen-1 3/4] COPY ./scripts/traffic-gen.sh /entrypoint.sh 0.0s + => CACHED [traffic-gen-1 4/4] RUN chmod +x /entrypoint.sh 0.0s + => [traffic-gen-2] exporting to image 0.0s + => => exporting layers 0.0s + => => exporting manifest sha256:2a5b7514893477d98546e00304bab0c5d3df5e691685413fdcd4c5d7f8a44b8e 0.0s + => => exporting config sha256:0820b0d32a4e01e75aa6f7fb86776dcb475af2f7513593360acbecaf13110a4c 0.0s + => => exporting attestation manifest sha256:47fe7dece22aee82abbc0e0fbad91b19cf38bea269f9c2a480c8d063a4b4391d 0.0s + => => exporting manifest list sha256:26a7229340f15481c403acf5125032cb86a70fcd48358256764d944d2a97a3c9 0.0s + => => naming to docker.io/library/virtual_load_network-traffic-gen-2:latest 0.0s + => => unpacking to docker.io/library/virtual_load_network-traffic-gen-2:latest 0.0s + => [traffic-gen-1] exporting to image 0.0s + => => exporting layers 0.0s + => => exporting manifest sha256:ba6645395e7cabb7c7babca4f5ae9e76586e524ee00b385e28b34ef6741d9964 0.0s + => => exporting config sha256:ab641cf5526440f2ce75212d98a7ce2c485b1a51d4215adfe0fc720d836c5774 0.0s + => => exporting attestation manifest sha256:4fd13be17b47ec34c6b497e30cbf496c9ebd542db2d7d1d03bee83e9dc8a4148 0.0s + => => exporting manifest list sha256:3f4c8d5955f564bcbf2b33634da0edab10ca65e99e21042d2ed671601e2078e1 0.0s + => => naming to docker.io/library/virtual_load_network-traffic-gen-1:latest 0.0s + => => unpacking to docker.io/library/virtual_load_network-traffic-gen-1:latest 0.0s + => [traffic-gen-1] resolving provenance for metadata file 0.0s + => [traffic-gen-2] resolving provenance for metadata file 0.0s +[+] Running 12/12 + ✔ traffic-gen-1 Built 0.0s + ✔ traffic-gen-2 Built 0.0s + ✔ Network virtual_load_network_testnet2 Created 0.0s + ✔ Network virtual_load_network_default Created 0.0s + ✔ Network virtual_load_network_testnet1 Created 0.0s + ✔ Container virtual_load_network-traffic-gen-2-2 Started 0.3s + ✔ Container virtual_load_network-pushgateway-1 Started 0.3s + ✔ Container virtual_load_network-traffic-gen-2-1 Started 0.4s + ✔ Container virtual_load_network-traffic-gen-1-2 Started 0.3s + ✔ Container virtual_load_network-traffic-gen-1-1 Started 0.4s + ✔ Container virtual_load_network-prometheus-1 Started 0.3s + ✔ Container virtual_load_network-grafana-1 Started 0.4s +Nothing to flush +Nothing to flush +Nothing to flush +Nothing to flush +Nothing to flush +Nothing to flush +Test stand is running + Filter VMs: filter1 (pid 176929), filter2 (pid 177022) + Controller: pid 178189 + Traffic generators: 7 containers +``` + +### 1. If you need to connect via SSH + +Connect to the controller VM via SSH: +```bash +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@10.0.2.3 +``` + +Connect to the filter1 VM via SSH: +```bash +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@10.0.2.1 +``` + +Connect to the filter2 VM via SSH: +```bash +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@10.0.2.2 +``` + + +### 2. Load Filtering Policy + +Detailed instructions for administering the policy are provided in grpc_server/admin/README.md + +## Installing dependencies +```bash +pip install -r requirements.txt +``` + +## Generating proto files +```bash +python -m grpc_tools.protoc \ + --python_out=. \ + --grpc_python_out=. \ + -I . \ + admin_service.proto +``` + +## Example of a toml file with policy +```toml +[global.rules] # Глобальные правила для всех воркеров +block_categories = ["Gambling", "Weapons"] # Категории для блокировки +block_domains = ["youtube.com", "tiktok.com"] # Домены для блокировки +allow_domains = ["github.com", "stackoverflow.com"] # Разрешённые домены (приоритет над блокировкой) +block_ips = ["192.168.0.1", "2001:0db8:85a3:0000:0000:8a2e:0370:7334"] # IP для блокировки (IPv4 и IPv6) +allow_ips = ["8.8.8.8"] # Разрешённые IP +ttl_ip = 604800 # TTL кэша IP +ttl_domain = 604800 # TTL кэша доменов +min_trust_level = 5 # Мин. уровень доверия + +[global.rules.block_by_trust] # Блокировка по уровню доверия +ENTERTAINMENT = 6 +NEWS = 4 + +[filters.filter_1] # Правила для воркера #1 +block_categories = ["Weapons", "Malware"] # Доп. категории к глобальным +block_domains = ["instagram.com"] # Доп. домены к глобальным +allow_domains = ["vk.com"] # Доп. разрешённые домены +min_trust_level = 0 # Переопределяет глобальный + +[filters.filter_1.block_by_trust] # Уровни доверия для воркера #1 +SOCIAL = 8 +ENTERTAINMENT = 7 + +[filters.filter_2] # Правила для воркера #2 +block_categories = ["Malware"] # Доп. категории к глобальным +allow_domains = ["github.com", "gitlab.com"] # Доп. разрешённые домены + +``` + +### Send a new policy to the controller + +```bash +python admin.py load --file .toml +``` + +### Get the current policy from the controller + +```bash +python admin.py get --save .toml +``` + +### Enable/disable filtering on the worker + +```bash +python admin.py toggle --id 1 --on #id - worker id +python admin.py toggle --id 1 --off #id - worker id +``` + + +## 5. Monitoring + +### Conroller + +Log monitoring +```bash +cd test/virtual_load_network/shared/logs +sudo tail -f controller.log +``` + +### Worker 1 + +Log monitoring +```bash +cd test/virtual_load_network/shared/logs +sudo tail -f worker1.log +``` + +Monitoring the port on which packets are received +```bash +sudo tcpdump -i tap-f1-in port 53 +``` + +Monitoring the port from which packets are emitted +```bash +sudo tcpdump -i tap-f1-out port 53 +``` + +### Worker 2 + +Log monitoring +```bash +cd test/virtual_load_network/shared/logs +sudo tail -f worker2.log +``` + +Monitoring the port on which packets are received +```bash +sudo tcpdump -i tap-f2-in port 53 +``` + +Monitoring the port from which packets are emitted +```bash +sudo tcpdump -i tap-f2-out port 53 +``` + + +## 4. Stop Test Stand + +```bash +cd test/virtual_load_network/scripts +sudo ./stop.sh +``` + diff --git a/worker/src/dpdk_filter/main.c b/worker/src/dpdk_filter/main.c index ea54728..95eacff 100644 --- a/worker/src/dpdk_filter/main.c +++ b/worker/src/dpdk_filter/main.c @@ -36,6 +36,8 @@ void forward_to_out(struct net_port *incoming_port, } int main(int argc, char **argv) { + // for test_stand disable bufferization + setvbuf(stdout, NULL, _IOLBF, 0); // since BASE_POLICY is filled when initializing worker, let’s initialize here struct BASE_POLICY policy; if (signal(SIGINT, signal_handler) == SIG_ERR) { @@ -134,4 +136,4 @@ int main(int argc, char **argv) { net_port_destroy(port_out); net_port_destroy(port_exception); return 0; -} \ No newline at end of file +}