Add IPv6 secondary interface support (prepare + CNO configuration) OSPRH-6486

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: tusharjadhav3302

collection/stages/roles/cleanup/tasks/cleanup_ipv6_secondary.yml

@@ -0,0 +1,36 @@
+---
+- name: Check if IPv6 secondary resources exist (via resources file)
+  ansible.builtin.include_vars:
+    file: "{{ resources_file }}"
+    name: registered_resources
+  ignore_errors: true  # noqa: ignore-errors
+  register: resources_load
+
+- name: Cleanup IPv6 secondary OpenStack resources
+  when:
+    - resources_load is succeeded
+    - registered_resources.ipv6_secondary_router_name is defined
+  block:
+    - name: Detach IPv6 secondary subnets from router
+      ansible.builtin.shell: >
+        openstack router remove subnet {{ registered_resources.ipv6_secondary_router_name }} {{ item }}
+      loop: "{{ registered_resources.ipv6_secondary_subnet_ids | default([]) }}"
+      environment:
+        OS_CLOUD: "{{ user_cloud }}"
+      changed_when: true
+      ignore_errors: true  # noqa: ignore-errors
+
+    - name: Delete IPv6 secondary router
+      openstack.cloud.router:
+        cloud: "{{ user_cloud }}"
+        state: absent
+        name: "{{ registered_resources.ipv6_secondary_router_name }}"
+      ignore_errors: true  # noqa: ignore-errors
+
+    - name: Delete IPv6 secondary networks
+      openstack.cloud.network:
+        cloud: "{{ user_cloud }}"
+        state: absent
+        name: "{{ item.net_name }}"
+      loop: "{{ ipv6_secondary_networks.networks }}"
+      ignore_errors: true  # noqa: ignore-errors

collection/stages/roles/cleanup/tasks/main.yml

@@ -68,6 +68,10 @@
             - name: Discover the OpenShift installation type
               ansible.builtin.include_tasks: detect_ocp_installation.yml
 
+            - name: Cleanup IPv6 secondary network resources if they exist
+              ansible.builtin.include_tasks: cleanup_ipv6_secondary.yml
+              when: ocp_deployment_topology.secondary_ip_protocol | default('') == 'ipv6'
+
             - name: Destroy the OpenShift cluster if it exists
               ansible.builtin.include_tasks: destroy_openshift_cluster.yml
               when: existing_ocp_installation_type != ''

collection/stages/roles/day2ops/files/worker-machineconfig-dhcp.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  labels:
+    machineconfiguration.openshift.io/role: worker
+  name: 05-worker-kernelarg-dhcp
+spec:
+  config:
+    ignition:
+      version: 3.2.0
+  kernelArguments:
+    - ip=dhcp,dhcp6

collection/stages/roles/day2ops/tasks/procedures/configure_ipv6_secondary.yml

@@ -0,0 +1,176 @@
+---
+- name: Apply DHCP MachineConfig to workers
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig }}"
+    state: present
+    definition: "{{ lookup('file', '../files/worker-machineconfig-dhcp.yaml') | from_yaml }}"
+
+- name: Wait for the MCP to finish the cluster updates
+  ansible.builtin.include_role:
+    name: tools_cluster_checks
+    tasks_from: wait_mcp_updated.yml
+  vars:
+    wait_retries: 60
+    wait_delay: 60
+
+- name: Active wait until all the ClusterOperators are ready
+  ansible.builtin.include_role:
+    name: tools_cluster_checks
+    tasks_from: wait_until_cluster_operators_ready.yml
+
+- name: Wait until OCP cluster is healthy
+  ansible.builtin.include_role:
+    name: tools_cluster_checks
+    tasks_from: wait_until_cluster_is_healthy.yml
+
+- name: Get OCP worker nodes
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ kubeconfig }}"
+    api_version: v1
+    kind: Node
+    label_selectors:
+      - node-role.kubernetes.io/worker
+  register: workers
+
+- name: Store the OCP worker node names
+  ansible.builtin.set_fact:
+    ocp_workers: "{{ workers | json_query('resources[*].metadata.name') | sort }}"
+    num_workers: "{{ workers.resources | length }}"
+
+- name: Discover IPv6 interfaces on first worker
+  ansible.builtin.shell: |
+    set -o pipefail && \
+    oc adm node-logs {{ ocp_workers[0] }} | \
+    grep '{{ item.cidr | regex_replace('::/64$', '') }}' | \
+    grep dev | \
+    sed 's/.*{{ item.cidr | regex_replace('::/64$', '') }}::.* dev \(\S\+\).*/\1/g' | \
+    tail -1
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  loop: "{{ ipv6_secondary_networks.networks }}"
+  register: ipv6_interfaces_results
+  changed_when: false
+  retries: 10
+  delay: 30
+  until: ipv6_interfaces_results.stdout != ""
+
+- name: Store IPv6 interface names
+  ansible.builtin.set_fact:
+    ipv6_interfaces: "{{ ipv6_interfaces_results.results | map(attribute='stdout') | list }}"
+
+- name: Display discovered IPv6 interfaces
+  ansible.builtin.debug:
+    var: ipv6_interfaces
+
+- name: Template CNO macvlan patch
+  ansible.builtin.template:
+    src: network-macvlan.yml.j2
+    dest: "/tmp/network-macvlan.yml"
+    mode: u=rw,g=rw,o=r
+
+- name: Create OCP projects for IPv6 network testing
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig }}"
+    api_version: project.openshift.io/v1
+    kind: Project
+    name: "{{ item }}"
+    state: present
+  loop: "{{ ipv6_secondary_networks.projects }}"
+
+- name: Patch CNO with macvlan additionalNetworks
+  ansible.builtin.shell: |
+    set -o pipefail && \
+    oc patch network.operator cluster --patch "$(cat /tmp/network-macvlan.yml)" --type merge
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  changed_when: true
+
+- name: Verify NetworkAttachmentDefinition CRs exist
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ kubeconfig }}"
+    api_version: k8s.cni.cncf.io/v1
+    kind: NetworkAttachmentDefinition
+  register: network_attachments
+  until:
+    - network_attachments is defined
+    - network_attachments is not failed
+    - network_attachments | json_query('resources[*].metadata.name') | list | sort == ipv6_secondary_networks.projects | sort
+  retries: 10
+  delay: 30
+
+- name: Template IPv6 test deployments
+  ansible.builtin.template:
+    src: ipv6-deployment.yml.j2
+    dest: "/tmp/ipv6-deployment-{{ item }}.yml"
+    mode: u=rw,g=rw,o=r
+  vars:
+    ipv6_project: "{{ item }}"
+  loop: "{{ ipv6_secondary_networks.projects }}"
+
+- name: Deploy hello-openshift pods with IPv6 network annotation
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig }}"
+    state: present
+    src: "/tmp/ipv6-deployment-{{ item }}.yml"
+    wait: true
+    wait_timeout: 300
+    namespace: "{{ item }}"
+  loop: "{{ ipv6_secondary_networks.projects }}"
+
+- name: Wait for pods to be ready in each IPv6 namespace
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ kubeconfig }}"
+    kind: Pod
+    namespace: "{{ item }}"
+    label_selectors:
+      - app=hello-openshift
+    field_selectors:
+      - status.phase=Running
+  register: running_pods
+  until: running_pods.resources | length == num_workers | int
+  retries: 20
+  delay: 15
+  loop: "{{ ipv6_secondary_networks.projects }}"
+
+- name: Get IPv6 network IDs for port security operations
+  openstack.cloud.networks_info:
+    cloud: "{{ user_cloud }}"
+  register: all_openstack_networks
+
+- name: Build list of IPv6 secondary network IDs
+  ansible.builtin.set_fact:
+    ipv6_net_ids: "{{ all_openstack_networks.networks | selectattr('name', 'in', ipv6_secondary_networks.networks | map(attribute='net_name') | list) | map(attribute='id') | list }}"
+
+- name: Get worker ports on IPv6 secondary networks
+  openstack.cloud.port_info:
+    cloud: "{{ user_cloud }}"
+    filters:
+      device_owner: compute:nova
+  register: all_compute_ports
+
+- name: Filter worker ports on IPv6 networks
+  ansible.builtin.set_fact:
+    ipv6_worker_ports: "{{ all_compute_ports.ports | selectattr('network_id', 'in', ipv6_net_ids) | list }}"
+
+- name: Disable port security on worker IPv6 ports
+  openstack.cloud.port:
+    cloud: "{{ user_cloud }}"
+    state: present
+    name: "{{ item.id }}"
+    port_security_enabled: false
+    no_security_groups: true
+  loop: "{{ ipv6_worker_ports }}"
+  loop_control:
+    label: "{{ item.id }}"
+
+- name: Verify pod-to-pod IPv6 connectivity on each network
+  ansible.builtin.include_tasks: procedures/verify_ipv6_connectivity.yml
+  loop: "{{ ipv6_secondary_networks.projects }}"
+  loop_control:
+    loop_var: ipv6_namespace
+
+- name: Verify external IPv6 reachability from worker nodes
+  ansible.builtin.include_tasks: procedures/verify_ipv6_external_reachability.yml
+  loop: "{{ ipv6_secondary_networks.projects }}"
+  loop_control:
+    loop_var: ipv6_namespace

collection/stages/roles/day2ops/tasks/procedures/verify_ipv6_connectivity.yml

@@ -0,0 +1,55 @@
+---
+- name: Get pods in namespace {{ ipv6_namespace }}
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ kubeconfig }}"
+    kind: Pod
+    namespace: "{{ ipv6_namespace }}"
+    label_selectors:
+      - app=hello-openshift
+    field_selectors:
+      - status.phase=Running
+  register: ipv6_pods
+
+- name: Store pod names for {{ ipv6_namespace }}
+  ansible.builtin.set_fact:
+    ipv6_pod_names: "{{ ipv6_pods.resources | map(attribute='metadata.name') | list }}"
+
+- name: Get pod IPv6 addresses on net1 interface
+  ansible.builtin.shell: |
+    set -o pipefail && \
+    oc exec {{ item }} -n {{ ipv6_namespace }} -- ip -6 addr show dev net1 scope global | \
+    awk '/inet6/{print $2}' | cut -f1 -d'/'
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  loop: "{{ ipv6_pod_names }}"
+  register: pod_ipv6_results
+  changed_when: false
+  retries: 20
+  delay: 30
+  until: pod_ipv6_results.stdout != ""
+
+- name: Build pod name to IPv6 address mapping
+  ansible.builtin.set_fact:
+    ipv6_pod_ips: "{{ ipv6_pod_ips | default({}) | combine({item.item: item.stdout}) }}"
+  loop: "{{ pod_ipv6_results.results }}"
+  loop_control:
+    label: "{{ item.item }}"
+
+- name: Display pod IPv6 addresses for {{ ipv6_namespace }}
+  ansible.builtin.debug:
+    var: ipv6_pod_ips
+
+- name: Check pod-to-pod IPv6 connectivity in {{ ipv6_namespace }}
+  ansible.builtin.shell: |
+    oc exec {{ item[0] }} -n {{ ipv6_namespace }} -- /bin/curl -s --connect-timeout 10 http://[{{ item[1] }}]:8080
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  with_nested:
+    - "{{ ipv6_pod_ips.keys() | list }}"
+    - "{{ ipv6_pod_ips.values() | list }}"
+  when: ipv6_pod_ips[item[0]] != item[1]
+  register: connectivity_check
+  changed_when: false
+  retries: 5
+  delay: 10
+  until: connectivity_check.stdout is search('Hello OpenShift!')

collection/stages/roles/day2ops/tasks/procedures/verify_ipv6_external_reachability.yml

@@ -0,0 +1,41 @@
+---
+- name: Get pods in namespace {{ ipv6_namespace }} for external reachability test
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ kubeconfig }}"
+    kind: Pod
+    namespace: "{{ ipv6_namespace }}"
+    label_selectors:
+      - app=hello-openshift
+    field_selectors:
+      - status.phase=Running
+  register: ipv6_ext_pods
+
+- name: Store pod names for external test
+  ansible.builtin.set_fact:
+    ipv6_ext_pod_names: "{{ ipv6_ext_pods.resources | map(attribute='metadata.name') | list }}"
+
+- name: Get pod IPv6 addresses for external test
+  ansible.builtin.shell: |
+    set -o pipefail && \
+    oc exec {{ item }} -n {{ ipv6_namespace }} -- ip -6 addr show dev net1 scope global | \
+    awk '/inet6/{print $2}' | cut -f1 -d'/'
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  loop: "{{ ipv6_ext_pod_names }}"
+  register: ext_pod_ipv6_results
+  changed_when: false
+
+- name: Verify external IPv6 reachability from worker node to pods in {{ ipv6_namespace }}
+  ansible.builtin.shell: |
+    oc debug node/{{ ocp_workers[0] }} -- chroot /host curl -s --connect-timeout 10 http://[{{ item.stdout }}]:8080
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  loop: "{{ ext_pod_ipv6_results.results }}"
+  loop_control:
+    label: "{{ item.item }}"
+  when: item.stdout != ""
+  register: external_check
+  changed_when: false
+  retries: 5
+  delay: 10
+  until: external_check.stdout is search('Hello OpenShift!')

collection/stages/roles/day2ops/templates/ipv6-deployment.yml.j2

@@ -0,0 +1,41 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hello-openshift
+  namespace: {{ ipv6_project }}
+spec:
+  replicas: {{ num_workers }}
+  selector:
+    matchLabels:
+      app: hello-openshift
+  template:
+    metadata:
+      labels:
+        app: hello-openshift
+      annotations:
+        k8s.v1.cni.cncf.io/networks: "{{ ipv6_project }}"
+    spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - hello-openshift
+              topologyKey: kubernetes.io/hostname
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: hello-openshift
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+          image: quay.io/openshift/origin-hello-openshift
+          ports:
+            - containerPort: 8080

collection/stages/roles/day2ops/templates/network-macvlan.yml.j2

@@ -0,0 +1,14 @@
+spec:
+  additionalNetworks:
+{% for network in ipv6_secondary_networks.networks %}
+  - name: {{ network.net_name }}
+    namespace: {{ network.net_name }}
+    rawCNIConfig: |-
+      {
+        "cniVersion": "0.3.1",
+        "name": "{{ network.net_name }}",
+        "type": "macvlan",
+        "master": "{{ ipv6_interfaces[loop.index0] }}"
+      }
+    type: Raw
+{% endfor %}