From ec46a23651c14f8c3e19fc04de95e6eca80a8210 Mon Sep 17 00:00:00 2001 From: michaelkedar Date: Thu, 20 Aug 2026 05:35:21 +0000 Subject: [PATCH 1/2] remove worker pool --- .../oss-vdb-test/workers-cves.yaml | 6 --- .../oss-vdb-test/workers-reimport.yaml | 6 --- .../environments/oss-vdb-test/workers.yaml | 6 --- .../environments/private/workers.yaml | 6 --- .../terraform/modules/osv/workers_gke.tf | 36 ------------------ .../terraform/modules/osv_pipeline/gke.tf | 38 ------------------- 6 files changed, 98 deletions(-) diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/workers-cves.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/workers-cves.yaml index 4120fbb8ac2..0fc4d0b6d7a 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/workers-cves.yaml +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/workers-cves.yaml @@ -5,12 +5,6 @@ metadata: spec: template: spec: - tolerations: - - key: workloadType - operator: Equal - value: worker-pool - nodeSelector: - workloadType: worker-pool containers: - name: workers-cves env: diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/workers-reimport.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/workers-reimport.yaml index 6b5668aeb73..d9023153eb8 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/workers-reimport.yaml +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/workers-reimport.yaml @@ -5,12 +5,6 @@ metadata: spec: template: spec: - tolerations: - - key: workloadType - operator: Equal - value: worker-pool - nodeSelector: - workloadType: worker-pool containers: - name: workers-reimport env: diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/workers.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/workers.yaml index cd557083016..7e81f3b9727 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/workers.yaml +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/workers.yaml @@ -5,12 +5,6 @@ metadata: spec: template: spec: - tolerations: - - key: workloadType - operator: Equal - value: worker-pool - nodeSelector: - workloadType: worker-pool containers: - name: worker-private env: diff --git a/deployment/clouddeploy/gke-workers/environments/private/workers.yaml b/deployment/clouddeploy/gke-workers/environments/private/workers.yaml index 6284f00f470..4e07357be3e 100644 --- a/deployment/clouddeploy/gke-workers/environments/private/workers.yaml +++ b/deployment/clouddeploy/gke-workers/environments/private/workers.yaml @@ -5,12 +5,6 @@ metadata: spec: template: spec: - tolerations: - - key: workloadType - operator: Equal - value: worker-pool - nodeSelector: - workloadType: worker-pool containers: - name: worker-private env: diff --git a/deployment/terraform/modules/osv/workers_gke.tf b/deployment/terraform/modules/osv/workers_gke.tf index 512185ef566..120fcfe0f01 100644 --- a/deployment/terraform/modules/osv/workers_gke.tf +++ b/deployment/terraform/modules/osv/workers_gke.tf @@ -118,42 +118,6 @@ resource "google_container_node_pool" "highend" { } } -resource "google_container_node_pool" "worker_pool_temp" { - count = var.project_id == "oss-vdb-test" ? 1 : 0 - project = var.project_id - name = "worker-pool-temp" - cluster = google_container_cluster.workers.name - location = google_container_cluster.workers.location - - lifecycle { - replace_triggered_by = [ - google_container_cluster.workers.id, - ] - } - - autoscaling { - min_node_count = 0 - max_node_count = 250 - location_policy = "BALANCED" - } - - node_config { - machine_type = "n4-highcpu-8" - disk_type = "hyperdisk-balanced" - - oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"] - - labels = { - workloadType = "worker-pool" - } - taint { - effect = "NO_EXECUTE" - key = "workloadType" - value = "worker-pool" - } - } -} - # Service account permissions data "google_compute_default_service_account" "default" { project = var.project_id diff --git a/deployment/terraform/modules/osv_pipeline/gke.tf b/deployment/terraform/modules/osv_pipeline/gke.tf index 1583796028a..692d62b8466 100644 --- a/deployment/terraform/modules/osv_pipeline/gke.tf +++ b/deployment/terraform/modules/osv_pipeline/gke.tf @@ -117,44 +117,6 @@ resource "google_container_node_pool" "highend" { } } -resource "google_container_node_pool" "worker_pool" { - project = var.project_id - name = "worker-pool" - cluster = google_container_cluster.workers.name - location = google_container_cluster.workers.location - - lifecycle { - replace_triggered_by = [ - google_container_cluster.workers.id, - ] - } - - autoscaling { - min_node_count = 0 - max_node_count = 250 - location_policy = "BALANCED" - } - - node_config { - service_account = google_service_account.worker_sa.email - machine_type = "n4-highcpu-8" - disk_type = "hyperdisk-balanced" - - oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"] - - labels = { - workloadType = "worker-pool" - } - taint { - effect = "NO_EXECUTE" - key = "workloadType" - value = "worker-pool" - } - } -} - - - # 6TiB SSD disk used by the gitter caching service resource "google_compute_disk" "gitter_disk" { project = var.project_id From dbf2738a3026e491bc65c6d38a0e5606f57e518b Mon Sep 17 00:00:00 2001 From: michaelkedar Date: Thu, 20 Aug 2026 06:45:34 +0000 Subject: [PATCH 2/2] move buncha terraform stuff --- .../environments/oss-vdb-test/main.tf | 41 +++-- .../environments/oss-vdb-test/moved.tf | 100 ++++++++++++ .../terraform/environments/oss-vdb/main.tf | 54 ++++--- .../terraform/environments/oss-vdb/moved.tf | 105 ++++++++++++ deployment/terraform/modules/osv/main.tf | 93 +++-------- deployment/terraform/modules/osv/outputs.tf | 21 +-- .../terraform/modules/osv/pubsub_tasks.tf | 111 ------------- deployment/terraform/modules/osv/variables.tf | 27 ---- .../terraform/modules/osv/workers_gke.tf | 151 ------------------ .../terraform/modules/osv/workers_network.tf | 63 -------- .../modules/osv_pipeline/database.tf | 20 +++ .../terraform/modules/osv_pipeline/gke.tf | 8 + .../terraform/modules/osv_pipeline/network.tf | 18 +++ .../terraform/modules/osv_pipeline/outputs.tf | 69 ++++++++ .../modules/osv_pipeline/pubsub_tasks.tf | 11 ++ .../modules/osv_pipeline/variables.tf | 12 ++ 16 files changed, 418 insertions(+), 486 deletions(-) create mode 100644 deployment/terraform/environments/oss-vdb-test/moved.tf create mode 100644 deployment/terraform/environments/oss-vdb/moved.tf delete mode 100644 deployment/terraform/modules/osv/pubsub_tasks.tf delete mode 100644 deployment/terraform/modules/osv/workers_gke.tf delete mode 100644 deployment/terraform/modules/osv/workers_network.tf create mode 100644 deployment/terraform/modules/osv_pipeline/outputs.tf diff --git a/deployment/terraform/environments/oss-vdb-test/main.tf b/deployment/terraform/environments/oss-vdb-test/main.tf index c63e2f3aed7..17a8d92a24a 100644 --- a/deployment/terraform/environments/oss-vdb-test/main.tf +++ b/deployment/terraform/environments/oss-vdb-test/main.tf @@ -26,26 +26,14 @@ locals { } } -module "osv_test" { - source = "../../modules/osv" - - project_id = "oss-vdb-test" +module "osv_pipeline" { + source = "../../modules/osv_pipeline" - public_import_logs_bucket = "osv-test-public-import-logs" + project_id = "oss-vdb-test" vulnerabilities_export_bucket = "osv-test-vulnerabilities" - logs_bucket = "osv-test-logs" - cve_osv_conversion_bucket = "osv-test-cve-osv-conversion" - debian_osv_conversion_bucket = "osv-test-debian-osv" - osv_dev_sitemap_bucket = "test-osv-dev-sitemap" - backups_bucket = "osv-test-backup" - backups_bucket_retention_days = 5 affected_commits_backups_bucket = "osv-test-affected-commits" affected_commits_backups_bucket_retention_days = 2 - gcs_log_dir = "gs://oss-vdb-tf/apply-logs" - - website_domain = "test.osv.dev" - api_url = "api.test.osv.dev" - esp_version = "2.55.3" + logs_bucket = "osv-test-logs" extra_work_pools = [ "reimport", @@ -53,6 +41,25 @@ module "osv_test" { ] } +module "osv_test" { + source = "../../modules/osv" + + project_id = "oss-vdb-test" + + public_import_logs_bucket = "osv-test-public-import-logs" + logs_bucket = "osv-test-logs" + cve_osv_conversion_bucket = "osv-test-cve-osv-conversion" + debian_osv_conversion_bucket = "osv-test-debian-osv" + osv_dev_sitemap_bucket = "test-osv-dev-sitemap" + backups_bucket = "osv-test-backup" + backups_bucket_retention_days = 5 + gcs_log_dir = "gs://oss-vdb-tf/apply-logs" + + website_domain = "test.osv.dev" + api_url = "api.test.osv.dev" + esp_version = "2.55.3" +} + module "k8s_cron_alert" { for_each = local.kube_manifests source = "../../modules/k8s_cron_alert" @@ -63,7 +70,7 @@ module "k8s_cron_alert" { } import { - to = module.osv_test.google_firestore_database.datastore + to = module.osv_pipeline.google_firestore_database.datastore id = "oss-vdb-test/(default)" } diff --git a/deployment/terraform/environments/oss-vdb-test/moved.tf b/deployment/terraform/environments/oss-vdb-test/moved.tf new file mode 100644 index 00000000000..7e9fe84d238 --- /dev/null +++ b/deployment/terraform/environments/oss-vdb-test/moved.tf @@ -0,0 +1,100 @@ +# Migration of pipeline resources from module.osv_test to module.osv_pipeline + +# Datastore & Storage +moved { + from = module.osv_test.google_firestore_database.datastore + to = module.osv_pipeline.google_firestore_database.datastore +} + +moved { + from = module.osv_test.google_storage_bucket.osv_vulnerabilities_export + to = module.osv_pipeline.google_storage_bucket.osv_vulnerabilities_export +} + +moved { + from = module.osv_test.google_storage_bucket.affected_commits_backups_bucket + to = module.osv_pipeline.google_storage_bucket.affected_commits_backups_bucket +} + +# GKE Cluster & Node Pools & Gitter Disk +moved { + from = module.osv_test.google_container_cluster.workers + to = module.osv_pipeline.google_container_cluster.workers +} + +moved { + from = module.osv_test.google_container_node_pool.default_pool + to = module.osv_pipeline.google_container_node_pool.default_pool +} + +moved { + from = module.osv_test.google_container_node_pool.highend + to = module.osv_pipeline.google_container_node_pool.highend +} + +moved { + from = module.osv_test.google_compute_disk.gitter_disk + to = module.osv_pipeline.google_compute_disk.gitter_disk +} + +# Network +moved { + from = module.osv_test.google_compute_subnetwork.my_subnet_0 + to = module.osv_pipeline.google_compute_subnetwork.my_subnet_0 +} + +moved { + from = module.osv_test.google_compute_router.router + to = module.osv_pipeline.google_compute_router.router +} + +moved { + from = module.osv_test.google_compute_router_nat.nat_config + to = module.osv_pipeline.google_compute_router_nat.nat_config +} + +# Pub/Sub +moved { + from = module.osv_test.google_pubsub_topic.tasks + to = module.osv_pipeline.google_pubsub_topic.tasks +} + +moved { + from = module.osv_test.google_pubsub_topic.failed_tasks + to = module.osv_pipeline.google_pubsub_topic.failed_tasks +} + +moved { + from = module.osv_test.google_pubsub_subscription.default_work + to = module.osv_pipeline.google_pubsub_subscription.default_work +} + +moved { + from = module.osv_test.google_pubsub_subscription.work_pools + to = module.osv_pipeline.google_pubsub_subscription.work_pools +} + +moved { + from = module.osv_test.google_project_service_identity.pubsub + to = module.osv_pipeline.google_project_service_identity.pubsub +} + +moved { + from = module.osv_test.google_pubsub_subscription_iam_member.default_work_service_subscriber + to = module.osv_pipeline.google_pubsub_subscription_iam_member.default_work_service_subscriber +} + +moved { + from = module.osv_test.google_pubsub_topic_iam_member.failed_tasks_service_publisher + to = module.osv_pipeline.google_pubsub_topic_iam_member.failed_tasks_service_publisher +} + +moved { + from = module.osv_test.google_pubsub_subscription.recovery + to = module.osv_pipeline.google_pubsub_subscription.recovery +} + +moved { + from = module.osv_test.google_pubsub_subscription_iam_member.recovery_service_subscriber + to = module.osv_pipeline.google_pubsub_subscription_iam_member.recovery_service_subscriber +} diff --git a/deployment/terraform/environments/oss-vdb/main.tf b/deployment/terraform/environments/oss-vdb/main.tf index 2375bd7c46f..2fa32683e8d 100644 --- a/deployment/terraform/environments/oss-vdb/main.tf +++ b/deployment/terraform/environments/oss-vdb/main.tf @@ -26,26 +26,14 @@ locals { } } -module "osv" { - source = "../../modules/osv" - - project_id = "oss-vdb" +module "osv_pipeline" { + source = "../../modules/osv_pipeline" - public_import_logs_bucket = "osv-public-import-logs" + project_id = "oss-vdb" vulnerabilities_export_bucket = "osv-vulnerabilities" - cve_osv_conversion_bucket = "cve-osv-conversion" - debian_osv_conversion_bucket = "debian-osv" - logs_bucket = "osv-logs" - osv_dev_sitemap_bucket = "osv-dev-sitemap" - backups_bucket = "osv-backup" - backups_bucket_retention_days = 60 affected_commits_backups_bucket = "osv-affected-commits" affected_commits_backups_bucket_retention_days = 3 - gcs_log_dir = "gs://oss-vdb-tf/apply-logs" - - website_domain = "osv.dev" - api_url = "api.osv.dev" - esp_version = "2.55.3" + logs_bucket = "osv-logs" extra_work_pools = [ "reimport", @@ -55,18 +43,32 @@ module "osv" { create_oss_fuzz_subnet = true } +module "osv" { + source = "../../modules/osv" + + project_id = "oss-vdb" + + public_import_logs_bucket = "osv-public-import-logs" + cve_osv_conversion_bucket = "cve-osv-conversion" + debian_osv_conversion_bucket = "debian-osv" + logs_bucket = "osv-logs" + osv_dev_sitemap_bucket = "osv-dev-sitemap" + backups_bucket = "osv-backup" + backups_bucket_retention_days = 60 + gcs_log_dir = "gs://oss-vdb-tf/apply-logs" + + website_domain = "osv.dev" + api_url = "api.osv.dev" + esp_version = "2.55.3" +} + module "oss_fuzz" { source = "../../modules/oss_fuzz" project_id = "oss-vdb" - tasks_topic_id = module.osv.tasks_topic_id - failed_tasks_topic_id = module.osv.failed_tasks_topic_id - pubsub_service_account_email = module.osv.pubsub_service_account_email - subnetwork = module.osv.oss_fuzz_subnet_self_link -} - -moved { - from = module.oss_fuzz.google_compute_subnetwork.oss_fuzz_subnet - to = module.osv.google_compute_subnetwork.oss_fuzz_subnet[0] + tasks_topic_id = module.osv_pipeline.tasks_topic_id + failed_tasks_topic_id = module.osv_pipeline.failed_tasks_topic_id + pubsub_service_account_email = module.osv_pipeline.pubsub_service_account_email + subnetwork = module.osv_pipeline.oss_fuzz_subnet_self_link } module "k8s_cron_alert" { @@ -79,7 +81,7 @@ module "k8s_cron_alert" { } import { - to = module.osv.google_firestore_database.datastore + to = module.osv_pipeline.google_firestore_database.datastore id = "oss-vdb/(default)" } diff --git a/deployment/terraform/environments/oss-vdb/moved.tf b/deployment/terraform/environments/oss-vdb/moved.tf new file mode 100644 index 00000000000..9dc1c5519d9 --- /dev/null +++ b/deployment/terraform/environments/oss-vdb/moved.tf @@ -0,0 +1,105 @@ +# Migration of pipeline resources from module.osv to module.osv_pipeline + +# Datastore & Storage +moved { + from = module.osv.google_firestore_database.datastore + to = module.osv_pipeline.google_firestore_database.datastore +} + +moved { + from = module.osv.google_storage_bucket.osv_vulnerabilities_export + to = module.osv_pipeline.google_storage_bucket.osv_vulnerabilities_export +} + +moved { + from = module.osv.google_storage_bucket.affected_commits_backups_bucket + to = module.osv_pipeline.google_storage_bucket.affected_commits_backups_bucket +} + +# GKE Cluster & Node Pools & Gitter Disk +moved { + from = module.osv.google_container_cluster.workers + to = module.osv_pipeline.google_container_cluster.workers +} + +moved { + from = module.osv.google_container_node_pool.default_pool + to = module.osv_pipeline.google_container_node_pool.default_pool +} + +moved { + from = module.osv.google_container_node_pool.highend + to = module.osv_pipeline.google_container_node_pool.highend +} + +moved { + from = module.osv.google_compute_disk.gitter_disk + to = module.osv_pipeline.google_compute_disk.gitter_disk +} + +# Network +moved { + from = module.osv.google_compute_subnetwork.my_subnet_0 + to = module.osv_pipeline.google_compute_subnetwork.my_subnet_0 +} + +moved { + from = module.osv.google_compute_router.router + to = module.osv_pipeline.google_compute_router.router +} + +moved { + from = module.osv.google_compute_router_nat.nat_config + to = module.osv_pipeline.google_compute_router_nat.nat_config +} + +moved { + from = module.osv.google_compute_subnetwork.oss_fuzz_subnet[0] + to = module.osv_pipeline.google_compute_subnetwork.oss_fuzz_subnet[0] +} + +# Pub/Sub +moved { + from = module.osv.google_pubsub_topic.tasks + to = module.osv_pipeline.google_pubsub_topic.tasks +} + +moved { + from = module.osv.google_pubsub_topic.failed_tasks + to = module.osv_pipeline.google_pubsub_topic.failed_tasks +} + +moved { + from = module.osv.google_pubsub_subscription.default_work + to = module.osv_pipeline.google_pubsub_subscription.default_work +} + +moved { + from = module.osv.google_pubsub_subscription.work_pools + to = module.osv_pipeline.google_pubsub_subscription.work_pools +} + +moved { + from = module.osv.google_project_service_identity.pubsub + to = module.osv_pipeline.google_project_service_identity.pubsub +} + +moved { + from = module.osv.google_pubsub_subscription_iam_member.default_work_service_subscriber + to = module.osv_pipeline.google_pubsub_subscription_iam_member.default_work_service_subscriber +} + +moved { + from = module.osv.google_pubsub_topic_iam_member.failed_tasks_service_publisher + to = module.osv_pipeline.google_pubsub_topic_iam_member.failed_tasks_service_publisher +} + +moved { + from = module.osv.google_pubsub_subscription.recovery + to = module.osv_pipeline.google_pubsub_subscription.recovery +} + +moved { + from = module.osv.google_pubsub_subscription_iam_member.recovery_service_subscriber + to = module.osv_pipeline.google_pubsub_subscription_iam_member.recovery_service_subscriber +} diff --git a/deployment/terraform/modules/osv/main.tf b/deployment/terraform/modules/osv/main.tf index 2643ddf2a5a..c1fff9876b8 100644 --- a/deployment/terraform/modules/osv/main.tf +++ b/deployment/terraform/modules/osv/main.tf @@ -1,18 +1,5 @@ # osv.dev terraform configuration -# Datastore -resource "google_firestore_database" "datastore" { - project = var.project_id - name = "(default)" - location_id = "us-west2" - type = "DATASTORE_MODE" - - lifecycle { - # Destroying the resource doesn't seem to delete the database - prevent_destroy = true - } -} - # MemoryStore resource "google_redis_instance" "frontend" { project = var.project_id @@ -97,46 +84,6 @@ resource "google_storage_bucket" "osv_public_import_logs" { } } -resource "google_storage_bucket" "osv_vulnerabilities_export" { - project = var.project_id - name = var.vulnerabilities_export_bucket - location = "US" - uniform_bucket_level_access = true - - versioning { - enabled = true - } - - lifecycle_rule { - action { - type = "Delete" - } - condition { - num_newer_versions = 673 - with_state = "ARCHIVED" - } - } - - lifecycle_rule { - action { - type = "Delete" - } - condition { - days_since_noncurrent_time = 7 - with_state = "ANY" - } - } - - lifecycle { - prevent_destroy = true - } - - logging { - log_bucket = var.logs_bucket - log_object_prefix = "osv-vulnerabilities" - } -} - resource "google_storage_bucket" "cve_osv_conversion" { project = var.project_id name = var.cve_osv_conversion_bucket @@ -188,24 +135,6 @@ resource "google_storage_bucket" "backups_bucket" { } } -resource "google_storage_bucket" "affected_commits_backups_bucket" { - project = var.project_id - name = var.affected_commits_backups_bucket - location = "US" - uniform_bucket_level_access = true - lifecycle_rule { - action { - type = "Delete" - } - condition { - age = var.affected_commits_backups_bucket_retention_days - } - } - lifecycle { - prevent_destroy = true - } -} - resource "google_storage_bucket" "osv_dev_sitemap_bucket" { project = var.project_id name = var.osv_dev_sitemap_bucket @@ -217,7 +146,29 @@ resource "google_storage_bucket" "osv_dev_sitemap_bucket" { } } +# Pub/Sub topics specific to public OSV +resource "google_pubsub_topic" "pypi_bridge" { + project = var.project_id + name = "pypi-bridge" +} + # Service account permissions +data "google_compute_default_service_account" "default" { + project = var.project_id +} + +resource "google_project_iam_member" "compute_service" { + project = var.project_id + role = "roles/editor" + member = "serviceAccount:${data.google_compute_default_service_account.default.email}" +} + +resource "google_project_iam_member" "compute_service_datastore" { + project = var.project_id + role = "roles/datastore.importExportAdmin" + member = "serviceAccount:${data.google_compute_default_service_account.default.email}" +} + resource "google_service_account" "deployment_service" { project = var.project_id account_id = "deployment" diff --git a/deployment/terraform/modules/osv/outputs.tf b/deployment/terraform/modules/osv/outputs.tf index 4206b45ce00..b698c83610d 100644 --- a/deployment/terraform/modules/osv/outputs.tf +++ b/deployment/terraform/modules/osv/outputs.tf @@ -1,24 +1,5 @@ output "project_id" { - value = var.project_id # Assuming you are using a variable for project_id within the module + value = var.project_id description = "The Google Cloud Project ID" } -output "tasks_topic_id" { - value = google_pubsub_topic.tasks.id - description = "The ID of the tasks Pub/Sub topic" -} - -output "failed_tasks_topic_id" { - value = google_pubsub_topic.failed_tasks.id - description = "The ID of the failed-tasks Pub/Sub topic" -} - -output "pubsub_service_account_email" { - value = google_project_service_identity.pubsub.email - description = "The email of the Pub/Sub service account" -} - -output "oss_fuzz_subnet_self_link" { - value = one(google_compute_subnetwork.oss_fuzz_subnet[*].self_link) - description = "The self link of the OSS-Fuzz subnetwork" -} diff --git a/deployment/terraform/modules/osv/pubsub_tasks.tf b/deployment/terraform/modules/osv/pubsub_tasks.tf deleted file mode 100644 index 07542116351..00000000000 --- a/deployment/terraform/modules/osv/pubsub_tasks.tf +++ /dev/null @@ -1,111 +0,0 @@ -# Pub/Sub worker tasks topics - -resource "google_pubsub_topic" "tasks" { - project = var.project_id - name = "tasks" - - labels = { - goog-dm = "pubsub" - } - - lifecycle { - prevent_destroy = true - } -} - -resource "google_pubsub_topic" "failed_tasks" { - project = var.project_id - name = "failed-tasks" -} - -resource "google_pubsub_subscription" "default_work" { - project = var.project_id - name = "default" - topic = google_pubsub_topic.tasks.id - message_retention_duration = "604800s" - ack_deadline_seconds = 600 - - dead_letter_policy { - dead_letter_topic = google_pubsub_topic.failed_tasks.id - max_delivery_attempts = 5 - } - - expiration_policy { - ttl = "" # never expires - } - - labels = { - goog-dm = "pubsub" - } - - filter = "attributes.work_pool = \"default\"" -} - -resource "google_pubsub_subscription" "work_pools" { - for_each = toset(var.extra_work_pools) - project = var.project_id - name = each.value - topic = google_pubsub_topic.tasks.id - message_retention_duration = "604800s" - ack_deadline_seconds = 600 - - dead_letter_policy { - dead_letter_topic = google_pubsub_topic.failed_tasks.id - max_delivery_attempts = 5 - } - - expiration_policy { - ttl = "" # never expires - } - - labels = { - goog-dm = "pubsub" - } - - filter = "attributes.work_pool = \"${each.value}\"" -} - -resource "google_pubsub_topic" "pypi_bridge" { - project = var.project_id - name = "pypi-bridge" -} - -# Service account permissions -resource "google_project_service_identity" "pubsub" { - provider = google-beta - project = var.project_id - service = "pubsub.googleapis.com" -} - -resource "google_pubsub_subscription_iam_member" "default_work_service_subscriber" { - project = var.project_id - subscription = google_pubsub_subscription.default_work.name - role = "roles/pubsub.subscriber" - member = "serviceAccount:${google_project_service_identity.pubsub.email}" -} - -resource "google_pubsub_topic_iam_member" "failed_tasks_service_publisher" { - project = var.project_id - topic = google_pubsub_topic.failed_tasks.name - role = "roles/pubsub.publisher" - member = "serviceAccount:${google_project_service_identity.pubsub.email}" -} - -resource "google_pubsub_subscription" "recovery" { - project = var.project_id - name = "recovery" - topic = google_pubsub_topic.failed_tasks.id - message_retention_duration = "604800s" # 7 days - ack_deadline_seconds = 600 - - expiration_policy { - ttl = "" # never expires - } -} - -resource "google_pubsub_subscription_iam_member" "recovery_service_subscriber" { - project = var.project_id - subscription = google_pubsub_subscription.recovery.name - role = "roles/pubsub.subscriber" - member = "serviceAccount:${google_project_service_identity.pubsub.email}" -} diff --git a/deployment/terraform/modules/osv/variables.tf b/deployment/terraform/modules/osv/variables.tf index 63ed0149d75..14d2a04ebb6 100644 --- a/deployment/terraform/modules/osv/variables.tf +++ b/deployment/terraform/modules/osv/variables.tf @@ -8,11 +8,6 @@ variable "public_import_logs_bucket" { description = "Name of bucket to write importer logs to." } -variable "vulnerabilities_export_bucket" { - type = string - description = "Name of bucket to export vulnerabilities to." -} - variable "logs_bucket" { type = string description = "Name of bucket to export logs to." @@ -28,16 +23,6 @@ variable "backups_bucket_retention_days" { description = "Number of days to retain osv backups" } -variable "affected_commits_backups_bucket" { - type = string - description = "Name of bucket to backup osv AffectedCommits entries to." -} - -variable "affected_commits_backups_bucket_retention_days" { - type = number - description = "Number of days to retain osv AffectedCommits backups" -} - variable "cve_osv_conversion_bucket" { type = string description = "Name of bucket to store converted CVEs in." @@ -68,15 +53,3 @@ variable "website_domain" { description = "Domain to serve the OSV website on. Domain ownership and DNS settings must be manually configured." } -variable "extra_work_pools" { - type = list(string) - description = "Additional Pub/Sub worker pool subscriptions to create" - default = [] -} - -variable "create_oss_fuzz_subnet" { - type = bool - description = "Whether to create the OSS-Fuzz subnetwork and add it to NAT." - default = false -} - diff --git a/deployment/terraform/modules/osv/workers_gke.tf b/deployment/terraform/modules/osv/workers_gke.tf deleted file mode 100644 index 120fcfe0f01..00000000000 --- a/deployment/terraform/modules/osv/workers_gke.tf +++ /dev/null @@ -1,151 +0,0 @@ -# GKE "workers" cluster and node pools - -resource "google_container_cluster" "workers" { - project = var.project_id - name = "workers" - location = "us-central1-f" - subnetwork = google_compute_subnetwork.my_subnet_0.self_link - - private_cluster_config { - enable_private_endpoint = false - enable_private_nodes = true - master_ipv4_cidr_block = "172.16.0.32/28" - } - - # We need to define this for private clusters, but all fields are optional. - ip_allocation_policy {} - - addons_config { - gce_persistent_disk_csi_driver_config { - enabled = true - } - gcp_filestore_csi_driver_config { - enabled = true - } - } - - # We can't create a cluster with no node pool defined, but we want to only use - # separately managed node pools. So we create the smallest possible default - # node pool and immediately delete it. - remove_default_node_pool = true - initial_node_count = 1 - lifecycle { - ignore_changes = [ - # importing from oss-vdb has initial_node_count set to 0, which is actually not a valid configuration for creating a cluster. - # Updating this value in terraform forces a replacement, even though the default pool is destroyed. Ignore it to prevent disruption. - initial_node_count, - ] - prevent_destroy = true - } - - monitoring_config { - managed_prometheus { - enabled = true - } - } -} - -resource "google_container_node_pool" "default_pool" { - project = var.project_id - name = "default-pool" - cluster = google_container_cluster.workers.name - location = google_container_cluster.workers.location - - lifecycle { - # Terraform doesn't automatically know to recreate node pools when the cluster is recreated. - # A bit redundant since the cluster has prevent_destroy = true. - replace_triggered_by = [ - google_container_cluster.workers.id, - ] - } - - autoscaling { - min_node_count = 1 - max_node_count = 1000 - location_policy = "BALANCED" - } - - - node_config { - machine_type = "n4-standard-8" - disk_type = "hyperdisk-balanced" - disk_size_gb = 64 - - oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"] - } -} - -resource "google_container_node_pool" "highend" { - project = var.project_id - name = "highend" - cluster = google_container_cluster.workers.name - location = google_container_cluster.workers.location - # For using the ephemeral storage local ssd config - provider = google-beta - - lifecycle { - # Terraform doesn't automatically know to recreate node pools when the cluster is recreated. - # A bit redundant since the cluster has prevent_destroy = true. - replace_triggered_by = [ - google_container_cluster.workers.id, - ] - } - - autoscaling { - min_node_count = 0 - max_node_count = 100 - location_policy = "BALANCED" - } - - - node_config { - machine_type = "n4-highmem-32" - disk_type = "hyperdisk-balanced" - disk_size_gb = 500 - - oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"] - - labels = { - workloadType = "highend" - } - - taint { - effect = "NO_EXECUTE" - key = "workloadType" - value = "highend" - } - - } -} - -# Service account permissions -data "google_compute_default_service_account" "default" { - project = var.project_id -} - -resource "google_project_iam_member" "compute_service" { - project = var.project_id - role = "roles/editor" - member = "serviceAccount:${data.google_compute_default_service_account.default.email}" -} - -resource "google_project_iam_member" "compute_service_datastore" { - project = var.project_id - role = "roles/datastore.importExportAdmin" - member = "serviceAccount:${data.google_compute_default_service_account.default.email}" -} - -resource "google_compute_disk" "gitter_disk" { - project = var.project_id - name = "gitter-disk" - type = "hyperdisk-balanced" - zone = google_container_cluster.workers.location - size = 6144 # 6TiB - - lifecycle { - ignore_changes = [ - type, - snapshot, - ] - } -} diff --git a/deployment/terraform/modules/osv/workers_network.tf b/deployment/terraform/modules/osv/workers_network.tf deleted file mode 100644 index 1ee40da6e57..00000000000 --- a/deployment/terraform/modules/osv/workers_network.tf +++ /dev/null @@ -1,63 +0,0 @@ -# Network configuration used by workers - -resource "google_compute_subnetwork" "my_subnet_0" { - project = var.project_id - name = "my-subnet-0" - network = "default" - ip_cidr_range = "10.45.32.0/22" - private_ip_google_access = true - region = "us-central1" - - lifecycle { - ignore_changes = [ - # oss-vdb has an auto-generated description from when it was created externally. - # It can't be changed/removed without recreating the resource, so ignore it. - description, - ] - } -} - -resource "google_compute_router" "router" { - project = var.project_id - name = "router" - network = "default" - region = "us-central1" -} - -resource "google_compute_subnetwork" "oss_fuzz_subnet" { - count = var.create_oss_fuzz_subnet ? 1 : 0 - project = var.project_id - name = "oss-fuzz-subnet" - network = "default" - ip_cidr_range = "10.45.36.0/22" - private_ip_google_access = true - region = "us-central1" -} - -resource "google_compute_router_nat" "nat_config" { - project = var.project_id - name = "nat-config" - router = google_compute_router.router.name - source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS" - nat_ip_allocate_option = "AUTO_ONLY" - region = google_compute_router.router.region - enable_endpoint_independent_mapping = false - - subnetwork { - name = google_compute_subnetwork.my_subnet_0.id - source_ip_ranges_to_nat = ["ALL_IP_RANGES"] - } - - dynamic "subnetwork" { - for_each = var.create_oss_fuzz_subnet ? [1] : [] - content { - name = google_compute_subnetwork.oss_fuzz_subnet[0].id - source_ip_ranges_to_nat = ["ALL_IP_RANGES"] - } - } - - log_config { - enable = false - filter = "ALL" - } -} diff --git a/deployment/terraform/modules/osv_pipeline/database.tf b/deployment/terraform/modules/osv_pipeline/database.tf index 3c4fd3fd76d..3ae81188b9d 100644 --- a/deployment/terraform/modules/osv_pipeline/database.tf +++ b/deployment/terraform/modules/osv_pipeline/database.tf @@ -4,6 +4,10 @@ resource "google_firestore_database" "datastore" { name = var.datastore_name location_id = "us-west2" type = "DATASTORE_MODE" + + lifecycle { + prevent_destroy = true + } } # GCP Bucket where protos and full JSON exports are stored @@ -36,6 +40,18 @@ resource "google_storage_bucket" "osv_vulnerabilities_export" { with_state = "ANY" } } + + lifecycle { + prevent_destroy = true + } + + dynamic "logging" { + for_each = var.logs_bucket != "" ? [1] : [] + content { + log_bucket = var.logs_bucket + log_object_prefix = "osv-vulnerabilities" + } + } } # GCP bucket where affected commits are backed up. @@ -52,4 +68,8 @@ resource "google_storage_bucket" "affected_commits_backups_bucket" { age = var.affected_commits_backups_bucket_retention_days } } + + lifecycle { + prevent_destroy = true + } } \ No newline at end of file diff --git a/deployment/terraform/modules/osv_pipeline/gke.tf b/deployment/terraform/modules/osv_pipeline/gke.tf index 692d62b8466..e4b4f00f109 100644 --- a/deployment/terraform/modules/osv_pipeline/gke.tf +++ b/deployment/terraform/modules/osv_pipeline/gke.tf @@ -35,6 +35,7 @@ resource "google_container_cluster" "workers" { # Updating this value in terraform forces a replacement, even though the default pool is destroyed. Ignore it to prevent disruption. initial_node_count, ] + prevent_destroy = true } monitoring_config { @@ -124,4 +125,11 @@ resource "google_compute_disk" "gitter_disk" { type = "hyperdisk-balanced" zone = google_container_cluster.workers.location size = var.gitter_disk_size_gb + + lifecycle { + ignore_changes = [ + type, + snapshot, + ] + } } \ No newline at end of file diff --git a/deployment/terraform/modules/osv_pipeline/network.tf b/deployment/terraform/modules/osv_pipeline/network.tf index 9440eede7b9..3303d68d9b7 100644 --- a/deployment/terraform/modules/osv_pipeline/network.tf +++ b/deployment/terraform/modules/osv_pipeline/network.tf @@ -26,6 +26,16 @@ resource "google_compute_router" "router" { region = "us-central1" } +resource "google_compute_subnetwork" "oss_fuzz_subnet" { + count = var.create_oss_fuzz_subnet ? 1 : 0 + project = var.project_id + name = "oss-fuzz-subnet" + network = "default" + ip_cidr_range = "10.45.36.0/22" + private_ip_google_access = true + region = "us-central1" +} + # Cloud NAT # Allows private GKE nodes to securely access the public internet. resource "google_compute_router_nat" "nat_config" { @@ -42,6 +52,14 @@ resource "google_compute_router_nat" "nat_config" { source_ip_ranges_to_nat = ["ALL_IP_RANGES"] } + dynamic "subnetwork" { + for_each = var.create_oss_fuzz_subnet ? [1] : [] + content { + name = google_compute_subnetwork.oss_fuzz_subnet[0].id + source_ip_ranges_to_nat = ["ALL_IP_RANGES"] + } + } + log_config { enable = false filter = "ALL" diff --git a/deployment/terraform/modules/osv_pipeline/outputs.tf b/deployment/terraform/modules/osv_pipeline/outputs.tf new file mode 100644 index 00000000000..bc7d6c74a9b --- /dev/null +++ b/deployment/terraform/modules/osv_pipeline/outputs.tf @@ -0,0 +1,69 @@ +output "project_id" { + value = var.project_id + description = "The Google Cloud Project ID" +} + +output "tasks_topic_id" { + value = google_pubsub_topic.tasks.id + description = "The ID of the tasks Pub/Sub topic" +} + +output "tasks_topic_name" { + value = google_pubsub_topic.tasks.name + description = "The name of the tasks Pub/Sub topic" +} + +output "failed_tasks_topic_id" { + value = google_pubsub_topic.failed_tasks.id + description = "The ID of the failed-tasks Pub/Sub topic" +} + +output "failed_tasks_topic_name" { + value = google_pubsub_topic.failed_tasks.name + description = "The name of the failed-tasks Pub/Sub topic" +} + +output "pubsub_service_account_email" { + value = google_project_service_identity.pubsub.email + description = "The email of the Pub/Sub service account" +} + +output "worker_service_account_email" { + value = google_service_account.worker_sa.email + description = "The email of the GKE worker service account" +} + +output "oss_fuzz_subnet_self_link" { + value = one(google_compute_subnetwork.oss_fuzz_subnet[*].self_link) + description = "The self link of the OSS-Fuzz subnetwork" +} + +output "cluster_name" { + value = google_container_cluster.workers.name + description = "The name of the GKE cluster" +} + +output "cluster_location" { + value = google_container_cluster.workers.location + description = "The location of the GKE cluster" +} + +output "cluster_id" { + value = google_container_cluster.workers.id + description = "The ID of the GKE cluster" +} + +output "subnet_self_link" { + value = google_compute_subnetwork.my_subnet_0.self_link + description = "The self link of the private subnetwork" +} + +output "vulnerabilities_export_bucket_name" { + value = google_storage_bucket.osv_vulnerabilities_export.name + description = "The name of the vulnerabilities export bucket" +} + +output "affected_commits_backups_bucket_name" { + value = google_storage_bucket.affected_commits_backups_bucket.name + description = "The name of the AffectedCommits backups bucket" +} diff --git a/deployment/terraform/modules/osv_pipeline/pubsub_tasks.tf b/deployment/terraform/modules/osv_pipeline/pubsub_tasks.tf index d24ef80b220..43723999790 100644 --- a/deployment/terraform/modules/osv_pipeline/pubsub_tasks.tf +++ b/deployment/terraform/modules/osv_pipeline/pubsub_tasks.tf @@ -7,6 +7,10 @@ resource "google_pubsub_topic" "tasks" { labels = { goog-dm = "pubsub" } + + lifecycle { + prevent_destroy = true + } } resource "google_pubsub_topic" "failed_tasks" { @@ -108,4 +112,11 @@ resource "google_pubsub_subscription" "recovery" { expiration_policy { ttl = "" # never expires } +} + +resource "google_pubsub_subscription_iam_member" "recovery_service_subscriber" { + project = var.project_id + subscription = google_pubsub_subscription.recovery.name + role = "roles/pubsub.subscriber" + member = "serviceAccount:${google_project_service_identity.pubsub.email}" } \ No newline at end of file diff --git a/deployment/terraform/modules/osv_pipeline/variables.tf b/deployment/terraform/modules/osv_pipeline/variables.tf index 178c508868b..d6a911ba139 100644 --- a/deployment/terraform/modules/osv_pipeline/variables.tf +++ b/deployment/terraform/modules/osv_pipeline/variables.tf @@ -121,3 +121,15 @@ variable "nat_name" { default = "nat-config" } +variable "logs_bucket" { + type = string + description = "The name of the GCS bucket for access logging." + default = "" +} + +variable "create_oss_fuzz_subnet" { + type = bool + description = "Whether to create the OSS-Fuzz subnetwork and add it to NAT." + default = false +} +