diff --git a/src/cli/artemis.toit b/src/cli/artemis.toit index 6d674468..5f83b2a8 100644 --- a/src/cli/artemis.toit +++ b/src/cli/artemis.toit @@ -14,7 +14,6 @@ import host.os import system import .cache as cache -import .cache show cache-key-service-image import .config import .device import .git diff --git a/src/cli/artemis_servers/http/base.toit b/src/cli/artemis_servers/http/base.toit index 9269b61e..4f88cff1 100644 --- a/src/cli/artemis_servers/http/base.toit +++ b/src/cli/artemis_servers/http/base.toit @@ -183,10 +183,6 @@ class ArtemisServerCliHttpToit implements ArtemisServerCli: if response.status-code != http.STATUS-OK and response.status-code != http.STATUS-IM-A-TEAPOT: throw "HTTP error: $response.status-code $response.status-message" - if (command == COMMAND-DOWNLOAD-SERVICE-IMAGE_) - and response.status-code != http.STATUS-IM-A-TEAPOT: - return utils.read-all response.body - decoded := json.decode-stream response.body if response.status-code == http.STATUS-IM-A-TEAPOT: throw "Broker error: $decoded" diff --git a/src/cli/cache.toit b/src/cli/cache.toit index 8cf2bf81..99dc830a 100644 --- a/src/cli/cache.toit +++ b/src/cli/cache.toit @@ -10,14 +10,6 @@ import .server-config Manages cache keys. */ -cache-key-service-image -> string - --service-version/string - --sdk-version/string - --artemis-config/ServerConfig - --chip-family/string - --word-size/int: - return "$artemis-config.cache-key/service/$service-version/$(sdk-version)-$(chip-family)-$(word-size).image" - cache-key-application-image id/Uuid --broker-config/ServerConfig -> string: return "$broker-config.cache-key/application/images/$(id).image" diff --git a/src/shared/constants.toit b/src/shared/constants.toit index b31d313f..b180fc51 100644 --- a/src/shared/constants.toit +++ b/src/shared/constants.toit @@ -17,12 +17,6 @@ COMMAND-ORGANIZATION-MEMBER-REMOVE_ ::= 11 COMMAND-ORGANIZATION-MEMBER-SET-ROLE_ ::= 12 COMMAND-GET-PROFILE_ ::= 13 COMMAND-UPDATE-PROFILE_ ::= 14 -/// Deprecated and unused in newer CLIs. -COMMAND-LIST-SDK-SERVICE-VERSIONS_ ::= 15 -/// Deprecated and unused in newer CLIs. -COMMAND-DOWNLOAD-SERVICE-IMAGE_ ::= 16 -/// Deprecated and unused in newer CLIs. -COMMAND-UPLOAD-SERVICE-IMAGE_ ::= 17 ARTEMIS-COMMAND-TO-STRING ::= { COMMAND-CHECK-IN_: "check-in", @@ -41,9 +35,6 @@ ARTEMIS-COMMAND-TO-STRING ::= { COMMAND-ORGANIZATION-MEMBER-SET-ROLE_: "organization-member-set-role", COMMAND-GET-PROFILE_: "get-profile", COMMAND-UPDATE-PROFILE_: "update-profile", - COMMAND-LIST-SDK-SERVICE-VERSIONS_: "list-sdk-service-versions", - COMMAND-DOWNLOAD-SERVICE-IMAGE_: "download-service-image", - COMMAND-UPLOAD-SERVICE-IMAGE_: "upload-service-image" } /* Broker commands */ diff --git a/tests/artemis-server.toit b/tests/artemis-server.toit index ddab8ecd..2af0e07a 100644 --- a/tests/artemis-server.toit +++ b/tests/artemis-server.toit @@ -31,18 +31,6 @@ interface ArtemisServerBackdoor: /** Whether there exists a '$type'-event for the given $hardware-id. */ has-event --hardware-id/Uuid --type/string -> bool - /** - Installs the given images. - - The $images parameter is a list of maps, each containing the - following entries: - - sdk_version: The SDK version of the image. - - service_version: The service version of the image. - - image: The image identifier. - - contents: The image contents (a byte array). - */ - install-service-images images/List - /** Creates a new device in the given $organization-id. @@ -94,20 +82,6 @@ class ToitHttpBackdoor implements ArtemisServerBackdoor: return true return false - install-service-images images/List -> none: - image-binaries := {:} - sdk-service-versions := [] - images.do: | entry/Map | - sdk-service-versions.add { - "sdk_version": entry["sdk_version"], - "service_version": entry["service_version"], - "image": entry["image"], - } - image-binaries[entry["image"]] = entry["content"] - - server.sdk-service-versions = sdk-service-versions - server.image-binaries = image-binaries - create-device --organization-id/Uuid -> Map: // TODO(florian): the server should automatically generate an alias // if none is given. @@ -203,43 +177,6 @@ class SupabaseBackdoor implements ArtemisServerBackdoor: return true return false - install-service-images images/List -> none: - with-backdoor-client_: | client/supabase.Client | - // Clear the sdks, service-versions and images table. - // Deletes require a where clause, so we use a filter that matches all IDs. - filter := greater-than-or-equal "id" 0 - client.rest.delete "sdks" --filters=[filter] - client.rest.delete "artemis_services" --filters=[filter] - client.rest.delete "service_images" --filters=[filter] - - sdk-versions := {:} - service-versions := {:} - - images.do: | entry/Map | - sdk-version := entry["sdk_version"] - service-version := entry["service_version"] - image := entry["image"] - contents := entry["contents"] - - sdk-id := sdk-versions.get sdk-version --init=: - new-entry := client.rest.insert "sdks" { - "version": sdk-version, - } - new-entry["id"] - service-id := service-versions.get service-version --init=: - new-entry := client.rest.insert "artemis_services" { - "version": service-version, - } - new-entry["id"] - - client.rest.insert "service_images" { - "sdk_id": sdk-id, - "service_id": service-id, - "image": image, - } - - client.storage.upload --path="service-images/$image" --content=contents - create-device --organization-id/Uuid -> Map: alias := random-uuid with-backdoor-client_: | client/supabase.Client | diff --git a/tests/supabase-policies-test.toit b/tests/supabase-policies-test.toit index e363319f..acbca4e6 100644 --- a/tests/supabase-policies-test.toit +++ b/tests/supabase-policies-test.toit @@ -485,213 +485,6 @@ main args: roles1 := client1.rest.select "roles" expect-equals 2 roles1.size - client-admin := supabase.Client --server-config=server-config - client-admin.auth.sign-in - --email="test-admin@toit.io" - --password="password" - - // Admins can change the sdk table. - // All other users (including auth) can only see it. - - // Clear the sdk table for simplicity. - // Delete requires a where clause, so we use a filter that is always true. - client-admin.rest.delete "sdks" --filters=[greater-than-or-equal "id" 0] - sdks := client-admin.rest.select "sdks" - expect-equals 0 sdks.size - - // Admin can insert. - sdk := client-admin.rest.insert "sdks" { - "version": "v1.0.0", - } - expect-equals "v1.0.0" sdk["version"] - sdkv1-id := sdk["id"] - - // Only one entry per version is allowed in the table. - expect-throws --contains="unique": client-admin.rest.insert "sdks" { - "version": "v1.0.0", - } - - // Check that auth and anon can see it. - sdks = client-anon.rest.select "sdks" - expect-equals 1 sdks.size - expect-equals "v1.0.0" sdks[0]["version"] - - sdks = client1.rest.select "sdks" - expect-equals 1 sdks.size - expect-equals "v1.0.0" sdks[0]["version"] - - // Neither anon, nor auth can insert. - expect-throws --contains="policy": client-anon.rest.insert "sdks" { - "version": "v2.0.0", - } - expect-throws --contains="policy": client1.rest.insert "sdks" { - "version": "v2.0.0", - } - - // Same is true for artemis services. - // Admins can change the sdk table. - // All other users (including auth) can only see it. - - // Clear the sdk table for simplicity. - // Delete requires a where clause, so we use a filter that is always true. - client-admin.rest.delete "artemis_services" --filters=[greater-than-or-equal "id" 0] - artemis-services := client-admin.rest.select "artemis_services" - expect-equals 0 artemis-services.size - - // Admin can insert. - services := client-admin.rest.insert "artemis_services" { - "version": "v9.8.7", - } - expect-equals "v9.8.7" services["version"] - service1-id := services["id"] - - // Only one entry per version is allowed in the table. - expect-throws --contains="unique": client-admin.rest.insert "artemis_services" { - "version": "v9.8.7", - } - - // Check that auth and anon can see it. - artemis-services = client-anon.rest.select "artemis_services" - expect-equals 1 artemis-services.size - expect-equals "v9.8.7" artemis-services[0]["version"] - - artemis-services = client1.rest.select "artemis_services" - expect-equals 1 artemis-services.size - expect-equals "v9.8.7" artemis-services[0]["version"] - - // Neither anon, nor auth can insert. - expect-throws --contains="policy": client-anon.rest.insert "artemis_services" { - "version": "2.0.0", - } - expect-throws --contains="policy": client1.rest.insert "artemis_services" { - "version": "2.0.0", - } - - // Same is true for images. - // Admins can change the sdk table. - // All other users (including auth) can only see it. - - // Clear the table for simplicity. - // Delete requires a where clause, so we use a filter that is always true. - client-admin.rest.delete "service_images" --filters=[greater-than-or-equal "id" 0] - images := client-admin.rest.select "service_images" - expect-equals 0 images.size - - image := "test-$(random).image" - - // Admin can insert. - service-snapshot := client-admin.rest.insert "service_images" { - "sdk_id": sdkv1-id, - "service_id": service1-id, - "image": image, - } - expect-equals sdkv1-id service-snapshot["sdk_id"] - expect-equals service1-id service-snapshot["service_id"] - expect-equals image service-snapshot["image"] - service-snapshot-id := service-snapshot["id"] - - // sdk/service pair must be unique. - expect-throws --contains="unique": client-admin.rest.insert "service_images" { - "sdk_id": sdkv1-id, - "service_id": service1-id, - "image": image, - } - - // Check that auth and anon can see it. - images = client-anon.rest.select "service_images" - expect-equals 1 images.size - expect-equals sdkv1-id images[0]["sdk_id"] - expect-equals service1-id service-snapshot["service_id"] - expect-equals image images[0]["image"] - - images = client1.rest.select "service_images" - expect-equals 1 images.size - expect-equals sdkv1-id images[0]["sdk_id"] - expect-equals service1-id service-snapshot["service_id"] - expect-equals image images[0]["image"] - - // Create a second service entry. - // We need it to avoid a unique constraint check for the failing tests below. - // We will also use it for organization-specific services. - services = client-admin.rest.insert "artemis_services" { - "version": "v9.9.9", - } - expect-equals "v9.9.9" services["version"] - service2-id := services["id"] - - // Neither anon, nor auth can insert. - expect-throws --contains="policy": client-anon.rest.insert "service_images" { - "sdk_id": sdkv1-id, - "service_id": service2-id, - "image": image, - } - expect-throws --contains="policy": client1.rest.insert "service_images" { - "sdk_id": sdkv1-id, - "service_id": service2-id, - "image": image, - } - - // Upload a service for a specific organization. - client-admin.rest.insert "service_images" { - "sdk_id": sdkv1-id, - "service_id": service2-id, - "image": image, - "organization_id": organization-id, - } - - // Client1 can see it. - images = client1.rest.select "service_images" - expect (images.any: it["sdk_id"] == sdkv1-id and it["service_id"] == service2-id) - - // Anon and client3 can't see it. - images = client-anon.rest.select "service_images" - expect-not (images.any: it["sdk_id"] == sdkv1-id and it["service_id"] == service2-id) - images = client3.rest.select "service_images" - expect-not (images.any: it["sdk_id"] == sdkv1-id and it["service_id"] == service2-id) - - // Admins can write to the storage. - // All other users (including auth) can only see it. - IMAGE-BUCKET ::= "service-images" - client-admin.storage.upload - --path="$IMAGE-BUCKET/$image" - --content="test".to-byte-array - - // Check that auth and anon can see it. - expect-equals "test".to-byte-array - client-anon.storage.download --path="$IMAGE-BUCKET/$image" - expect-equals "test".to-byte-array - client1.storage.download --path="$IMAGE-BUCKET/$image" - expect-equals "test".to-byte-array - client-anon.storage.download --public --path="$IMAGE-BUCKET/$image" - expect-equals "test".to-byte-array - client1.storage.download --public --path="$IMAGE-BUCKET/$image" - - snapshot := "test-$(random).snapshot" - - // Admins can read and write to the snapshot storage. - // All other users (including auth) can not see it. - SNAPSHOT-BUCKETS ::= [ "service-snapshots", "cli-snapshots" ] - - SNAPSHOT-BUCKETS.do: | bucket/string | - client-admin.storage.upload - --path="$bucket/$snapshot" - --content="test snapshot".to-byte-array - - // Check that admin can see it, but auth and anon can not see it. - expect-equals "test snapshot".to-byte-array - client-admin.storage.download --path="$bucket/$snapshot" - - expect-throws --contains="Not found": - client-anon.storage.download --path="$bucket/$image" - expect-throws --contains="Not found": - client1.storage.download --path="$bucket/$image" - - // Auth and anon don't have public access either. - expect-throws --contains="Not found": - client-anon.storage.download --public --path="$bucket/$image" - expect-throws --contains="Not found": - client1.storage.download --public --path="$bucket/$image" - expect-throws --contains/string [block]: exception := catch: block.call diff --git a/tools/http_servers/artemis-server.toit b/tools/http_servers/artemis-server.toit index a65808eb..c3cac4a2 100644 --- a/tools/http_servers/artemis-server.toit +++ b/tools/http_servers/artemis-server.toit @@ -89,25 +89,11 @@ class HttpArtemisServer extends HttpServer: errors/List := [] - sdk-service-versions := [] - image-binaries := {:} - constructor port/int: super port run-command command/int encoded/ByteArray user-id/string? -> any: - data := ? - - if command == COMMAND-UPLOAD-SERVICE-IMAGE_: - meta-end := encoded.index-of '\0' - meta := encoded[0..meta-end] - contents := encoded[meta-end + 1 ..] - data = { - "meta": meta, - "contents": contents, - } - else: - data = json.decode encoded + data := json.decode encoded print "$Time.now: Artemis request $(ARTEMIS-COMMAND-TO-STRING.get command) ($command) for $user-id with $data." if user-id and not users.contains user-id: @@ -145,12 +131,6 @@ class HttpArtemisServer extends HttpServer: return get-profile data user-id if command == COMMAND-UPDATE-PROFILE_: return update-profile data user-id - if command == COMMAND-LIST-SDK-SERVICE-VERSIONS_: - return list-sdk-service-versions data user-id - if command == COMMAND-DOWNLOAD-SERVICE-IMAGE_: - return download-service-image data - if command == COMMAND-UPLOAD-SERVICE-IMAGE_: - return upload-service-image data else: throw "BAD COMMAND $command" @@ -336,63 +316,6 @@ class HttpArtemisServer extends HttpServer: if data.contains "name": user.name = data["name"] if data.contains "email": user.email = data["email"] - list-sdk-service-versions data/Map user-id/string? -> List: - sdk-version := data.get "sdk_version" - service-version := data.get "service_version" - organization-id := data.get "organization_id" - - // Only return matching versions. - return sdk-service-versions.filter: | entry/Map | - if sdk-version and entry["sdk_version"] != sdk-version: - continue.filter false - if service-version and entry["service_version"] != service-version: - continue.filter false - entry-org := entry.get "organization_id" - if entry-org: - // Only return the versions for the given organization. - if entry-org != organization-id: continue.filter false - // But also check that the user is a member of the organization. - if not user-id: continue.filter false - organization := organizations.get entry["organization_id"] - if not organization: continue.filter false - if not organization.members.contains user-id: continue.filter false - true - return sdk-service-versions - - upload-service-image data/Map: - meta := json.decode data["meta"] - contents := data["contents"] - sdk-version := meta["sdk_version"] - service-version := meta["service_version"] - image-id := meta["image_id"] - organization-id := meta.get "organization_id" - force := meta.get "force" - - image-binaries[image-id] = contents - // Update any existing entry if there is already one. - sdk-service-versions.do: | entry/Map | - if entry["sdk_version"] == sdk-version and entry["service_version"] == service-version: - if not force: - throw "Service version already exists" - - entry["image"] = image-id - if organization-id: - entry["organization_id"] = organization-id - return - new-entry := { - "sdk_version": sdk-version, - "service_version": service-version, - "image": image-id, - } - if organization-id: - new-entry["organization_id"] = organization-id - sdk-service-versions.add new-entry - - download-service-image data/Map -> BinaryResponse: - image-id := data["image"] - image-bin := image-binaries.get image-id - return BinaryResponse image-bin image-bin.size - sign-up data/Map: email := data["email"] password := data["password"] diff --git a/tools/service_image_downloader/CMakeLists.txt b/tools/service_image_downloader/CMakeLists.txt index 15b6335b..96727b36 100644 --- a/tools/service_image_downloader/CMakeLists.txt +++ b/tools/service_image_downloader/CMakeLists.txt @@ -1,21 +1,3 @@ # Copyright (C) 2023 Toitware ApS. -toit_project(uploader "${CMAKE_CURRENT_LIST_DIR}") - -set(DOWNLOADER_SOURCE "${CMAKE_CURRENT_LIST_DIR}/downloader.toit") -set(DOWNLOADER_EXE "${CMAKE_BINARY_DIR}/bin/downloader${CMAKE_EXECUTABLE_SUFFIX}") -set(DOWNLOADER_DEP "${CMAKE_CURRENT_BINARY_DIR}/downloader.toit.dep") - -ADD_TOIT_EXE( - ${DOWNLOADER_SOURCE} - ${DOWNLOADER_EXE} - ${DOWNLOADER_DEP} - "" -) - -add_custom_target( - build_uploader_exes - DEPENDS ${UPLOADER_EXE} ${DOWNLOADER_EXE} -) - -add_dependencies(build build_uploader_exes) +toit_project(sdk_downloader "${CMAKE_CURRENT_LIST_DIR}") diff --git a/tools/service_image_downloader/client.toit b/tools/service_image_downloader/client.toit deleted file mode 100644 index 0b6656d7..00000000 --- a/tools/service_image_downloader/client.toit +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright (C) 2023 Toitware ApS. All rights reserved. - -import ar -import certificate-roots -import cli show * -import encoding.base64 -import encoding.json -import http -import io -import net -import supabase -import supabase.filter show equals - -import artemis.cli.config - show - CONFIG-ARTEMIS-DEFAULT-KEY - CONFIG-SERVER-AUTHS-KEY - ConfigLocalStorage -import artemis.cli.server-config show * -import artemis.cli.utils.supabase show * -import artemis.shared.constants show * - -import .utils - -AR-SNAPSHOT-HEADER ::= "" - -interface UploadClient: - close - upload - --sdk-version/string --service-version/string - --image-id/string --image-contents/ByteArray - --snapshots/Map // From chip-family to ByteArray. - --organization-id/string? - --force/bool - - upload --snapshot-uuid/string cli-snapshot/ByteArray - -get-artemis-config --cli/Cli -> ServerConfig: - result := get-server-from-config --key=CONFIG-ARTEMIS-DEFAULT-KEY --cli=cli - return result - - -with-upload-client invocation/Invocation [block]: - server-config := get-artemis-config --cli=invocation.cli - if server-config is ServerConfigSupabase: - with-upload-client-supabase invocation block - else if server-config is ServerConfigHttp: - with-upload-client-http (server-config as ServerConfigHttp) --cli=invocation.cli block - else: - throw "Unsupported server type" - -class UploadClientSupabase implements UploadClient: - client_/supabase.Client - cli_/Cli - - constructor .client_ --cli/Cli: - cli_ = cli - - close: - client_.close - - upload - --sdk-version/string --service-version/string - --image-id/string --image-contents/ByteArray - --snapshots/Map // From chip-family to ByteArray. - --organization-id/string? - --force/bool: - ui := cli_.ui - client_.ensure-authenticated: | reason/string | - print "Authentication failure: $reason" - supabase-ui := SupabaseUi cli_ - client_.auth.sign_in --provider="github" --ui=supabase-ui - - ui.emit --info "Uploading image archive." - - // TODO(florian): share constants with the CLI. - sdk-ids := client_.rest.select "sdks" --filters=[ - equals "version" sdk-version, - ] - sdk-id := ? - if not sdk-ids.is-empty: - sdk-id = sdk-ids[0]["id"] - else: - inserted := client_.rest.insert "sdks" { - "version": sdk-version, - } - sdk-id = inserted["id"] - - service-ids := client_.rest.select "artemis_services" --filters=[ - equals "version" service-version, - ] - service-id := ? - if not service-ids.is-empty: - service-id = service-ids[0]["id"] - else: - inserted := client_.rest.insert "artemis_services" { - "version": service-version, - } - service-id = inserted["id"] - - if not force: - existing-images := client_.rest.select "service_images" --filters=[ - equals "sdk_id" sdk-id, - equals "service_id" service-id, - ] - if not existing-images.is-empty: - existing-org := existing-images[0].get "organization_id" - suffix := existing-org ? " in organization $existing-org" : "" - ui.emit --error "Image already exists$suffix." - ui.emit --error "Use --force to overwrite." - ui.abort - - client_.storage.upload - --path="service-images/$image-id" - --content=image-contents - - // In theory we should be able to use 'upsert' here, but - // there are unique constraints on the columns that we - // are updating, and that makes things a bit more difficult. - // Given a complete Supabase API (and probably better Postgres - // knowledge) it should be possible, but just checking for - // the entry is significantly easier. - rows := client_.rest.select "service_images" --filters=[ - equals "sdk_id" sdk-id, - equals "service_id" service-id, - ] - if rows.is-empty: - client_.rest.insert "service_images" { - "sdk_id": sdk-id, - "service_id": service-id, - "image": image-id, - "organization_id": organization-id, - } - else: - client_.rest.update "service_images" --filters=[ - equals "sdk_id" sdk-id, - equals "service_id" service-id, - ] { - "image": image-id, - "organization_id": organization-id, - } - - ui.emit --info "Successfully uploaded $service-version into service-images/$image-id." - - ui.emit --info "Uploading snapshots." - buffer := io.Buffer - ar-writer := ar.ArWriter buffer - ar-writer.add AR-SNAPSHOT-HEADER "" - snapshots.do: | chip-family/string snapshot/ByteArray | - ar-writer.add chip-family snapshot - client_.storage.upload - --path="service-snapshots/$image-id" - --content=buffer.bytes - ui.emit --info "Successfully uploaded the snapshot." - - upload --snapshot-uuid/string cli-snapshot/ByteArray: - client_.ensure-authenticated: it.sign-in --provider="github" --cli=cli_ - client_.storage.upload - --path="cli-snapshots/$snapshot-uuid" - --content=cli-snapshot - -with-upload-client-supabase invocation/Invocation [block]: - with-supabase-client invocation: | client/supabase.Client | - upload-client := UploadClientSupabase client --cli=invocation.cli - try: - block.call upload-client - finally: - upload-client.close - -class UploadClientHttp implements UploadClient: - client_/http.Client - server-config_/ServerConfigHttp - cli_/Cli - network_/net.Interface - - constructor .server-config_ --cli/Cli: - cli_ = cli - network_ = net.open - client_ = http.Client network_ - - close: - // TODO(florian): we would like to close the http client here. - network_.close - - upload - --sdk-version/string --service-version/string - --image-id/string --image-contents/ByteArray - --snapshots/Map // From chip-family to ByteArray. - --organization-id/string? - --force/bool: - // We only upload the image. - send-request_ COMMAND-UPLOAD-SERVICE-IMAGE_ --contents=image-contents { - "sdk_version": sdk-version, - "service_version": service-version, - "image_id": image-id, - "organization_id": organization-id, - "force": force, - } - - upload --snapshot-uuid/string cli-snapshot/ByteArray: - throw "UNIMPLEMENTED" - - // TODO(florian): share this code with the cli and the service. - send-request_ command/int meta-data/Map --contents/ByteArray -> any: - encoded-meta := json.encode meta-data - encoded := #[command] + encoded-meta + #[0] + contents - - headers := null - if server-config_.admin-headers: - headers = http.Headers - server-config_.admin-headers.do: | key value | - headers.add key value - - response := client_.post encoded - --host=server-config_.host - --port=server-config_.port - --path=server-config_.path - --headers=headers - - if response.status-code != 200 and response.status-code != http.STATUS-IM-A-TEAPOT: - throw "HTTP error: $response.status-code $response.status-message" - - decoded := json.decode-stream response.body - - if response.status-code == http.STATUS-IM-A-TEAPOT: - throw "Broker error: $decoded" - - return decoded - -with-upload-client-http server-config/ServerConfigHttp --cli/Cli [block]: - upload-client := UploadClientHttp server-config --cli=cli - try: - block.call upload-client - finally: - upload-client.close diff --git a/tools/service_image_downloader/downloader.toit b/tools/service_image_downloader/downloader.toit deleted file mode 100755 index f00cbe71..00000000 --- a/tools/service_image_downloader/downloader.toit +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env toit.run - -// Copyright (C) 2023 Toitware ApS. All rights reserved. - -import ar -import certificate-roots -import cli show * -import io -import host.file -import snapshot show cache-snapshot -import supabase -import supabase.filter show equals - -import .client show AR-SNAPSHOT-HEADER -import .utils - -main args/List: - // Use the same application name as the - // This way we get the same config and cache. - // The config gives as the server configurations and oauth tokens. - // The cache the SDKs. - cli := Cli "artemis" - main args --cli=cli - -main args/List --cli/Cli?: - certificate-roots.install-all-trusted-roots - - cmd := Command "downloader" - --help=""" - Downloads snapshots from the Artemis server and stores them in the Jaguar cache. - - The snapshots can be filtered by SDK version and service version for service - snapshots. There is no way to filter by CLI version. - - If no arguments are given, all snapshots are downloaded. - """ - --options=[ - Option "sdk-version" - --help="The version of the SDK to use.", - Option "service-version" - --help="The version of the service to use.", - Option "output-directory" - --help="The directory to store the downloaded snapshots in.", - Option "server" - --help="The server to download from.", - ] - --examples=[ - Example "Download all snapshots:" --arguments="", - Example """ - Download the snapshot for service snapshot v0.1.0 and SDK version v2.0.0-alpha.62:""" - --arguments="--service-version=v0.1.0 --sdk-version=v2.0.0-alpha.62", - ] - --run=:: download it - - cmd.run args --cli=cli - -download invocation/Invocation: - sdk-version := invocation["sdk-version"] - service-version := invocation["service-version"] - output-directory := invocation["output-directory"] - - cli := invocation.cli - ui := cli.ui - - with-supabase-client invocation: | client/supabase.Client | - client.ensure-authenticated: ui.emit --error it - - // Get a list of snapshots to download. - filters := [] - if sdk-version: filters.add (equals "sdk_version" "$sdk-version") - if service-version: filters.add (equals "service_version" "$service-version") - service-images := client.rest.select "sdk_service_versions" --filters=filters - ui.emit --info "Downloading snapshots for:" - ui.emit-table --info - --header={"sdk_version": "SDK", "service_version": "Service"} - service-images - - successfully-downloaded := 0 - service-images.do: | row | - e := catch --trace: - image := row["image"] - cache-key := "snapshot-downloader/$image" - // We only download a snapshot if we don't have it in our Artemis cache. - // This means that it is possible to remove a snapshot from the snapshot-directory - // and not get it back by calling this function (since the Artemis cache - // would still be there). - // In that case, one would need to remove the Artemis cache. - snapshot := cli.cache.get cache-key: | store/FileStore | - ui.emit --info "Downloading $row["sdk_version"]-$row["service_version"]." - snapshot/ByteArray? := null - exception := catch: - snapshot = client.storage.download --path="service-snapshots/$image" - if exception: - if successfully-downloaded > 0: - // Assume it's an issue with that particular snapshot. - throw "DOWNLOAD_ERROR" - else: - ui.emit --error "Failed to download $row["sdk_version"]-$row["service_version"]." - ui.emit --error "Are you logged in as an admin?" - ui.emit --error exception - ui.abort "Failed to download snapshot." - - ar-reader := ar.ArReader (io.Reader snapshot) - artemis-header := ar-reader.find AR-SNAPSHOT-HEADER - if not artemis-header: - // Deprecated direct snapshot format. - uuid := cache-snapshot snapshot --output-directory=output-directory - ui.emit --info "Wrote service snapshot $uuid." - else: - // Reset the reader. - // We are right after the header, which should be the first file. - // Since we don't need the header anymore (and we will in fact skip it), - // we could just continue reading, but by resetting we avoid hard-to-find bugs. - ar-reader = ar.ArReader (io.Reader snapshot) - while file/ar.ArFile? := ar-reader.next: - if file.name == AR-SNAPSHOT-HEADER: - continue - uuid := cache-snapshot file.contents --output-directory=output-directory - ui.emit --info "Wrote service snapshot $uuid." - store.save snapshot - successfully-downloaded++ - if e: print "Caught: $e" - - if not sdk-version and not service-version: - // Download all CLI snapshots. - available-snapshots := client.storage.list "cli-snapshots" - available-snapshots.do: | file-description/Map | - name := file-description["name"] - cache-key := "snapshot-downloader/$name" - // Same as above: we only download/write snapshots if we don't have any - // entry in the Artemis cache. If the snapshot files have been deleted, - // then one might need to remove the Artemis cache. - cli.cache.get cache-key: | store/FileStore | - ui.emit --info "Downloading $name." - snapshot := client.storage.download --path="cli-snapshots/$name" - uuid := cache-snapshot snapshot --output-directory=output-directory - ui.emit --info "Wrote CLI snapshot $uuid." - store.save snapshot diff --git a/tools/service_image_downloader/sdk-downloader.toit b/tools/service_image_downloader/sdk-downloader.toit index 7966f894..8dbcc1be 100644 --- a/tools/service_image_downloader/sdk-downloader.toit +++ b/tools/service_image_downloader/sdk-downloader.toit @@ -13,8 +13,6 @@ import snapshot show cache-snapshot import supabase import system -import .utils - main args/List: // Use the same application name as the // This way we get the same config and cache. diff --git a/tools/service_image_downloader/utils.toit b/tools/service_image_downloader/utils.toit deleted file mode 100644 index 756f7de0..00000000 --- a/tools/service_image_downloader/utils.toit +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (C) 2023 Toitware ApS. All rights reserved. - -import ar -import certificate-roots -import cli show Cli Invocation -import host.file -import host.directory -import supabase - -import artemis.cli.config show - CONFIG-ARTEMIS-DEFAULT-KEY - CONFIG-SERVER-AUTHS-KEY - ConfigLocalStorage -import artemis.cli.server-config show * - -with-supabase-client invocation/Invocation [block]: - cli := invocation.cli - server-config := get-server-from-config --key=CONFIG-ARTEMIS-DEFAULT-KEY --cli=cli - local-storage := ConfigLocalStorage - --auth-key="$(CONFIG-SERVER-AUTHS-KEY).$(server-config.name)" - --cli=cli - supabase-config := server-config as supabase.ServerConfig - client := supabase.Client - --server-config=supabase-config - --local-storage=local-storage - try: - block.call client - finally: - client.close - -with-tmp-directory [block]: - tmp-dir := directory.mkdtemp "/tmp/artemis-uploader-" - try: - block.call tmp-dir - finally: - directory.rmdir --recursive tmp-dir - -write-file --path/string contents: - stream := file.Stream.for-write path - try: - stream.out.write contents - finally: - stream.close