diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9be81f61..dde9bc90 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,6 +22,11 @@ env: 'build_file': 'kafka-connect-oura-source/Dockerfile', 'authors': 'Pim van Nierop , Pauline Conde ', 'description': 'RADAR-base Oura connector application' + },{ + 'name': 'kafka-connect-huawei-source', + 'build_file': 'kafka-connect-huawei-source/Dockerfile', + 'authors': 'Yatharth Ranjan , Yatharth Ranjan ', + 'description': 'RADAR-base Huawei Health Kit connector application' }] jobs: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c70528ad..6f43aa67 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,8 +16,13 @@ env: },{ 'name': 'kafka-connect-oura-source', 'build_file': 'kafka-connect-oura-source/Dockerfile', - 'authors': 'Pauline Conde , Yatharth Ranjan ', + 'authors': 'Pauline Conde , Yatharth Ranjan , Yatharth Ranjan ', 'description': 'RADAR-base Oura connector application' + },{ + 'name': 'kafka-connect-huawei-source', + 'build_file': 'kafka-connect-huawei-source/Dockerfile', + 'authors': 'Yatharth Ranjan , Yatharth Ranjan ', + 'description': 'RADAR-base Huawei Health Kit connector application' }] jobs: diff --git a/.gitignore b/.gitignore index 530c0530..0aa8b485 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,6 @@ docker/users docker/source-fitbit.properties docker/source-oura.properties bin/ -.DS_Store \ No newline at end of file +.DS_Store +# Output of scripts/huawei-api-probe.sh (contains health data) +huawei-responses/ diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 00000000..07f96e14 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,309 @@ +# Architecture + +This document describes how RADAR-REST-Connector is put together, so that future contributors +(human or agent) can orient themselves quickly and add new device/API integrations consistently +with the existing patterns. + +## What this repo is + +A multi-module Gradle project providing Kafka Connect **source connectors** that poll third-party +REST APIs (wearable vendor APIs) on behalf of RADAR-base study participants and publish the +resulting data as Avro records on Kafka topics. It currently ships three concrete connectors — +**Fitbit**, **Oura**, and **Huawei Health Kit** — the latter two built on the "library + thin +Connect glue" pattern described below; Fitbit predates that pattern and uses the older, generic +`kafka-connect-rest-source` framework instead. + +``` +RADAR-REST-Connector/ +├── kafka-connect-rest-source/ # Generic Kafka Connect REST-source framework (Java) +├── kafka-connect-fitbit-source/ # Fitbit connector (Java), oldest/original implementation +├── oura-library/ # Oura domain logic: routes, converters, requests (Kotlin, no Kafka Connect deps) +├── kafka-connect-oura-source/ # Oura Kafka Connect glue (Java+Kotlin), wraps oura-library +├── huawei-library/ # Huawei domain logic: routes, converters, requests (Kotlin, no Kafka Connect deps) +├── kafka-connect-huawei-source/ # Huawei Kafka Connect glue (Java+Kotlin), wraps huawei-library +├── docker/ # Docker Compose config templates, launch/ensure scripts, log4j +├── scripts/REDCAP-FITBIT-AUTH-AUTO/ # Standalone Python helper for REDCap-driven Fitbit auth +└── docker-compose.yml # Full local Kafka stack + both connectors, for manual testing +``` + +Root Gradle config (`build.gradle.kts`, `settings.gradle.kts`, `gradle/libs.versions.toml`) uses +the `org.radarbase.radar-kotlin` / `radar-root-project` plugins (from `radar-commons`) for shared +build conventions (Kotlin/Java toolchain, Sentry, versioning). All dependency versions are +centralized in `gradle/libs.versions.toml` (a Gradle version catalog) — add new deps there, not +inline in module build files. + +Avro **schemas are not defined in this repo**. They come from the external `radar-schemas-commons` +artifact (`org.radarbase:radar-schemas-commons`, versioned in the catalog), generated from the +[RADAR-Schemas](https://github.com/RADAR-base/RADAR-Schemas) repository. Adding a new data type +therefore requires a schema to exist there first (e.g. `org.radarcns.connector.oura.OuraDailyActivity`), +before this repo can build an Avro record for it. + +## Two architectural generations + +The repo contains two different design generations. Understand both before adding Huawei, and +prefer the **Oura pattern** for new integrations — it is the more recent, more testable design. + +### 1. Generic `kafka-connect-rest-source` framework (used by Fitbit) + +Located at `kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/`. Defines a small +set of interfaces meant to be generic across arbitrary REST APIs: + +- `AbstractRestSourceConnector` — Kafka Connect `SourceConnector` base class. Loads + `RestSourceConnectorConfig` from properties and hands out `RestSourceTask` as the task class. +- `RestSourceConnectorConfig` — `AbstractConfig` wrapper exposing the generic + `rest.source.*` properties (base URL, poll interval, topic selector class, payload converter + class, request generator class — all pluggable via `ConfigDef.Type.CLASS`). +- `RequestGenerator` (`request/`) — produces a `Stream` of `RestRequest`s to issue and knows when + the next request is due (`getTimeOfNextRequest()`), driven by the Kafka Connect offset storage + (`setOffsetStorageReader`). +- `RequestRoute` / `PollingRequestRoute` (`request/`) — one "route" = one logical polling + endpoint/data type. Routes own their own per-user polling cadence, backoff, and offset state, and + are notified of `requestSucceeded` / `requestEmpty` / `requestFailed`. +- `PayloadToSourceRecordConverter` (`converter/`) — turns a raw HTTP response body into one or more + Kafka Connect `SourceRecord`s. +- `RestSourceTask` — the actual Kafka Connect `SourceTask`. Its `poll()` loop: sleep until the next + request is due, iterate `requestGenerator.requests()`, execute the first request that yields + records, return them. + +This module is intentionally protocol-agnostic; it has no notion of "Fitbit" or OAuth. It's a +reasonable place to fix or extend genuinely generic REST-polling behavior (e.g. topic selection, +generic retry semantics), but new device integrations do **not** need to hook into it directly — +see the Oura pattern below. + +### 2. Fitbit connector (`kafka-connect-fitbit-source`) — first concrete integration + +Built directly on the generic framework above, entirely in Java: + +- `FitbitSourceConnector extends AbstractRestSourceConnector` — schedules a periodic + (`application.loop.interval.ms`) user-repository refresh; if the user set changes, requests task + reconfiguration (`context.requestTaskReconfiguration()`). Divides users across `tasks.max` tasks + by hashing `user.getVersionedId()`. +- `FitbitRequestGenerator extends RequestGeneratorRouter` — builds the list of enabled + `RequestRoute`s (one per Fitbit data type: sleep, activity log, resting heart rate, and — if + `fitbit.api.intraday=true` — steps, heart rate, HRV, breathing rate, skin temperature, calories, + SpO2) and an OkHttp client per user with a `TokenAuthenticator` (auto-refreshes on HTTP 401). +- `route/Fitbit*Route` — one class per data type, extending `FitbitPollingRoute`, which implements + a fairly elaborate polling algorithm: don't poll more than once per `pollInterval`; walk history + back to `HISTORICAL_TIME_DAYS`; avoid re-reading the last `LOOKBACK_TIME` to tolerate + late-arriving data from other devices; back off per-user on HTTP 429 (`TOP_OF_HOUR` or + `ROLLING_WINDOW` cooldown strategy) and after `fitbit.request.max.forbidden` consecutive HTTP 403s. +- `converter/Fitbit*AvroConverter` — one class per data type, converts JSON to the corresponding + Avro record from `radar-schemas-commons`. +- `user/UserRepository` (interface) + implementations: + - `YamlUserRepository` — reads per-user YAML files from a directory (`docker/fitbit-user.yml.template` + shows the format: id, projectId, userId, sourceId, startDate/endDate, externalUserId, OAuth2 + access/refresh tokens). + - `ServiceUserRepository` (Kotlin) — talks to an external "rest-source-authorizer" webservice + (typically fronted by ManagementPortal) for user lists and token refresh/storage, using + OAuth2 client-credentials auth. + - `firebase/FirebaseUserRepository` / `CovidCollabFirebaseUserRepository` — legacy + Firestore-backed repository for a specific historical deployment. +- `request/TokenAuthenticator` — OkHttp `Authenticator` that refreshes the access token via the + `UserRepository` on a 401 and retries the request. + +### 3. Oura connector (`oura-library` + `kafka-connect-oura-source`) — newer pattern + +This is the template to follow for a **new** vendor integration such as Huawei. It splits cleanly +into: + +- **`oura-library`** (pure Kotlin, no Kafka Connect / OkHttp-Authenticator coupling to Connect + internals) — all vendor-specific domain logic, independently unit-testable and in principle + reusable outside Kafka Connect: + - `user/User`, `user/UserRepository` — user model and repository interface (`get`, `stream`, + `getAccessToken`); note `refreshAccessToken` lives on the connector-side implementation, not + here, in current Oura code (asymmetry vs. Fitbit — be aware when implementing). + - `route/Route` (interface) + `route/OuraRoute` (abstract base) + `route/Oura*Route` (one per + data type: daily activity, readiness, sleep, SpO2, heart rate, personal info, sessions, + workouts, tags, ring configuration, stress, VO2 max, resilience, cardiovascular age, enhanced + tags, rest-mode periods, sleep-time recommendations, etc.) — each route knows its API sub-path + and builds one or more `RestRequest`s covering a `[start, end)` window, chunked by + `maxIntervalPerRequest`. + - `route/OuraRouteFactory` — central list of all routes (used mainly by tests / defaults; the + connector module actually builds its own filtered list based on config flags — see below). + - `converter/OuraDataConverter` (→ `RecordConverter`) + `converter/Oura*Converter` — one per data + type, parses the JSON response into `TopicData(topic, key, value, offset)` where `value` is a + generated Avro `SpecificRecord` from `radar-schemas-commons` (e.g. `OuraDailyActivity`). This + is the direct analogue of Fitbit's `Fitbit*AvroConverter`, but returns plain data objects + instead of Kafka Connect `SourceRecord`s directly — the Connect-specific wrapping happens in + the connector module. + - `request/OuraRequestGenerator` — the polling brain: for each `(route, user)` pair, computes the + offset to resume from (via `OuraOffsetManager`), decides whether to use a large "historical" + chunk (`HISTORICAL_QUERY_RANGE` = 1 year, once `timeSinceStart > HISTORICAL_DATA_THRESHOLD` = 1 + year) or normal recent-data chunking, and interprets HTTP responses + (`handleResponse`/`requestSuccessful`/`requestFailed`) into typed `OuraResult`/`OuraError` + sealed hierarchies with per-(route,user) backoff bookkeeping (`routeNextRequest` map) for 429 / + 403 / 401 / 400 / 422 / 404 / other. + - `request/OuraOffsetManager` (interface) — abstraction for reading/writing per-(route,user) + offsets; the Kafka Connect implementation is `KafkaOffsetManager` in the connector module. + - `offset/Offset`, `offset/Offsets` — plain offset value types. +- **`kafka-connect-oura-source`** (Java, some Kotlin) — the thin Kafka Connect glue: + - `OuraSourceConnector` — same role as `FitbitSourceConnector`: periodic user refresh, + reconfiguration on user-set change, hash-based task partitioning. + - `OuraSourceTask` — Kafka Connect `SourceTask`. Builds the enabled `Route` list from config + flags, constructs `OuraRequestGenerator`, and in `poll()` round-robins across routes + (`getRotatedRoutes()`, so one slow/rate-limited route doesn't starve the others), executes one + HTTP request via a shared `OkHttpClient`, and converts the resulting `TopicData` list into + Kafka Connect `SourceRecord`s using `AvroData` (Confluent's Kotlin/Avro↔Connect-schema bridge). + - `offset/KafkaOffsetManager` — `OuraOffsetManager` backed by Kafka Connect's + `OffsetStorageReader`. + - `user/OuraUserRepository` (abstract) / `OuraServiceUserRepository` — the concrete + "rest-source-authorizer" HTTP client, built on **Ktor** (not OkHttp) with `radar-commons`'s + `CachedSet`/`CachedValue`/`clientCredentials` helpers for user list caching and per-user OAuth2 + token caching/refresh. This is the modern replacement for Fitbit's + `ServiceUserRepository`/`TokenAuthenticator` combo, and is the pattern to copy for Huawei. + - `OuraRestSourceConnectorConfig` — `ConfigDef` with one `oura..enabled` boolean and topic + name per data type, plus `oura.user.repository.*` connection settings. + +**Key structural difference from Fitbit**: Oura does *not* extend the generic +`kafka-connect-rest-source` interfaces (`RequestRoute`, `PollingRequestRoute`, +`PayloadToSourceRecordConverter`) at all — `OuraSourceTask` implements Kafka Connect's `SourceTask` +directly and drives `oura-library`'s own `Route`/`RequestGenerator`/`RecordConverter` abstractions. +This was a deliberate move to (a) get domain logic under unit test without spinning up Kafka +Connect, and (b) avoid the generic framework's assumptions (e.g. its polling-interval math) that +didn't fit Oura's simpler historical/recent chunking model. + +### 4. Huawei Health Kit connector (`huawei-library` + `kafka-connect-huawei-source`) + +Structurally identical to the Oura pattern above (pure-Kotlin domain library + thin Connect glue +module), but with two differences worth knowing about: + +- **Three request "shapes" instead of one.** The Huawei Health Kit Data API doesn't have a single + uniform per-route request shape like Oura's `GET .../{subPath}?start_date=...&end_date=...`. It + exposes `POST /healthkit/v1/sampleSet:polymerize` (raw sample points, or day-aggregated + statistics when a `groupByTime` block is added to the JSON body) for most data types, plus two + GET endpoints — `activityRecords` and `healthRecords` — for workout sessions and clinical-style + records (blood pressure sessions, heart-rate alerts, menstrual cycle, sleep). `HuaweiRoute` is + the shared abstract base (OAuth2-authorized request building + time-range chunking); + `HuaweiSampleSetRoute`, `HuaweiHealthRecordRoute`, and `HuaweiActivityRecordRoute` are the three + concrete route kinds. +- **A single route registry drives both the route list and the Connect config**, instead of + Oura/Fitbit's one-hand-written-`ConfigDef`-entry-per-data-type approach. Huawei has ~54 data + types (see the `radar-huawei-connector` schema spec in RADAR-Schemas, + `specifications/connector/radar-huawei-connector-1.0.0.yml`), several of which reuse the same + Avro schema (`HuaweiStatistics` alone backs 14 different `*.statistics` topics) — hand-duplicating + a `ConfigDef.define(...)` block and a route-construction branch per type, Fitbit/Oura-style, + would mean ~110 near-identical static fields. Instead, `huawei-library`'s + `route/HuaweiRouteFactory.definitions` is a `List` (config key, default + topic, default enabled, and a `(UserRepository, topic) -> HuaweiRoute` builder) — the single + source of truth for "what Huawei data types exist." `HuaweiRestSourceConnectorConfig.conf()` + loops over it to generate `huawei..enabled`/`huawei..topic` `ConfigDef` entries, and + `HuaweiSourceTask.getRoutes()` loops over the same list filtered by that config to build the + actual `Route` instances — so the config and the polled routes can't drift out of sync. If you + add a data type to a future connector with a similarly large surface, prefer this registry + pattern over copy-pasting Oura's per-type `ConfigDef` blocks. + +Field-value key names inside `HuaweiRouteFactory`'s record builders (what JSON key a given Avro +field is read from) are a best-effort mapping to Huawei's documented `Field` naming convention — +verify them against a real Health Kit API response and adjust before relying on this in +production; see the KDoc at the top of that file. + +## Runtime data flow (both connectors, conceptually) + +```mermaid +sequenceDiagram + participant connector as SourceConnector + participant task as SourceTask + participant userRepo as User Repository (rest-source-authorizer) + participant api as Vendor API (Fitbit/Oura/…) + participant kafka as Kafka + + connector ->> userRepo: Poll for users/config changes (periodic) + connector ->> connector: Partition users across tasks.max tasks + loop poll() + task ->> task: Determine next due (route, user) request + task ->> userRepo: Get/refresh OAuth2 access token + task ->> api: GET data for date range + api -->> task: JSON response + task ->> task: Convert JSON -> Avro SourceRecord(s) + task ->> kafka: Return records (Connect framework produces them) + task ->> task: Update in-memory + Connect offset state + end +``` + +User authentication/authorization data (OAuth2 tokens, study/user/source IDs, start/end dates) is +**not** stored in this repo. In production it's served by a "rest-source-authorizer" webservice +(part of RADAR-base, typically backed by ManagementPortal); for local/manual testing, Fitbit also +supports flat YAML files under `docker/users/` (`YamlUserRepository`). + +## Configuration model + +Every connector exposes its settings as a Kafka Connect `ConfigDef` (`org.apache.kafka.common.config`), +loaded from a Java `.properties` file (see `docker/source-fitbit.properties.template` and +`docker/source-oura.properties.template`) referenced by `connector.class`, `name`, `tasks.max`, +plus vendor-specific keys, e.g.: + +- `.api.client` / `.api.secret` — OAuth2 app credentials. +- `.user.repository.class` — pluggable `UserRepository` implementation. +- `.user.repository.url` / `.client.id` / `.client.secret` / `.oauth2.token.url` — + rest-source-authorizer connection details. +- `..topic` / `.enabled` — per-data-type Kafka topic name and on/off switch, so + studies can disable data types they don't need. + +The full current list for Fitbit is documented in `README.md`; Oura's config lives in +`OuraRestSourceConnectorConfig` (no README table yet — check the class directly). Huawei's +per-data-type keys are generated from `HuaweiRouteFactory.definitions` (see below) rather than +hand-written — check that list, or a running connector's `GET /connectors//config`, for the +current set. + +## Docker / deployment + +Each connector module has its own multi-stage `Dockerfile` (Gradle build stage → base image +`confluentinc/cp-kafka-connect-base`), publishing built jars plus third-party deps into +`$CONNECT_PLUGIN_PATH//`. `docker/launch` and `docker/ensure` are modified Confluent +entrypoint scripts (env-var → properties translation, Kafka-readiness wait). `docker-compose.yml` +spins up a full local Zookeeper+Kafka+SchemaRegistry+REST-proxy cluster plus all three connectors +for manual end-to-end testing (`docker-compose up -d --build`, inspect with +`kafka-avro-console-consumer`). Sentry error monitoring is wired in via `radarKotlin { sentryEnabled = true }` +and configured purely through `SENTRY_DSN`/`SENTRY_*` env vars — see README "Sentry monitoring". + +## Testing + +- `kafka-connect-rest-source/src/test`, `kafka-connect-fitbit-source/src/test`, + `kafka-connect-oura-source/src/test`, `kafka-connect-huawei-source/src/test` currently only + contain config-parsing tests (`*ConnectorConfigTest`) plus one task test — test coverage of the + actual polling/conversion logic is thin. `wiremock` and `mockito` are on the version catalog for + HTTP-level testing but not yet exercised much; the `oura-library`/`huawei-library` pure-Kotlin + design makes those the easiest place to add real unit tests for new routes/converters without + Kafka Connect scaffolding. +- CI (`.github/workflows/main.yml`) runs `./gradlew assemble` and `./gradlew check` on every push/PR + to `master`/`dev`, then builds (and on `push`, publishes) multi-arch Docker images per connector + module via a matrix job. `release.yml` does the same on GitHub Release publish, additionally + uploading built jars as release assets, tagged `vX.Y.Z` from `gradle.properties`/version catalog. +- **Sandbox note:** in a network-restricted environment (no access to `packages.confluent.io`, or + to whichever host actually serves a given `-SNAPSHOT` dependency), only the pure-Kotlin library + modules (`oura-library`, `huawei-library`) may be compilable — the `kafka-connect-*-source` + glue modules depend on `io.confluent:kafka-connect-avro-converter` / + `org.apache.kafka:connect-api` from Confluent's Maven repo and won't resolve. If you hit this, + it's an environment limitation, not a code problem: check whether the library module alone + compiles before concluding the code is broken, and consider publishing a needed `-SNAPSHOT` + dependency to `mavenLocal()` (e.g. `gradle :radar-schemas-commons:publishToMavenLocal` from a + RADAR-Schemas checkout) to verify domain logic against the real generated classes. + +## Adding a new vendor integration + +Follow the **Oura/Huawei pattern**, not the Fitbit one — see the Huawei section above for a +worked example, including the route-registry technique for connectors with a large number of +data types: + +1. New Gradle module `-library` (pure Kotlin, mirrors `oura-library`/`huawei-library`): + `user/`, `route/`, `converter/`, `request/`, `offset/` packages. No Kafka Connect or + OkHttp-Connect-specific types here — keep it independently testable. +2. New Gradle module `kafka-connect--source` (mirrors `kafka-connect-oura-source`/ + `kafka-connect-huawei-source`): `SourceConnector`, `SourceTask`, + `RestSourceConnectorConfig`, `offset/KafkaOffsetManager`, + `user/ServiceUserRepository` (Ktor-based rest-source-authorizer client, copy + `OuraServiceUserRepository`'s/`HuaweiServiceUserRepository`'s structure), plus a `Dockerfile`. +3. Register both modules in `settings.gradle.kts`; add any new dependency versions to + `gradle/libs.versions.toml` first. If the vendor's schemas need a newer `radar-schemas-commons` + than the other connectors use, add a separate version-catalog entry for it (see + `radarSchemasHuawei`) so it doesn't force every other module onto that version. +4. Confirm (or add) the required Avro schemas in the external RADAR-Schemas project and bump the + catalog version once published — this repo cannot invent schemas locally. +5. One `Route`/`Converter` per vendor data type. For a small number of data types, per-type classes + (Oura's approach) are fine; for a large or schema-reuse-heavy surface (Huawei's ~54 types + sharing a handful of Avro schemas), prefer a single registry (`HuaweiRouteFactory.definitions`) + that both the `ConfigDef` builder and the route-construction code iterate over. +6. Add `docker/source-.properties.template`, a `docker-compose.yml` service entry, and a + README section, following the Fitbit/Oura/Huawei sections as templates. +7. Add the new Docker image to the `IMAGES` matrix in both `.github/workflows/main.yml` and + `release.yml`. diff --git a/README.md b/README.md index 585c87e0..8b93f3d2 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Kafka Connect REST Source and Fitbit Source This project contains a Kafka Connect source connector for a general REST API, for -specific Fitbit and Oura devices. The documentation of the Kafka Connect REST source still needs to -be done. +specific Fitbit, Oura, and Huawei Health Kit devices. The documentation of the Kafka Connect REST +source still needs to be done. @@ -10,6 +10,7 @@ be done. * [Fitbit source connector](#fitbit-source-connector) * [Installation](#installation) * [Usage](#usage) + * [Huawei Health Kit source connector](#huawei-health-kit-source-connector) * [Sentry monitoring](#sentry-monitoring) * [Contributing](#contributing) @@ -208,9 +209,76 @@ sequenceDiagram connector ->> connector: Update offset times ``` +## Huawei Health Kit source connector + +The `kafka-connect-huawei-source` module polls the +[Huawei Health Kit Data API](https://developer.huawei.com/consumer/en/doc/HMSCore-References/rest-overview-0000001254420693) +for the data types documented in the +[`radar-huawei-connector` schema specification](https://github.com/RADAR-base/RADAR-Schemas/blob/huawei_schemas/specifications/connector/radar-huawei-connector-1.0.0.yml) +(RADAR-Schemas, `huawei_schemas` branch) — activity records, continuous/instantaneous sample +statistics (steps, distance, calories, heart rate, SpO2, blood pressure, breathing rate, ECG, +sleep stages, and more), health records (ambulatory blood pressure, heart rate alerts, +hyperthermia, low SpO2 alerts, menstrual cycle, sleep), and daily summaries. It follows the same +`rest.source.*`, `huawei.api.client`/`huawei.api.secret`, and `huawei.user.repository.*` +configuration conventions as the Fitbit and Oura connectors above, plus one +`huawei..enabled` / `huawei..topic` pair per Huawei data type — see +`org.radarbase.huawei.route.HuaweiRouteFactory` for the full list of `` keys and their +default topic names, and `docker/source-huawei.properties.template` for a minimal example. + +To inspect raw Huawei API responses for every data type the connector queries, run +`scripts/huawei-api-probe.sh`. It sends the connector's exact request for each route and saves +the responses to `huawei-responses/.json`. It can refresh the access token first: + +```shell +HUAWEI_CLIENT_ID= HUAWEI_CLIENT_SECRET= \ +USER_FILE=docker/users/test.yml DAYS=30 ./scripts/huawei-api-probe.sh +``` + +Run it from a machine in the same Huawei site (region) as the user's account; requests from +elsewhere are rejected with error `121001 "request forbidden due to site cross"`. + +This connector's schemas were released in `radar-schemas-commons` `0.9.1` (the +`radarSchemasHuawei` version in `gradle/libs.versions.toml`). + +### Testing locally + +The easiest way to try out the Huawei connector without standing up a +`rest-source-authorizer` webservice is the file-based +`org.radarbase.connect.rest.huawei.user.HuaweiYamlUserRepository`, which reads one YAML file per +user from a local directory, mirroring the Fitbit `YamlUserRepository` above. + +1. [Register a Huawei Health Kit OAuth 2.0 app](https://developer.huawei.com/consumer/en/doc/HMSCore-Guides/config-agc-0000001050170137) + and obtain an access token and refresh token for one test user by hand, using Huawei's + [OAuth 2.0 authorization code flow](https://developer.huawei.com/consumer/en/doc/HMSCore-Guides/authorization-code-0000001053629189). +2. Copy `docker/huawei-user.yml.template` to a file in `docker/users/` (e.g. `docker/users/test.yml`) + and fill in the `externalUserId`, `oauth2.accessToken`, and `oauth2.refreshToken` fields. +3. Copy `docker/source-huawei.properties.template` to `docker/source-huawei.properties`, set + `huawei.api.client` / `huawei.api.secret` to your Huawei app's client ID and secret, and change + `huawei.user.repository.class` to `org.radarbase.connect.rest.huawei.user.HuaweiYamlUserRepository` + (the `huawei.user.repository.url`/`.client.id`/`.client.secret`/`.oauth2.token.url` properties + are only used by the webservice-based repository and can be left as-is or removed). +4. Run the full stack with `docker-compose up -d --build` and inspect the connector's progress with + `docker-compose logs -f radar-huawei-connector`. +5. To inspect the data coming out of a specific route, run, for example: + + ```shell + docker-compose exec schema-registry-1 kafka-avro-console-consumer \ + --bootstrap-server kafka-1:9092,kafka-2:9092,kafka-3:9092 \ + --from-beginning \ + --topic connect_huawei_activity_record + ``` + + (replace the topic with any `huawei..topic` default from + `org.radarbase.huawei.route.HuaweiRouteFactory`). + +For a full RADAR-base deployment, use the webservice-based +`org.radarbase.connect.rest.huawei.user.HuaweiServiceUserRepository` (the default) against a +`rest-source-authorizer` instance instead, following the same pattern as the Fitbit connector's +ManagementPortal setup above. + ## Sentry monitoring -To enable Sentry monitoring for the generic REST, Fitbit, or Oura source connector service: +To enable Sentry monitoring for the generic REST, Fitbit, Oura, or Huawei source connector service: 1. Set a `SENTRY_DSN` environment variable that points to the desired Sentry DSN. 2. (Optional) Set the `SENTRY_LOG_LEVEL` environment variable to control the minimum log level of diff --git a/docker-compose.yml b/docker-compose.yml index 53c62257..6243286b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,7 @@ version: "2.4" volumes: fitbit-logs: {} oura-logs: {} + huawei-logs: {} services: #---------------------------------------------------------------------------# @@ -231,3 +232,49 @@ services: # SENTRY_DSN: 'https://000000000000.ingest.de.sentry.io/000000000000' # SENTRY_ATTACHSTACKTRACE: true # SENTRY_STACKTRACE_APP_PACKAGES: io.confluent.connect,org.radarbase.connect.rest + + #---------------------------------------------------------------------------# + # RADAR Huawei connector # + #---------------------------------------------------------------------------# + radar-huawei-connector: + build: + context: . + dockerfile: ./kafka-connect-huawei-source/Dockerfile + image: radarbase/radar-connect-huawei-source + restart: on-failure + volumes: + - ./docker/source-huawei.properties:/etc/kafka-connect/source-huawei.properties + - ./docker/users:/var/lib/kafka-connect-huawei-source/users + - huawei-logs:/var/lib/kafka-connect-huawei-source/logs + depends_on: + - zookeeper-1 + - zookeeper-2 + - zookeeper-3 + - kafka-1 + - kafka-2 + - kafka-3 + - schema-registry-1 + environment: + CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092 + CONNECT_REST_PORT: 8083 + CONNECT_GROUP_ID: "default" + CONNECT_CONFIG_STORAGE_TOPIC: "default.config" + CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets" + CONNECT_STATUS_STORAGE_TOPIC: "default.status" + CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter" + CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter" + CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081" + CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/var/lib/kafka-connect-huawei-source/logs/connect.offsets" + CONNECT_REST_ADVERTISED_HOST_NAME: "radar-huawei-connector" + CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181 + CONNECTOR_PROPERTY_FILE_PREFIX: "source-huawei" + KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m" + KAFKA_BROKERS: 3 + CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR" + # SENTRY_LOG_LEVEL: 'ERROR' + # SENTRY_DSN: 'https://000000000000.ingest.de.sentry.io/000000000000' + # SENTRY_ATTACHSTACKTRACE: true + # SENTRY_STACKTRACE_APP_PACKAGES: io.confluent.connect,org.radarbase.connect.rest diff --git a/docker/huawei-user.yml.template b/docker/huawei-user.yml.template new file mode 100644 index 00000000..99e5187e --- /dev/null +++ b/docker/huawei-user.yml.template @@ -0,0 +1,24 @@ +--- +# Unique user key +id: test +# Project ID to be used in org.radarcns.kafka.ObservationKey record keys +projectId: radar-test +# User ID to be used in org.radarcns.kafka.ObservationKey record keys +userId: test +# Source ID to be used in org.radarcns.kafka.ObservationKey record keys +sourceId: huawei-watch +# Date from when to collect data. +startDate: 2018-08-06T00:00:00Z +# Date until when to collect data. +endDate: 2099-01-01T00:00:00Z +# Huawei user ID as returned by the Huawei OAuth 2.0 authentication procedure +externalUserId: ? +oauth2: + # Huawei Health Kit OAuth 2.0 access token as returned by the Huawei authentication procedure + accessToken: ? + # Huawei Health Kit OAuth 2.0 refresh token as returned by the Huawei authentication procedure + refreshToken: ? + # Optional expiry time of the access token. If absent, it will be estimated to one hour + # when the source connector starts. When an authentication error occurs, a new access token will + # be fetched regardless of the value in this field. + #expiresAt: 2018-08-06T00:00:00Z diff --git a/docker/source-huawei.properties.template b/docker/source-huawei.properties.template new file mode 100644 index 00000000..69607aa6 --- /dev/null +++ b/docker/source-huawei.properties.template @@ -0,0 +1,18 @@ +name=radar-huawei-source +connector.class=org.radarbase.connect.rest.huawei.HuaweiSourceConnector +tasks.max=4 +rest.source.base.url=https://health-api.cloud.huawei.com/healthkit/v1 +rest.source.poll.interval.ms=5000 +huawei.api.client=? +huawei.api.secret=? +# For local testing without a rest-source-authorizer webservice, use the file-based +# repository instead, backed by per-user YAML files under huawei.user.dir - see +# docker/huawei-user.yml.template and the "Testing locally" section in README.md. +#huawei.user.repository.class=org.radarbase.connect.rest.huawei.user.HuaweiYamlUserRepository +#huawei.user.dir=/var/lib/kafka-connect-huawei-source/users + +huawei.user.repository.class=org.radarbase.connect.rest.huawei.user.HuaweiServiceUserRepository +huawei.user.repository.url=http://localhost:8080/ +huawei.user.repository.client.id=radar_huawei_connector +huawei.user.repository.client.secret= +huawei.user.repository.oauth2.token.url= diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1bad2750..1971e421 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -16,6 +16,9 @@ sentryOpenTelemetryAgent = "8.36.0" okhttp = "4.12.0" firebaseAdmin = "9.8.0" radarSchemas = "0.8.16" +# First radar-schemas-commons release with the Huawei connector schemas; kept separate so the +# other connectors can move to it independently. +radarSchemasHuawei = "0.9.1" # @pin Upgrade to 3.x.x requires kotlin v2 minimum ktor = "2.3.13" wiremock = "3.0.1" @@ -28,6 +31,7 @@ lz4 = "1.10.1" lz4 = { module = "at.yawk.lz4:lz4-java", version.ref = "lz4" } radar-commons-kotlin = { module = "org.radarbase:radar-commons-kotlin", version.ref = "radarCommons" } radar-schemas-commons = { module = "org.radarbase:radar-schemas-commons", version.ref = "radarSchemas" } +radar-schemas-commons-huawei = { module = "org.radarbase:radar-schemas-commons", version.ref = "radarSchemasHuawei" } kafka-connect-api = "org.apache.kafka:connect-api:7.8.1-ce" kafka-connect-avro-converter = { module = "io.confluent:kafka-connect-avro-converter", version.ref = "confluent" } okhttp = { module = "com.squareup.okhttp3:okhttp", version.ref = "okhttp" } @@ -55,6 +59,7 @@ mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockito" } wiremock = { module = "com.github.tomakehurst:wiremock", version.ref = "wiremock" } kotlin-test = { module = "org.jetbrains.kotlin:kotlin-test", version.ref = "kotlin" } kotlin-test-junit = { module = "org.jetbrains.kotlin:kotlin-test-junit", version.ref = "kotlin" } +kotlin-test-junit5 = { module = "org.jetbrains.kotlin:kotlin-test-junit5", version.ref = "kotlin" } [plugins] radar-root-project = { id = "org.radarbase.radar-root-project", version.ref = "radarCommons" } diff --git a/huawei-library/build.gradle b/huawei-library/build.gradle new file mode 100644 index 00000000..d69340f2 --- /dev/null +++ b/huawei-library/build.gradle @@ -0,0 +1,48 @@ + +group = 'org.radarbase' +version = '0.0.1' + +apply plugin: 'maven-publish' + +repositories { + // Use jcenter for resolving dependencies. + // You can declare any Maven/Ivy/file repository here. + mavenCentral() +} + +dependencies { + // Use the Kotlin JDK 8 standard library. + implementation libs.kotlin.stdlib + + implementation libs.okhttp + + implementation libs.radar.schemas.commons.huawei + + implementation libs.jackson.annotations + + implementation libs.jackson.databind + + implementation libs.avro + + implementation libs.jackson.datatype.jsr310 + + // Use the Kotlin test library. + testImplementation libs.kotlin.test + + // Use the Kotlin JUnit 5 integration (matches the JUnit Platform the radar-kotlin + // Gradle plugin configures the `test` task with). + testImplementation libs.kotlin.test.junit5 +} + +project.afterEvaluate { + publishing { + publications { + library(MavenPublication) { + setGroupId "$group" + setArtifactId "huawei-library" + version "$version" + from components.java + } + } + } +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/FieldValues.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/FieldValues.kt new file mode 100644 index 00000000..10c97002 --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/FieldValues.kt @@ -0,0 +1,154 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.converter + +import com.fasterxml.jackson.databind.JsonNode + +/** + * Typed accessor for a single Huawei Health Kit sample point's field values. + * + * The Health Kit Data API (`sampleSet:polymerize`) represents each field of a sample point using + * the same typed-value wrapper as the on-device HiHealth SDK's `Field`/`Value` model: a list of + * objects shaped like `{"fieldName": "steps_delta", "integerValue": 123}` (or `floatValue`, + * `longValue`, `stringValue` depending on the field's declared type). This class also tolerates a + * flattened `{"fieldName": value, ...}` object, in case a particular endpoint or API version + * returns the simplified shape, so a single parser can be reused across all sample-set based + * routes. + * + * Field name constants follow Huawei's public `Field` identifiers (e.g. `steps_delta`, `calories`, + * `avg`, `max`, `min`), as documented for the on-device and REST Health Kit APIs. + * + * @author yatharthranjan + */ +class FieldValues private constructor(private val values: Map) { + + /** Huawei's own ID of the record these fields belong to, where the endpoint returns one + * (e.g. `healthRecords`). */ + var recordId: String? = null + private set + + /** Field values of the associated detail sample points of a health record (its + * `subDataDetails`), in response order, when they were requested and returned. */ + var subData: List = emptyList() + private set + + /** Copy of these field values with the given record-level context attached. */ + fun withRecord(recordId: String?, subData: List): FieldValues = + FieldValues(values).also { + it.recordId = recordId + it.subData = subData + } + + /* + * Every accessor accepts one or more candidate keys and returns the first one present, so a + * field whose casing Huawei's docs don't pin down unambiguously can list both spellings. + */ + + fun getInt(vararg fields: String): Int? = number(fields)?.let { + if (it.isNumber) it.asInt() else it.asText().trim().toIntOrNull() + } + + fun getLong(vararg fields: String): Long? = number(fields)?.let { + if (it.isNumber) it.asLong() else it.asText().trim().toLongOrNull() + } + + fun getDouble(vararg fields: String): Double? = number(fields)?.let { + if (it.isNumber) it.asDouble() else it.asText().trim().toDoubleOrNull() + } + + fun getFloat(vararg fields: String): Float? = getDouble(*fields)?.toFloat() + + /** Textual fields are returned as-is; array/object-valued fields (e.g. the ECG voltage sample + * list) are returned as their JSON serialization rather than Jackson's empty `asText()`. */ + fun getString(vararg fields: String): String? = lookup(fields)?.let { + if (it.isContainerNode) it.toString() else it.asText() + } + + /** + * Reads a `Map`-typed field (e.g. Huawei's exercise-type-to-duration map), + * as an object whose keys are stringified (Avro maps require string keys). Best-effort: the + * exact wire shape of a Huawei map-typed field is not confirmed against a live API response + * (Huawei's typed-value array uses `integerValue`/`floatValue`/`stringValue`/`longValue` for + * scalars, so `mapValue` is assumed by the same `Value` convention). Accepts a plain + * `{"key": 1}` object, an object of typed values `{"key": {"integerValue": 1}}`, or an array of + * `{"key": ..., "value": ...}` entries. + */ + fun getIntMap(vararg fields: String): Map? { + val node = lookup(fields) ?: return null + return when { + node.isObject -> node.properties().mapNotNull { (key, value) -> + unwrap(value)?.let { key to it.asInt() } + }.toMap() + node.isArray -> node.mapNotNull { entry -> + val key = entry.get("key")?.takeUnless { it.isNull }?.asText() + ?: return@mapNotNull null + unwrap(entry.get("value"))?.let { key to it.asInt() } + }.toMap() + else -> null + } + } + + private fun lookup(fields: Array): JsonNode? = + fields.firstNotNullOfOrNull { field -> values[field]?.takeUnless { it.isNull } } + + /** Numeric or textual node; Jackson's `asInt()` etc. would turn anything else (and + * non-numeric text) into 0 rather than null. */ + private fun number(fields: Array): JsonNode? = + lookup(fields)?.takeIf { it.isNumber || it.isTextual } + + companion object { + private const val FIELD_NAME_KEY = "fieldName" + private val VALUE_KEYS = + listOf("integerValue", "floatValue", "longValue", "stringValue", "mapValue", "value") + + /** Unwraps a typed-value wrapper (`{"integerValue": 1}`) to its value; returns other + * nodes unchanged. */ + private fun unwrap(node: JsonNode?): JsonNode? { + if (node == null || node.isNull || node.isMissingNode) return null + if (!node.isObject) return node + return VALUE_KEYS.firstNotNullOfOrNull { key -> node.get(key) } ?: node + } + + fun from(node: JsonNode?): FieldValues { + if (node == null || node.isMissingNode || node.isNull) { + return FieldValues(emptyMap()) + } + if (node.isArray) { + val map = LinkedHashMap() + node.forEach { entry -> + val name = entry.get(FIELD_NAME_KEY)?.asText() ?: return@forEach + // Prefer the known typed-value keys; fall back to whatever other single + // property the entry carries, in case Huawei uses a type name not listed here. + val value = VALUE_KEYS.firstNotNullOfOrNull { key -> entry.get(key) } + ?: entry.properties().firstOrNull { (key, _) -> key != FIELD_NAME_KEY } + ?.value + if (value != null) { + map[name] = value + } + } + return FieldValues(map) + } + if (node.isObject) { + val map = LinkedHashMap() + node.properties().forEach { (name, value) -> map[name] = value } + return FieldValues(map) + } + return FieldValues(emptyMap()) + } + } +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/HuaweiActivityRecordConverter.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/HuaweiActivityRecordConverter.kt new file mode 100644 index 00000000..cbc228df --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/HuaweiActivityRecordConverter.kt @@ -0,0 +1,100 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.radarbase.huawei.user.User +import org.radarcns.connector.huawei.HuaweiActivityRecord +import java.time.Instant + +/** + * Converts `GET /healthkit/v2/activityRecords` responses into [HuaweiActivityRecord]s. + * + * Field names below follow the Huawei Health Kit `ActivityRecord`/`Device`/`ActivitySummary` + * model (activity record id, name, description, time zone, activity type, device manufacturer and + * type, and a nested activity summary with pace/data/section statistics). Nested JSON structures + * that map to free-form Avro `string` fields (pace map, data summary, section summary) are kept as + * their raw JSON text, since their internal shape varies by activity type. + * + * @author yatharthranjan + */ +class HuaweiActivityRecordConverter( + private val topic: String = "connect_huawei_activity_record", +) : HuaweiDataConverter { + + override fun processRecords(root: JsonNode, user: User): Sequence> { + val timeReceived = Instant.now() + val records = root.get("activityRecord") + ?: root.get("activityRecords") + ?: root.get("records") + ?: return emptySequence() + return records.asSequence() + .mapCatching { record -> + val startTime = record.epochInstant("startTime") + ?: error("Huawei activity record is missing startTime") + TopicData( + topic = topic, + key = user.observationKey, + offset = startTime.epochSecond, + value = record.toActivityRecord(startTime, timeReceived), + ) + } + } + + private fun JsonNode.toActivityRecord( + startTime: Instant, + timeReceived: Instant, + ): HuaweiActivityRecord { + val device = this.get("device") + val summary = this.get("activitySummary") + return HuaweiActivityRecord.newBuilder().apply { + time = startTime.toEpoch() + this.timeReceived = timeReceived.toEpoch() + endTime = epochInstant("endTime")?.toEpoch() + activityRecordId = textOrNull("id") ?: textOrNull("activityRecordId") + name = textOrNull("name") + description = textOrNull("desc") ?: textOrNull("description") + timeZone = textOrNull("timeZone") + activityTypeId = textOrNull("activityType") ?: textOrNull("activityTypeId") + activeTimeMillis = longOrNull("activeTime") ?: longOrNull("activeTimeMillis") + isKeepGoing = boolOrNull("isKeepGoing") + deviceManufacturer = device?.textOrNull("manufacturer") + deviceType = device?.intOrNull("type") + activitySummaryAvgPace = summary?.doubleOrNull("avgPace") + activitySummaryBestPace = summary?.doubleOrNull("bestPace") + activitySummaryPaceMap = summary?.get("paceMap")?.toString() + activitySummaryDataSummary = summary?.get("dataSummary")?.toString() + activitySummarySectionSummary = summary?.get("sectionSummary")?.toString() + }.build() + } + + private fun JsonNode.textOrNull(field: String): String? = + this.get(field)?.takeUnless { it.isNull }?.asText() + + private fun JsonNode.intOrNull(field: String): Int? = + this.get(field)?.takeUnless { it.isNull }?.asInt() + + private fun JsonNode.longOrNull(field: String): Long? = + this.get(field)?.takeUnless { it.isNull }?.asLong() + + private fun JsonNode.doubleOrNull(field: String): Double? = + this.get(field)?.takeUnless { it.isNull }?.asDouble() + + private fun JsonNode.boolOrNull(field: String): Boolean? = + this.get(field)?.takeUnless { it.isNull }?.asBoolean() +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/HuaweiDailyPolymerizeConverter.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/HuaweiDailyPolymerizeConverter.kt new file mode 100644 index 00000000..7a52460d --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/HuaweiDailyPolymerizeConverter.kt @@ -0,0 +1,38 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.converter + +import org.apache.avro.specific.SpecificRecord +import java.time.Instant + +/** + * Converter for `POST /healthkit/v2/sampleSet:dailyPolymerize` responses. Each day's result is + * wrapped in a `group[]` entry containing its own `sampleSet[].samplePoints[]`, which + * [HuaweiSampleSetConverter] already handles, so this only exists to name the endpoint. + * + * @author yatharthranjan + */ +class HuaweiDailyPolymerizeConverter( + topic: String, + buildRecord: ( + fields: FieldValues, + startTime: Instant, + endTime: Instant?, + timeReceived: Instant, + ) -> SpecificRecord, +) : HuaweiDataConverter by HuaweiSampleSetConverter(topic, buildRecord) diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/HuaweiDataConverter.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/HuaweiDataConverter.kt new file mode 100644 index 00000000..532e2ecd --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/HuaweiDataConverter.kt @@ -0,0 +1,60 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.converter + +import com.fasterxml.jackson.databind.JsonNode +import okhttp3.Headers +import org.radarbase.huawei.request.HuaweiRequestGenerator.Companion.JSON_READER +import org.radarbase.huawei.request.RestRequest +import org.radarbase.huawei.user.User +import java.time.Instant + +/** + * Converts a Huawei Health Kit HTTP JSON response body to zero or more [TopicData] records. + * + * @author yatharthranjan + */ +interface HuaweiDataConverter : RecordConverter { + /** Process the JSON records generated by given request. */ + fun processRecords( + root: JsonNode, + user: User, + ): Sequence> + + override fun convert( + request: RestRequest, + headers: Headers, + data: ByteArray, + ): List = convert(request, JSON_READER.readTree(data)) + + /** Convert an already-parsed response body, logging and skipping records that fail. */ + fun convert(request: RestRequest, root: JsonNode): List = + this.processRecords(root, request.user) + .mapNotNull { r -> + r.fold( + { it }, + { + logger.error("Data conversion failed for {}: {}", request, it.toString()) + null + }, + ) + } + .toList() + + fun Instant.toEpoch(): Double = this.toEpochMilli() / 1000.0 +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/HuaweiHealthRecordConverter.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/HuaweiHealthRecordConverter.kt new file mode 100644 index 00000000..dfa22f90 --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/HuaweiHealthRecordConverter.kt @@ -0,0 +1,77 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.apache.avro.specific.SpecificRecord +import org.radarbase.huawei.user.User +import java.time.Instant + +/** + * Generic converter for `GET /healthkit/v2/healthRecords` responses: iterates every record + * returned for the requested `dataType` and builds one Avro record per entry via + * [buildRecord]. The record's `id` and any associated detail sample points are exposed through + * [FieldValues.recordId] and [FieldValues.subData]. + * + * @author yatharthranjan + */ +class HuaweiHealthRecordConverter( + private val topic: String, + private val buildRecord: ( + fields: FieldValues, + startTime: Instant, + endTime: Instant?, + timeReceived: Instant, + ) -> SpecificRecord, +) : HuaweiDataConverter { + + /** Detail sample points returned with a record when `subDataType` was requested. The + * HealthRecord model nests them as sample sets; bare sample points are accepted too. */ + private fun JsonNode.subDataPoints(): Sequence { + val details = get("subDataDetails")?.takeIf { it.isArray } ?: return emptySequence() + return details.asSequence().flatMap { detail -> + detail.get("samplePoints")?.takeIf { it.isArray }?.asSequence() + ?: sequenceOf(detail) + } + } + + override fun processRecords(root: JsonNode, user: User): Sequence> { + val timeReceived = Instant.now() + val records = root.get("healthRecords") ?: root.get("records") ?: return emptySequence() + return records.asSequence() + .mapCatching { record -> + val startTime = record.epochInstant("startTime") + ?: error("Huawei health record is missing startTime") + val endTime = record.epochInstant("endTime") + val fieldValues = FieldValues.from( + record.get("value") ?: record.get("fieldValues") ?: record.get("field"), + ).withRecord( + recordId = record.get("id")?.takeIf { it.isTextual }?.asText(), + subData = record.subDataPoints() + .map { FieldValues.from(it.get("value") ?: it.get("fieldValues")) } + .toList(), + ) + TopicData( + topic = topic, + key = user.observationKey, + offset = startTime.epochSecond, + value = buildRecord(fieldValues, startTime, endTime, timeReceived), + ) + } + } +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/HuaweiSampleSetConverter.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/HuaweiSampleSetConverter.kt new file mode 100644 index 00000000..837732d7 --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/HuaweiSampleSetConverter.kt @@ -0,0 +1,86 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.converter + +import com.fasterxml.jackson.databind.JsonNode +import org.apache.avro.specific.SpecificRecord +import org.radarbase.huawei.user.User +import java.time.Instant + +/** + * Generic converter for `sampleSet:polymerize` and `sampleSet:dailyPolymerize` responses: + * iterates every sample point of every sample set in the response and builds one Avro record per + * point via [buildRecord]. + * + * Huawei wraps sample sets in a `group[]` array (one entry per aggregation bucket or day), each + * holding `sampleSet[].samplePoints[]`. A bare top-level `sampleSet[]` is also accepted. Sample + * point times are nanoseconds in Huawei's examples, while group times are milliseconds; units are + * inferred per value (see [epochInstant]). + * + * This single converter is reused for the large majority of Huawei Health Kit data types, since + * they all share the same response envelope and differ only in which Avro record type their field + * values are mapped onto. + * + * @author yatharthranjan + */ +class HuaweiSampleSetConverter( + private val topic: String, + private val buildRecord: ( + fields: FieldValues, + startTime: Instant, + endTime: Instant?, + timeReceived: Instant, + ) -> SpecificRecord, +) : HuaweiDataConverter { + + override fun processRecords(root: JsonNode, user: User): Sequence> { + val timeReceived = Instant.now() + return root.samplePoints() + .mapCatching { point -> + val startTime = point.epochInstant("startTime") + ?: error("Huawei sample point is missing startTime") + val endTime = point.epochInstant("endTime") + val fieldValues = FieldValues.from(point.get("value") ?: point.get("fieldValues")) + TopicData( + topic = topic, + key = user.observationKey, + offset = startTime.epochSecond, + value = buildRecord(fieldValues, startTime, endTime, timeReceived), + ) + } + } + + companion object { + private fun JsonNode.child(vararg names: String): JsonNode? = + names.firstNotNullOfOrNull { name -> get(name)?.takeIf { it.isArray } } + + /** All sample points in a (daily) polymerize response, with or without `group[]`. */ + internal fun JsonNode.samplePoints(): Sequence { + val sampleSets = child("group", "groups") + ?.asSequence() + ?.flatMap { group -> + group.child("sampleSet", "sampleSets")?.asSequence().orEmpty() + } + ?: child("sampleSet", "sampleSets")?.asSequence() + ?: emptySequence() + return sampleSets.flatMap { sampleSet -> + sampleSet.child("samplePoints", "samplePoint")?.asSequence().orEmpty() + } + } + } +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/HuaweiTimestamps.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/HuaweiTimestamps.kt new file mode 100644 index 00000000..7a00c70d --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/HuaweiTimestamps.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.huawei.converter + +import com.fasterxml.jackson.databind.JsonNode +import java.time.Instant + +/** + * Reads an epoch timestamp field from a Huawei response node, whatever its unit. + * + * Huawei's endpoints are inconsistent about timestamp units - `activityRecords` uses milliseconds, + * `healthRecords` and sample points use nanoseconds, and `dailyPolymerize` groups use milliseconds + * around nanosecond sample points - and the docs don't always say which. Any plausible timestamp + * (after 2001) differs by at least three orders of magnitude between units, so the unit is + * inferred from the value's magnitude instead of hard-coded per endpoint. Both numeric and + * stringified numbers are accepted. + * + * @author yatharthranjan + */ +internal fun JsonNode.epochInstant(field: String): Instant? { + val node = this.get(field) ?: return null + if (node.isNull) return null + val value = if (node.isTextual) node.asText().trim().toLongOrNull() else node.asLong() + return value?.let { epochInstantOf(it) } +} + +internal fun epochInstantOf(value: Long): Instant = when { + value >= NANOS_THRESHOLD -> Instant.ofEpochSecond( + Math.floorDiv(value, 1_000_000_000L), + Math.floorMod(value, 1_000_000_000L), + ) + value >= MICROS_THRESHOLD -> Instant.ofEpochSecond( + Math.floorDiv(value, 1_000_000L), + Math.floorMod(value, 1_000_000L) * 1_000L, + ) + value >= MILLIS_THRESHOLD -> Instant.ofEpochMilli(value) + else -> Instant.ofEpochSecond(value) +} + +// 1e9 s is 2001-09-09, so any post-2001 time is >= 1e12 in ms, >= 1e15 in us and >= 1e18 in ns. +// Lower thresholds by 10x to keep boundaries far from any realistic value in the adjacent unit. +private const val MILLIS_THRESHOLD = 100_000_000_000L +private const val MICROS_THRESHOLD = 100_000_000_000_000L +private const val NANOS_THRESHOLD = 100_000_000_000_000_000L diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/RecordConverter.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/RecordConverter.kt new file mode 100644 index 00000000..ca82e213 --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/RecordConverter.kt @@ -0,0 +1,39 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.converter + +import okhttp3.Headers +import org.radarbase.huawei.request.RestRequest +import org.slf4j.LoggerFactory +import java.io.IOException + +/** + * @author yatharthranjan + */ +interface RecordConverter { + @Throws(IOException::class) + fun convert( + request: RestRequest, + headers: Headers, + data: ByteArray, + ): List + + companion object { + var logger = LoggerFactory.getLogger(RecordConverter::class.java) + } +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/SequenceExtensions.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/SequenceExtensions.kt new file mode 100644 index 00000000..62746512 --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/SequenceExtensions.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.converter + +import org.slf4j.LoggerFactory + +/** + * @author yatharthranjan + */ +val logger = LoggerFactory.getLogger("org.radarbase.huawei.converter.SequenceExtensions") + +internal fun Sequence.mapCatching(fn: (T) -> S): Sequence> = map { t -> + runCatching { + fn(t) + } +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/TopicData.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/TopicData.kt new file mode 100644 index 00000000..aadd6a09 --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/converter/TopicData.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.converter + +import org.apache.avro.specific.SpecificRecord + +/** Single value for a topic. */ +/** + * @author yatharthranjan + */ +data class TopicData( + val topic: String, + val key: SpecificRecord, + val value: SpecificRecord, + val offset: Long, +) diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/offset/Offset.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/offset/Offset.kt new file mode 100644 index 00000000..791b0065 --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/offset/Offset.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.offset + +import org.radarbase.huawei.route.Route +import org.radarbase.huawei.user.User +import java.time.Instant + +/** + * @author yatharthranjan + */ +data class Offset( + val user: User, + val route: Route, + val offset: Instant, +) diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/offset/Offsets.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/offset/Offsets.kt new file mode 100644 index 00000000..1c31c8b7 --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/offset/Offsets.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.offset + +/** + * @author yatharthranjan + */ +data class Offsets( + val offsets: List, +) diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/request/HuaweiOffsetManager.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/request/HuaweiOffsetManager.kt new file mode 100644 index 00000000..0709396f --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/request/HuaweiOffsetManager.kt @@ -0,0 +1,33 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.request + +import org.radarbase.huawei.offset.Offset +import org.radarbase.huawei.route.Route +import org.radarbase.huawei.user.User +import java.time.Instant + +/** + * @author yatharthranjan + */ +interface HuaweiOffsetManager { + + fun getOffset(route: Route, user: User): Offset? + + fun updateOffsets(route: Route, user: User, offset: Instant) +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/request/HuaweiRequestGenerator.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/request/HuaweiRequestGenerator.kt new file mode 100644 index 00000000..3de16e49 --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/request/HuaweiRequestGenerator.kt @@ -0,0 +1,348 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.request + +import com.fasterxml.jackson.core.JsonFactory +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.MissingNode +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import okhttp3.Response +import org.radarbase.huawei.converter.TopicData +import org.radarbase.huawei.route.Route +import org.radarbase.huawei.user.User +import org.radarbase.huawei.user.UserNotAuthorizedException +import org.radarbase.huawei.user.UserRepository +import org.slf4j.LoggerFactory +import java.io.IOException +import java.time.Duration +import java.time.Instant +import java.util.concurrent.ConcurrentHashMap + +/** + * @author yatharthranjan + */ +class HuaweiRequestGenerator( + private val userRepository: UserRepository, + private val huaweiOffsetManager: HuaweiOffsetManager, + val routes: List, +) : RequestGenerator { + private val routeNextRequest: MutableMap = ConcurrentHashMap() + + /** Routes whose last request failed, until when. Stops the rest of a lazily generated + * request sequence for that route and user, so a later chunk can't advance the offset past + * a chunk that failed. */ + private val routeFailedUntil: MutableMap = ConcurrentHashMap() + + var nextRequestTime: Instant = Instant.MIN + + override fun requests(user: User, max: Int): Sequence = + routes.asSequence() + .flatMap { route -> requests(route, user, max) } + + override fun requests(route: Route, max: Int): Sequence = + userRepository.stream() + .flatMap { user -> requests(route, user, max) } + + override fun requests(route: Route, user: User, max: Int): Sequence = + if (routeReady(user, route)) { + generateRequests(route, user) + } else { + logger.debug( + "Skip {} for {}: route in backoff until {}", + route, + user.versionedId, + routeNextRequest[routeKey(route, user)], + ) + emptySequence() + } + + fun generateRequests(route: Route, user: User): Sequence { + val startOffset = currentOffset(route, user) + val endDate = user.endDate?.coerceAtMost(Instant.now()) ?: Instant.now() + if (!startOffset.isBefore(endDate)) { + logger.debug( + "Skip {} for {}: interval empty (startOffset={} >= endDate={})", + route, + user.versionedId, + startOffset, + endDate, + ) + return emptySequence() + } + val key = routeKey(route, user) + return route.generateRequests(user, startOffset, endDate, USER_MAX_REQUESTS) + .guarded(route, user) + .takeWhile { !isBlocked(key) } + } + + /** + * Requests are built lazily, and building one fetches the user's access token, which throws + * if the user is no longer authorized or the token endpoint is unreachable. Contain that to + * this route and user (backing it off) instead of letting it escape the source task's poll. + */ + private fun Sequence.guarded(route: Route, user: User): Sequence { + val source = this + return sequence { + val iterator = source.iterator() + while (true) { + val next = try { + if (!iterator.hasNext()) break + iterator.next() + } catch (ex: UserNotAuthorizedException) { + logger.warn("User {} is not authorized: {}", user.versionedId, ex.message) + backOff(route, user, USER_BACK_OFF_TIME) + break + } catch (ex: Exception) { + logger.warn( + "Failed to create {} request for {}: {}", + route, + user.versionedId, + ex.toString(), + ) + backOff(route, user, BACK_OFF_TIME) + break + } + yield(next) + } + } + } + + private fun currentOffset(route: Route, user: User): Instant { + val offset = huaweiOffsetManager.getOffset(route, user) + return if (offset == null) { + logger.info("No offsets found for {} on {}, using the start date.", user, route) + user.startDate + } else { + offset.offset.coerceAtLeast(user.startDate) + } + } + + fun handleResponse(req: RestRequest, response: Response): HuaweiResult> { + if (response.code == 400 && response.peekBody(PEEK_BYTES).string().contains(NO_COLLECTOR)) { + // Huawei's way of saying the user has no data source (so no data) for this type. + logger.debug("No data source of {} for {}", req.route, req.user.versionedId) + return HuaweiResult.Success(recordsReceived(req, MissingNode.getInstance())) + } + return if (response.isSuccessful) { + try { + HuaweiResult.Success(requestSuccessful(req, response)) + } catch (ex: IOException) { + // Unreadable or malformed body: don't advance the offset, and retry later. + logger.warn("Failed to read response of {}: {}", req, ex.toString()) + backOff(req.route, req.user, BACK_OFF_TIME) + HuaweiResult.Success(emptyList()) + } + } else { + try { + HuaweiResult.Error(requestFailed(req, response)) + } catch (e: TooManyRequestsException) { + HuaweiResult.Success(emptyList()) + } + } + } + + /** + * Converts the response and advances the route's offset. + * + * Huawei devices sync to the Huawei cloud with a delay of minutes to days, so data for a + * period can appear after that period was already queried. The offset is therefore never + * advanced past [LATE_SYNC_WINDOW] ago on the basis of an empty response: only past the + * latest record actually received. Records starting before the current offset were already + * emitted (the offset sits just past the latest one) and are dropped, since Huawei also + * returns samples that merely overlap the queried window. + */ + override fun requestSuccessful(request: RestRequest, response: Response): List { + logger.debug("Request successful: {}..", request.request) + val data = response.body?.bytes() ?: ByteArray(0) + val root: JsonNode = if (data.isEmpty()) { + MissingNode.getInstance() + } else { + JSON_READER.readTree(data) ?: MissingNode.getInstance() + } + return recordsReceived(request, root) + } + + private fun recordsReceived(request: RestRequest, root: JsonNode): List { + val now = Instant.now() + val currentOffset = currentOffset(request.route, request.user) + val records = request.route.converters + .flatMap { it.convert(request, root) } + .filter { it.offset >= currentOffset.epochSecond } + + val settledEnd = request.endDate.coerceAtMost(now.minus(LATE_SYNC_WINDOW)) + val maxOffset = records.maxOfOrNull { it.offset } + val nextOffset = if (maxOffset != null) { + val afterLatest = Instant.ofEpochSecond(maxOffset + 1) + if (root.path("hasMoreData").asBoolean(false)) { + // Only part of this window was returned; continue right after the latest record. + logger.info( + "More {} data available for {} than returned; continuing after {}", + request.route, + request.user.versionedId, + afterLatest, + ) + afterLatest + } else { + afterLatest.coerceAtLeast(settledEnd) + } + } else { + settledEnd.coerceAtLeast(currentOffset) + } + if (nextOffset.isAfter(currentOffset)) { + huaweiOffsetManager.updateOffsets(request.route, request.user, nextOffset) + } + + val key = routeKey(request.route, request.user) + routeFailedUntil -= key + routeNextRequest[key] = if (records.isEmpty() && request.endDate > settledEnd) { + // Caught up to data that may still be syncing: check again later. + now.plus(CAUGHT_UP_BACK_OFF_TIME) + } else { + now.plus(SUCCESS_BACK_OFF_TIME) + } + return records + } + + override fun requestFailed(request: RestRequest, response: Response): HuaweiError { + return when (response.code) { + 429 -> { + logger.info( + "Too many requests, rate limit reached. Backing off... {}", + response.body?.string(), + ) + nextRequestTime = Instant.now() + BACK_OFF_TIME + backOff(request.route, request.user, BACK_OFF_TIME) + HuaweiRateLimitError("Rate limit reached..", TooManyRequestsException(), "429") + } + 403 -> { + val body = response.body?.string() ?: "no response body" + logger.warn( + "User {} does not have access to this Huawei Health Kit data type: {}", + request.user, + body, + ) + backOff(request.route, request.user, USER_BACK_OFF_TIME) + HuaweiAccessForbiddenError( + "Huawei Health Kit scope not granted or data not available: $body", + IOException("Forbidden"), + "403", + ) + } + 401 -> { + val body = response.body?.string() ?: "no response body" + logger.warn( + "User {} access token is expired, malformed, or revoked: {}", + request.user, + body, + ) + // Refetch the token on the next attempt; a user whose refresh token is also no + // longer valid is then backed off when building that request. + userRepository.invalidateAccessToken(request.user) + backOff(request.route, request.user, BACK_OFF_TIME) + HuaweiUnauthorizedAccessError( + "Access token expired or revoked: $body", + IOException("Unauthorized"), + "401", + ) + } + 400 -> { + val body = response.body?.string() ?: "no response body" + logger.warn("Client exception for request {}: {}", request, body) + // An unknown data type name won't become valid by retrying soon. + val permanent = body.contains(INVALID_DATA_TYPE) + backOff( + request.route, + request.user, + if (permanent) USER_BACK_OFF_TIME else BACK_OFF_TIME, + ) + HuaweiClientException( + "Client unsupported or unauthorized: $body", + IOException("Invalid client"), + "400", + ) + } + 422 -> { + logger.warn("Request failed (validation error): {}, {}", request, response) + backOff(request.route, request.user, BACK_OFF_TIME) + HuaweiValidationError( + response.body?.string() ?: "validation error", + IOException("Validation error"), + "422", + ) + } + 404 -> { + logger.warn("Not found: {}", request) + backOff(request.route, request.user, BACK_OFF_TIME) + HuaweiNotFoundError( + response.body?.string() ?: "not found", + IOException("Data not found"), + "404", + ) + } + else -> { + val body = response.body?.string() ?: "unknown error" + logger.warn("Request failed: {}: {}", request, body) + backOff(request.route, request.user, BACK_OFF_TIME) + HuaweiGenericError( + body, + IOException("Unknown error"), + response.code.toString(), + ) + } + } + } + + private fun backOff(route: Route, user: User, duration: Duration) { + val key = routeKey(route, user) + val until = Instant.now().plus(duration) + routeNextRequest[key] = until + routeFailedUntil[key] = until + } + + private fun routeReady(user: User, route: Route): Boolean { + val now = Instant.now() + val key = routeKey(route, user) + return now > nextRequestTime && routeNextRequest[key]?.let { now > it } ?: true + } + + /** Whether requests for this route and user must stop, after a failure or rate limit. */ + private fun isBlocked(key: String): Boolean { + val now = Instant.now() + return now <= nextRequestTime || routeFailedUntil[key]?.let { now <= it } ?: false + } + + private fun routeKey(route: Route, user: User): String = user.versionedId + "#" + route + + companion object { + private val logger = LoggerFactory.getLogger(HuaweiRequestGenerator::class.java) + private val BACK_OFF_TIME = Duration.ofMinutes(10L) + private val USER_BACK_OFF_TIME = Duration.ofHours(12L) + private val SUCCESS_BACK_OFF_TIME = Duration.ofSeconds(10L) + private val CAUGHT_UP_BACK_OFF_TIME = Duration.ofMinutes(30L) + + /** How long after the fact Huawei data may still be synced to the cloud. */ + private val LATE_SYNC_WINDOW = Duration.ofDays(7L) + private const val USER_MAX_REQUESTS = 1000 + private const val PEEK_BYTES = 64L * 1024L + private const val NO_COLLECTOR = "no default dataCollector found" + private const val INVALID_DATA_TYPE = "Invalid dataTypeName" + val JSON_FACTORY = JsonFactory() + val JSON_READER = ObjectMapper(JSON_FACTORY).registerModule(JavaTimeModule()).reader() + } +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/request/HuaweiResult.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/request/HuaweiResult.kt new file mode 100644 index 00000000..0f9dda6d --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/request/HuaweiResult.kt @@ -0,0 +1,76 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.request + +/** + * @author yatharthranjan + */ +sealed class HuaweiResult { + data class Success(val value: T) : HuaweiResult() + data class Error(val error: HuaweiError) : HuaweiResult() +} + +sealed interface HuaweiError + +sealed class HuaweiErrorBase( + val message: String, + val cause: Exception? = null, + val code: String, +) : HuaweiError + +class HuaweiRateLimitError( + message: String, + cause: Exception? = null, + code: String, +) : HuaweiErrorBase(message, cause, code) + +class HuaweiClientException( + message: String, + cause: Exception? = null, + code: String, +) : HuaweiErrorBase(message, cause, code) + +class HuaweiUnauthorizedAccessError( + message: String, + cause: Exception? = null, + code: String, +) : HuaweiErrorBase(message, cause, code) + +class HuaweiAccessForbiddenError( + message: String, + cause: Exception? = null, + code: String, +) : HuaweiErrorBase(message, cause, code) + +class HuaweiValidationError( + message: String, + cause: Exception? = null, + code: String, +) : HuaweiErrorBase(message, cause, code) + +class HuaweiGenericError( + message: String, + cause: Exception? = null, + code: String, +) : HuaweiErrorBase(message, cause, code) + +class HuaweiNotFoundError( + message: String, + cause: Exception? = null, + code: String, +) : HuaweiErrorBase(message, cause, code) diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/request/RequestGenerator.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/request/RequestGenerator.kt new file mode 100644 index 00000000..bf48a457 --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/request/RequestGenerator.kt @@ -0,0 +1,39 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.request + +import okhttp3.Response +import org.radarbase.huawei.converter.TopicData +import org.radarbase.huawei.route.Route +import org.radarbase.huawei.user.User + +/** + * @author yatharthranjan + */ +interface RequestGenerator { + + fun requests(user: User, max: Int): Sequence + + fun requests(route: Route, user: User, max: Int): Sequence + + fun requests(route: Route, max: Int): Sequence + + fun requestSuccessful(request: RestRequest, response: Response): List + + fun requestFailed(request: RestRequest, response: Response): HuaweiError +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/request/RestRequest.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/request/RestRequest.kt new file mode 100644 index 00000000..ef199dbf --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/request/RestRequest.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.request + +import okhttp3.Request +import org.radarbase.huawei.route.HuaweiRoute +import org.radarbase.huawei.user.User +import java.time.Instant + +/** + * @author yatharthranjan + */ +data class RestRequest( + val request: Request, + val user: User, + val route: HuaweiRoute, + val startDate: Instant, + val endDate: Instant, +) diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/request/TooManyRequestsException.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/request/TooManyRequestsException.kt new file mode 100644 index 00000000..db54d069 --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/request/TooManyRequestsException.kt @@ -0,0 +1,23 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.request + +/** + * @author yatharthranjan + */ +class TooManyRequestsException : RuntimeException() diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiActivityRecordRoute.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiActivityRecordRoute.kt new file mode 100644 index 00000000..c8c8b347 --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiActivityRecordRoute.kt @@ -0,0 +1,67 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.route + +import org.radarbase.huawei.converter.HuaweiActivityRecordConverter +import org.radarbase.huawei.converter.HuaweiDataConverter +import org.radarbase.huawei.request.RestRequest +import org.radarbase.huawei.user.User +import org.radarbase.huawei.user.UserRepository +import java.time.Duration +import java.time.Instant + +/** + * Route backed by `GET /healthkit/v2/activityRecords`, covering the Huawei Health Kit Activity + * Records API (workout / physical-activity sessions). + * + * @author yatharthranjan + */ +class HuaweiActivityRecordRoute( + userRepository: UserRepository, + private val topic: String = "connect_huawei_activity_record", + maxIntervalPerRequest: Duration = Duration.ofDays(30L), +) : HuaweiRoute(userRepository, maxIntervalPerRequest) { + + override val converters: List = + listOf(HuaweiActivityRecordConverter(topic)) + + override fun toString(): String = "huawei_activity_record" + + override fun generateRequests( + user: User, + start: Instant, + end: Instant, + max: Int, + ): Sequence = chunkedRanges(start, end, max).map { (rangeStart, rangeEnd) -> + RestRequest( + request = createGetRequest( + user, + "activityRecords", + mapOf( + "startTime" to rangeStart.toEpochMilli().toString(), + "endTime" to rangeEnd.toEpochMilli().toString(), + ), + baseUrl = HUAWEI_API_BASE_URL_V2, + ), + user = user, + route = this, + startDate = rangeStart, + endDate = rangeEnd, + ) + } +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiDailyPolymerizeRoute.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiDailyPolymerizeRoute.kt new file mode 100644 index 00000000..211c952f --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiDailyPolymerizeRoute.kt @@ -0,0 +1,118 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.route + +import com.fasterxml.jackson.databind.ObjectMapper +import org.apache.avro.specific.SpecificRecord +import org.radarbase.huawei.converter.FieldValues +import org.radarbase.huawei.converter.HuaweiDailyPolymerizeConverter +import org.radarbase.huawei.converter.HuaweiDataConverter +import org.radarbase.huawei.request.RestRequest +import org.radarbase.huawei.user.User +import org.radarbase.huawei.user.UserRepository +import java.time.Duration +import java.time.Instant +import java.time.ZoneOffset +import java.time.format.DateTimeFormatter +import java.time.temporal.ChronoUnit + +/** + * Route backed by `POST /healthkit/v2/sampleSet:dailyPolymerize`, used for every Huawei + * `.statistics` data type. + * + * Huawei's `sampleSet:polymerize` endpoint (see [HuaweiSampleSetRoute]) does not accept a + * `groupByTime`-aggregated query for every data type - some (confirmed live: `resting_calories`) + * reject it with `"does not support the query mode, please use dailyPolymerize API"`. This route + * calls that dedicated day-granularity statistics endpoint instead, which takes a day-string range + * (`startDay`/`endDay`, format `yyyyMMdd`, at most 31 days apart) rather than epoch timestamps. + * + * @author yatharthranjan + */ +open class HuaweiDailyPolymerizeRoute( + userRepository: UserRepository, + private val dataTypeName: String, + private val topic: String, + // dailyPolymerize accepts at most 31 days (inclusive) per request. + maxIntervalPerRequest: Duration = Duration.ofDays(30L), + buildRecord: ( + fields: FieldValues, + startTime: Instant, + endTime: Instant?, + timeReceived: Instant, + ) -> SpecificRecord, +) : HuaweiRoute(userRepository, maxIntervalPerRequest) { + + override val converters: List = + listOf(HuaweiDailyPolymerizeConverter(topic, buildRecord)) + + override fun toString(): String = "huawei_" + topic.removePrefix("connect_huawei_") + + /** + * Requests whole UTC days only: from the first midnight at or after [start] up to the last + * midnight at least [COMPLETION_DELAY] ago (capped at [end]), so a day's statistics are only + * fetched once that day is over and has had time to sync, and consecutive requests never + * overlap. Each request's `endDay` is inclusive, so it names the day before the exclusive + * range end. + */ + override fun generateRequests( + user: User, + start: Instant, + end: Instant, + max: Int, + ): Sequence { + val firstDay = start.ceilToDay() + val lastDayEnd = end.coerceAtMost(Instant.now().minus(COMPLETION_DELAY)).floorToDay() + if (!firstDay.isBefore(lastDayEnd)) return emptySequence() + return chunkedRanges(firstDay, lastDayEnd, max).map { (rangeStart, rangeEnd) -> + RestRequest( + request = createPostRequest( + user, + "sampleSet:dailyPolymerize", + buildRequestBody(rangeStart, rangeEnd.minus(Duration.ofDays(1))), + baseUrl = HUAWEI_API_BASE_URL_V2, + ), + user = user, + route = this, + startDate = rangeStart, + endDate = rangeEnd, + ) + } + } + + private fun buildRequestBody(firstDay: Instant, lastDay: Instant): String { + val root = MAPPER.createObjectNode() + root.putArray("dataTypes").add(dataTypeName) + root.put("startDay", DAY_FORMATTER.format(firstDay)) + root.put("endDay", DAY_FORMATTER.format(lastDay)) + root.put("timeZone", "+0000") + return MAPPER.writeValueAsString(root) + } + + companion object { + /** How long after the end of a UTC day its statistics are first requested. */ + private val COMPLETION_DELAY = Duration.ofHours(12L) + private val MAPPER = ObjectMapper() + + private fun Instant.floorToDay(): Instant = truncatedTo(ChronoUnit.DAYS) + + private fun Instant.ceilToDay(): Instant = floorToDay().let { + if (it == this) it else it.plus(Duration.ofDays(1)) + } + private val DAY_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd").withZone(ZoneOffset.UTC) + } +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiHealthRecordRoute.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiHealthRecordRoute.kt new file mode 100644 index 00000000..3d21a59c --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiHealthRecordRoute.kt @@ -0,0 +1,95 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.route + +import org.apache.avro.specific.SpecificRecord +import org.radarbase.huawei.converter.FieldValues +import org.radarbase.huawei.converter.HuaweiDataConverter +import org.radarbase.huawei.converter.HuaweiHealthRecordConverter +import org.radarbase.huawei.request.RestRequest +import org.radarbase.huawei.user.User +import org.radarbase.huawei.user.UserRepository +import java.time.Duration +import java.time.Instant + +/** + * Route backed by `GET /healthkit/v2/healthRecords`, used for the `health.record.*` data types + * (ambulatory blood pressure sessions, heart rate alerts, hyperthermia, low SpO2 alerts, + * menstrual cycle phases, and comprehensive sleep records) and other record-style types such as + * ECG measurement records, optionally with their associated detail data (`subDataType`). + * + * Per the official Health Kit REST API reference, this endpoint is on API version `v2` (unlike + * `sampleSet:polymerize`/`activityRecords`, which are on `v1`), takes the data type under the + * `dataType` query parameter (not `dataTypeName`), and its `startTime`/`endTime` parameters (and + * the `startTime`/`endTime` fields of each returned record) are in **nanoseconds** since the + * epoch, not milliseconds. + * + * @author yatharthranjan + */ +open class HuaweiHealthRecordRoute( + userRepository: UserRepository, + private val dataTypeName: String, + private val topic: String, + /** Associated detail data types to return with each record (`subDataType`). */ + private val subDataTypes: List = emptyList(), + maxIntervalPerRequest: Duration = Duration.ofDays(30L), + buildRecord: ( + fields: FieldValues, + startTime: Instant, + endTime: Instant?, + timeReceived: Instant, + ) -> SpecificRecord, +) : HuaweiRoute(userRepository, maxIntervalPerRequest) { + + override val converters: List = + listOf(HuaweiHealthRecordConverter(topic, buildRecord)) + + override fun toString(): String = "huawei_" + topic.removePrefix("connect_huawei_") + + override fun generateRequests( + user: User, + start: Instant, + end: Instant, + max: Int, + ): Sequence = chunkedRanges(start, end, max).map { (rangeStart, rangeEnd) -> + RestRequest( + request = createGetRequest( + user, + "healthRecords", + buildMap { + put("dataType", dataTypeName) + put("startTime", rangeStart.toEpochNanos().toString()) + put("endTime", rangeEnd.toEpochNanos().toString()) + if (subDataTypes.isNotEmpty()) { + put( + "subDataType", + subDataTypes.joinToString(","), + ) + } + }, + baseUrl = HUAWEI_API_BASE_URL_V2, + ), + user = user, + route = this, + startDate = rangeStart, + endDate = rangeEnd, + ) + } + + private fun Instant.toEpochNanos(): Long = epochSecond * 1_000_000_000L + nano.toLong() +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiRoute.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiRoute.kt new file mode 100644 index 00000000..9c7cd11e --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiRoute.kt @@ -0,0 +1,104 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.route + +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import org.radarbase.huawei.converter.HuaweiDataConverter +import org.radarbase.huawei.request.RestRequest +import org.radarbase.huawei.user.User +import org.radarbase.huawei.user.UserRepository +import java.time.Duration +import java.time.Instant + +/** + * Base class for all Huawei Health Kit routes. + * + * Handles OAuth2-authorized request construction (both `GET` with query parameters and `POST` + * with a JSON body, since the Health Kit Data API mixes both styles across its endpoints) and + * generic time-range chunking, shared by all concrete route types. + * + * @author yatharthranjan + */ +abstract class HuaweiRoute( + private val userRepository: UserRepository, + override val maxIntervalPerRequest: Duration = DEFAULT_INTERVAL_PER_REQUEST, +) : Route { + abstract val converters: List + + protected fun createGetRequest( + user: User, + path: String, + queryParams: Map, + baseUrl: String = HUAWEI_API_BASE_URL, + ): Request { + val accessToken = userRepository.getAccessToken(user) + val urlBuilder = "$baseUrl/$path".toHttpUrl().newBuilder() + queryParams.forEach { (key, value) -> urlBuilder.addQueryParameter(key, value) } + return Request.Builder() + .url(urlBuilder.build()) + .header("Authorization", "Bearer $accessToken") + .get() + .build() + } + + protected fun createPostRequest( + user: User, + path: String, + jsonBody: String, + baseUrl: String = HUAWEI_API_BASE_URL, + ): Request { + val accessToken = userRepository.getAccessToken(user) + return Request.Builder() + .url("$baseUrl/$path".toHttpUrl()) + .header("Authorization", "Bearer $accessToken") + .post(jsonBody.toRequestBody(JSON_MEDIA_TYPE)) + .build() + } + + /** + * Split `[start, end)` into consecutive windows of at most [maxIntervalPerRequest], capped at + * [max] windows. + */ + protected fun chunkedRanges( + start: Instant, + end: Instant, + max: Int, + ): Sequence> = + generateSequence(start) { it + maxIntervalPerRequest } + .takeWhile { it < end } + .take(max) + .map { rangeStart -> + rangeStart to (rangeStart + maxIntervalPerRequest).coerceAtMost(end) + } + + override fun generateRequests( + user: User, + start: Instant, + end: Instant, + ): Sequence = generateRequests(user, start, end, Int.MAX_VALUE) + + companion object { + const val HUAWEI_API_BASE_URL = "https://health-api.cloud.huawei.com/healthkit/v1" + const val HUAWEI_API_BASE_URL_V2 = "https://health-api.cloud.huawei.com/healthkit/v2" + private val JSON_MEDIA_TYPE = "application/json; charset=utf-8".toMediaType() + private val DEFAULT_INTERVAL_PER_REQUEST = Duration.ofDays(30L) + } +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiRouteDefinition.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiRouteDefinition.kt new file mode 100644 index 00000000..80a3679d --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiRouteDefinition.kt @@ -0,0 +1,38 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.route + +import org.radarbase.huawei.user.UserRepository + +/** + * A single registered Huawei Health Kit data type: a short config key (used to build + * `huawei..enabled` / `huawei..topic` connector properties), the default Kafka topic + * name, and a factory for the [HuaweiRoute] that queries it. + * + * Using one shared registry (see [HuaweiRouteFactory]) for both the Kafka Connect config + * definition and the set of routes actually polled avoids hand-duplicating each of the ~54 Huawei + * data types across a `ConfigDef` and a route-construction switch. + * + * @author yatharthranjan + */ +data class HuaweiRouteDefinition( + val key: String, + val defaultTopic: String, + val enabledByDefault: Boolean = true, + val build: (UserRepository, topic: String) -> HuaweiRoute, +) diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiRouteFactory.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiRouteFactory.kt new file mode 100644 index 00000000..9a238d33 --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiRouteFactory.kt @@ -0,0 +1,1197 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.route + +import com.fasterxml.jackson.databind.ObjectMapper +import org.apache.avro.specific.SpecificRecord +import org.radarbase.huawei.converter.FieldValues +import org.radarcns.connector.huawei.HuaweiActiveHours +import org.radarcns.connector.huawei.HuaweiCgmBloodGlucose +import org.radarcns.connector.huawei.HuaweiContinuousActivityStatistics +import org.radarcns.connector.huawei.HuaweiContinuousAltitudeStatistics +import org.radarcns.connector.huawei.HuaweiContinuousBloodGlucoseStatistics +import org.radarcns.connector.huawei.HuaweiContinuousBodyBloodPressureStatistics +import org.radarcns.connector.huawei.HuaweiContinuousBreatheRateStatistics +import org.radarcns.connector.huawei.HuaweiContinuousCaloriesBurnt +import org.radarcns.connector.huawei.HuaweiContinuousCaloriesBurntTotal +import org.radarcns.connector.huawei.HuaweiContinuousDistanceDelta +import org.radarcns.connector.huawei.HuaweiContinuousDistanceTotal +import org.radarcns.connector.huawei.HuaweiContinuousEcgDetail +import org.radarcns.connector.huawei.HuaweiContinuousExerciseIntensity +import org.radarcns.connector.huawei.HuaweiContinuousExerciseIntensityV2 +import org.radarcns.connector.huawei.HuaweiContinuousExerciseIntensityV2Statistics +import org.radarcns.connector.huawei.HuaweiContinuousSleepFragment +import org.radarcns.connector.huawei.HuaweiContinuousSpo2Statistics +import org.radarcns.connector.huawei.HuaweiContinuousStepsDelta +import org.radarcns.connector.huawei.HuaweiContinuousStepsTotal +import org.radarcns.connector.huawei.HuaweiDailyActivitySummary +import org.radarcns.connector.huawei.HuaweiEmotion +import org.radarcns.connector.huawei.HuaweiHealthRecordDynamicBp +import org.radarcns.connector.huawei.HuaweiHealthRecordHeartRateAlert +import org.radarcns.connector.huawei.HuaweiHealthRecordHyperthermia +import org.radarcns.connector.huawei.HuaweiHealthRecordLowSpo2Alert +import org.radarcns.connector.huawei.HuaweiHealthRecordMenstrualCycle +import org.radarcns.connector.huawei.HuaweiHealthRecordSleep +import org.radarcns.connector.huawei.HuaweiHeartRateVariability +import org.radarcns.connector.huawei.HuaweiRestingCaloriesStatistics +import org.radarcns.connector.huawei.HuaweiSleepOnOffBedRecord +import org.radarcns.connector.huawei.HuaweiSleepRespiratoryDetail +import org.radarcns.connector.huawei.HuaweiSleepRespiratoryEvent +import org.radarcns.connector.huawei.HuaweiStatistics +import org.radarcns.connector.huawei.HuaweiTemperature +import org.radarcns.connector.huawei.HuaweiVo2Max +import java.time.Instant + +/** + * Registry of every Huawei Health Kit data type this connector supports, mapping each one to the + * Kafka Connect REST endpoint (`sampleSet:polymerize`, `healthRecords`, or `activityRecords`) and + * Avro record type documented in the `radar-huawei-connector` schema specification + * (RADAR-base/RADAR-Schemas, `huawei_schemas` branch). + * + * Huawei `dataTypeName`/`subDataTypeName` values below are taken verbatim from that + * specification's `doc` strings (prefixed with the vendor namespace `com.huawei.`), which in turn + * describe the Huawei Health Kit REST Data API's own data type identifiers. + * + * Field-value key names used in the record builders are the `fieldName`s documented in Huawei's + * per-data-type references. Their casing is not consistent across data types (e.g. `steps_delta` + * and `sleep_state` but `emotionStatus` and `onOffBedState`), so each key is taken from its own data + * type's reference; keys not documented there are marked best-effort. + * + * @author yatharthranjan + */ +object HuaweiRouteFactory { + + private const val VENDOR_PREFIX = "com.huawei." + private val MAPPER = ObjectMapper() + + private fun Instant.toEpoch(): Double = toEpochMilli() / 1000.0 + + private fun HuaweiStatistics.Builder.populateCommon( + startTime: Instant, + endTime: Instant?, + timeReceived: Instant, + fields: FieldValues, + ) { + time = startTime.toEpoch() + this.timeReceived = timeReceived.toEpoch() + this.endTime = endTime?.toEpoch() + avg = fields.getDouble("avg") + max = fields.getDouble("max") + min = fields.getDouble("min") + last = fields.getDouble("last") + // Stress statistics documents its count as "measure_count". + count = fields.getInt("count", "measure_count") + } + + /** Data types that reuse the generic [HuaweiStatistics] schema: (config key, Huawei data type name, default topic). */ + private val genericStatisticsTypes = listOf( + Triple( + "continuous_body_temperature_rest_statistics", + "continuous.body.temperature.rest.statistics", + "connect_huawei_continuous_body_temperature_rest_statistics", + ), + Triple( + "continuous_calories_bmr_statistics", + "continuous.calories.bmr.statistics", + "connect_huawei_continuous_calories_bmr_statistics", + ), + Triple( + "continuous_exercise_heart_rate_statistics", + "continuous.exercise_heart_rate.statistics", + "connect_huawei_continuous_exercise_heart_rate_statistics", + ), + Triple( + "continuous_power_statistics", + "continuous.power.statistics", + "connect_huawei_continuous_power_statistics", + ), + Triple( + "continuous_speed_statistics", + "continuous.speed.statistics", + "connect_huawei_continuous_speed_statistics", + ), + Triple( + "continuous_steps_rate_statistics", + "continuous.steps.rate.statistics", + "connect_huawei_continuous_steps_rate_statistics", + ), + Triple( + "continuous_stroke_rate_statistics", + "continuous.stroke_rate.statistics", + "connect_huawei_continuous_stroke_rate_statistics", + ), + Triple( + "instantaneous_resting_heart_rate_statistics", + "instantaneous.resting_heart_rate.statistics", + "connect_huawei_instantaneous_resting_heart_rate_statistics", + ), + Triple( + "instantaneous_stress_statistics", + "instantaneous.stress.statistics", + "connect_huawei_instantaneous_stress_statistics", + ), + Triple("vo2max_statistics", "vo2max.statistics", "connect_huawei_vo2max_statistics"), + ) + + /** + * Keys of [genericStatisticsTypes] whose data type Huawei rejected live with + * "Invalid dataTypeName" (none appears in Huawei's data type references either), so disabled + * unless explicitly enabled. + */ + private val undocumentedStatisticsTypes = setOf( + "continuous_body_temperature_rest_statistics", + "continuous_calories_bmr_statistics", + "continuous_exercise_heart_rate_statistics", + "continuous_power_statistics", + "continuous_speed_statistics", + "continuous_steps_rate_statistics", + "continuous_stroke_rate_statistics", + ) + + /** Full registry of Huawei Health Kit data types supported by this connector. */ + val definitions: List = buildList { + add( + HuaweiRouteDefinition( + "activity_record", + "connect_huawei_activity_record", + ) { repo, topic -> + HuaweiActivityRecordRoute(repo, topic) + }, + ) + + // cgm_blood_glucose (+ .statistics variant) + add( + sampleSetDefinition( + "cgm_blood_glucose", + "cgm_blood_glucose", + "connect_huawei_cgm_blood_glucose", + ) { f, start, _, received -> + HuaweiCgmBloodGlucose.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + level = f.getDouble("level") + }.build() + }, + ) + add( + sampleSetDefinition( + "cgm_blood_glucose_statistics", + "cgm_blood_glucose.statistics", + "connect_huawei_cgm_blood_glucose_statistics", + ) { f, start, _, received -> + HuaweiCgmBloodGlucose.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + avg = f.getInt("avg") + max = f.getInt("max") + min = f.getInt("min") + last = f.getInt("last") + }.build() + }, + ) + + add( + // "com.huawei.daily_activity_summary" is itself a documented "Atomic Sampling + // Statistical Data Type" (per the official "Daily Activity Data" reference) queried by + // day via dailyPolymerize - it is not derived from a separate raw type, and it is NOT + // the separate sampleConfigs-based "Workout Goals" endpoint. Its field names are + // camelCase (matching the Avro schema field names directly), unlike most other Huawei + // data types' snake_case field names. + sampleSetDefinition( + "daily_activity_summary", + "daily_activity_summary", + "connect_huawei_daily_activity_summary", + useDailyPolymerize = true, + ) { f, start, end, received -> + HuaweiDailyActivitySummary.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + steps = f.getInt("steps") + activeCalories = f.getInt("activeCalories") + exerciseTime = f.getInt("exerciseTime") + activeHours = f.getInt("activeHours") + stepsGoal = f.getInt("stepsGoal") + activeCaloriesGoal = f.getInt("activeCaloriesGoal") + exerciseTimeGoal = f.getInt("exerciseTimeGoal") + activeHoursGoal = f.getInt("activeHoursGoal") + }.build() + }, + ) + + add( + sampleSetDefinition( + "active_hours", + "active_hours", + "connect_huawei_active_hours", + ) { f, start, end, received -> + f.toRawActiveHours(start, end, received) + }, + ) + add( + sampleSetDefinition( + "active_hours_statistics", + "active_hours.statistics", + "connect_huawei_active_hours_statistics", + ) { f, start, end, received -> + f.toActiveHoursStatistics(start, end, received) + }, + ) + + add( + sampleSetDefinition( + "continuous_activity_fragment", + "continuous.activity.fragment", + "connect_huawei_continuous_activity_fragment", + ) { f, start, end, received -> + f.toContinuousActivityStatistics(start, end, received) + }, + ) + add( + sampleSetDefinition( + "continuous_activity_statistics", + "continuous.activity.statistics", + "connect_huawei_continuous_activity_statistics", + // Rejected live with "Invalid dataTypeName" for com.huawei.continuous.activity. + enabledByDefault = false, + ) { f, start, end, received -> + f.toContinuousActivityStatistics(start, end, received) + }, + ) + + add( + sampleSetDefinition( + "continuous_altitude_statistics", + "continuous.altitude.statistics", + "connect_huawei_continuous_altitude_statistics", + // Statistics variant is documented under "continuous.", but its underlying raw + // detailed data type is "com.huawei.instantaneous.altitude" - a different + // namespace, per the official "Altitude" data type reference. + queryDataTypeSuffix = "instantaneous.altitude", + useDailyPolymerize = true, + ) { f, start, end, received -> + HuaweiContinuousAltitudeStatistics.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + avg = f.getDouble("avg") + max = f.getDouble("max") + min = f.getDouble("min") + ascentTotal = f.getDouble("ascent_total") + descentTotal = f.getDouble("descent_total") + }.build() + }, + ) + + add( + sampleSetDefinition( + "continuous_blood_glucose_statistics", + "continuous.blood_glucose.statistics", + "connect_huawei_continuous_blood_glucose_statistics", + // Raw detailed type is "com.huawei.instantaneous.blood_glucose" per the official + // "Blood Glucose" reference. + queryDataTypeSuffix = "instantaneous.blood_glucose", + ) { f, start, end, received -> + HuaweiContinuousBloodGlucoseStatistics.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + avg = f.getDouble("avg") + max = f.getDouble("max") + min = f.getDouble("min") + correlationWithMealtime = f.getInt("correlate_mealtime") + meal = f.getInt("meal") + correlationWithSleepState = f.getInt("correlate_sleep") + sampleSource = f.getInt("sample_source") + }.build() + }, + ) + + add( + sampleSetDefinition( + "continuous_breathe_rate_statistics", + "continuous.breathe_rate.statistics", + "connect_huawei_continuous_breathe_rate_statistics", + // Statistics variant is documented under "continuous.", but its underlying raw + // detailed data type is "com.huawei.instantaneous.breathe_rate" - the same + // namespace mismatch already confirmed for SpO2/Altitude. + queryDataTypeSuffix = "instantaneous.breathe_rate", + useDailyPolymerize = true, + ) { f, start, end, received -> + HuaweiContinuousBreatheRateStatistics.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + // camelCase per the official "Respiratory Rate" data type reference. + maxBreatheRate = f.getInt("maxBreatheRate") + minBreatheRate = f.getInt("minBreatheRate") + avgBreatheRate = f.getInt("avgBreatheRate") + minBreathrateBaseline = f.getInt("minBreathrateBaseline") + maxBreathrateBaseline = f.getInt("maxBreathrateBaseline") + }.build() + }, + ) + + add( + sampleSetDefinition( + "continuous_body_blood_pressure_statistics", + "continuous.body.blood_pressure.statistics", + "connect_huawei_continuous_body_blood_pressure_statistics", + // Raw detailed type is "com.huawei.instantaneous.blood_pressure" per the official + // "Blood Pressure" reference. + queryDataTypeSuffix = "instantaneous.blood_pressure", + ) { f, start, end, received -> + HuaweiContinuousBodyBloodPressureStatistics.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + systolicPressureAvg = f.getDouble("systolic_pressure_avg") + systolicPressureMax = f.getDouble("systolic_pressure_max") + systolicPressureMin = f.getDouble("systolic_pressure_min") + diastolicPressureAvg = f.getDouble("diastolic_pressure_avg") + diastolicPressureMax = f.getDouble("diastolic_pressure_max") + diastolicPressureMin = f.getDouble("diastolic_pressure_min") + sphygmusAvg = f.getDouble("sphygmus_avg") + sphygmusMax = f.getDouble("sphygmus_max") + sphygmusMin = f.getDouble("sphygmus_min") + sphygmusLast = f.getDouble("sphygmus_last") + }.build() + }, + ) + + genericStatisticsTypes.forEach { (key, dataType, topic) -> + add( + sampleSetDefinition( + key, + dataType, + topic, + enabledByDefault = key !in undocumentedStatisticsTypes, + ) { f, start, end, received -> + HuaweiStatistics.newBuilder().apply { + populateCommon( + start, + end, + received, + f, + ) + }.build() + }, + ) + } + + // Statistics variants whose "continuous."-labelled name doesn't match their underlying + // raw detailed data type's namespace (it's "instantaneous." instead, per the official + // Health Sampling references) - so they need an explicit queryDataTypeSuffix override + // rather than the generic loop above. + listOf( + Triple( + "continuous_heart_rate_statistics", + "continuous.heart_rate.statistics", + "connect_huawei_continuous_heart_rate_statistics", + ) to "instantaneous.heart_rate", + Triple( + "continuous_body_temperature_statistics", + "continuous.body.temperature.statistics", + "connect_huawei_continuous_body_temperature_statistics", + ) to "instantaneous.body.temperature", + Triple( + "continuous_skin_temperature_statistics", + "continuous.skin.temperature.statistics", + "connect_huawei_continuous_skin_temperature_statistics", + ) to "instantaneous.skin.temperature", + ).forEach { (definition, rawDataType) -> + val (key, dataType, topic) = definition + add( + sampleSetDefinition( + key, + dataType, + topic, + queryDataTypeSuffix = rawDataType, + useDailyPolymerize = true, + ) { f, start, end, received -> + HuaweiStatistics.newBuilder().apply { + populateCommon( + start, + end, + received, + f, + ) + }.build() + }, + ) + } + + add( + sampleSetDefinition( + "continuous_body_fat_rate_statistics", + "continuous.body.fat.rate.statistics", + "connect_huawei_continuous_body_fat_rate_statistics", + // Huawei has no separate body fat data type: per the official "Weight" reference, + // body fat percentage is part of com.huawei.instantaneous.body_weight, whose daily + // statistics carry it as avg/max/min_body_fat_rate (avg/max/min/last there are the + // weight itself). + queryDataTypeSuffix = "instantaneous.body_weight", + useDailyPolymerize = true, + ) { f, start, end, received -> + HuaweiStatistics.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + avg = f.getDouble("avg_body_fat_rate") + max = f.getDouble("max_body_fat_rate") + min = f.getDouble("min_body_fat_rate") + }.build() + }, + ) + + // Individual temperature readings (both share the HuaweiTemperature schema), per the + // official "Body Temperature" reference: a single float "temperature" field (°C). + listOf( + "instantaneous_body_temperature" to "instantaneous.body.temperature", + "instantaneous_skin_temperature" to "instantaneous.skin.temperature", + ).forEach { (key, dataType) -> + add( + sampleSetDefinition(key, dataType, "connect_huawei_$key") { f, start, _, received -> + HuaweiTemperature.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + temperature = f.getFloat("temperature") + }.build() + }, + ) + } + + add( + sampleSetDefinition( + "continuous_calories_burnt", + "continuous.calories.burnt", + "connect_huawei_continuous_calories_burnt", + ) { f, start, end, received -> + HuaweiContinuousCaloriesBurnt.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + calories = f.getDouble("calories") + }.build() + }, + ) + add( + sampleSetDefinition( + "continuous_calories_consumed", + "continuous.calories.consumed", + "connect_huawei_continuous_calories_consumed", + // Rejected live with "no default dataCollector found for: + // com.huawei.continuous.calories.consumed" and absent from Huawei's data type + // references, so disabled unless explicitly enabled. + enabledByDefault = false, + ) { f, start, end, received -> + HuaweiContinuousCaloriesBurnt.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + calories = f.getDouble("calories") + }.build() + }, + ) + add( + sampleSetDefinition( + "continuous_calories_burnt_total", + "continuous.calories.burnt.total", + "connect_huawei_continuous_calories_burnt_total", + queryDataTypeSuffix = "continuous.calories.burnt", + useDailyPolymerize = true, + ) { f, start, end, received -> + HuaweiContinuousCaloriesBurntTotal.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + caloriesTotal = f.getDouble("calories_total") + }.build() + }, + ) + + add( + sampleSetDefinition( + "continuous_distance_delta", + "continuous.distance.delta", + "connect_huawei_continuous_distance_delta", + ) { f, start, end, received -> + HuaweiContinuousDistanceDelta.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + distanceDelta = f.getDouble("distance_delta") + }.build() + }, + ) + add( + sampleSetDefinition( + "continuous_distance_total", + "continuous.distance.total", + "connect_huawei_continuous_distance_total", + queryDataTypeSuffix = "continuous.distance.delta", + useDailyPolymerize = true, + ) { f, start, end, received -> + HuaweiContinuousDistanceTotal.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + distance = f.getDouble("distance") + }.build() + }, + ) + + add( + // Keeps its historical key/topic (matching the HuaweiContinuousEcgDetail schema), but + // is backed by the "ECG Measurement Records" health record type + // com.huawei.continuous.ecg_record: ECG measurement details + // (com.huawei.continuous.ecg_detail) are only open through health record queries, as + // the detail data associated with each record. + healthRecordDefinition( + "continuous_ecg_detail", + "continuous.ecg_record", + "connect_huawei_continuous_ecg_detail", + subDataTypes = listOf(VENDOR_PREFIX + "continuous.ecg_detail"), + ) { f, start, end, received -> + HuaweiContinuousEcgDetail.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + ecgRecordId = f.recordId + ecgType = f.getInt("ecg_type") + averageHeartRate = f.getFloat("avg_heart_rate") + // Bit flags (1: sinus rhythm, 2: atrial premature beats, ..., 128: poor + // signals), documented as a long but only using the low 8 bits. + ecgArrhythmiaType = f.getLong("ecg_arrhythmia_type")?.toInt() + // Bit flags of user-selected symptoms (bit 0: no discomfort ... bit 9: other). + userSymptom = f.getLong("user_symptom") + samplingFrequency = f.getInt("sampling_frequency") + ecgAlgorithmVersion = f.getString("ecg_algorithm_version") + // Name of the device vendor that provided the ECG data. + ecgDataSources = f.getString("ecg_data_sources") + ecgDataLength = f.getInt("ecg_data_length") + packageName = f.getString("package_name") + voltageData = f.subData.voltageData() + }.build() + }, + ) + + add( + sampleSetDefinition( + "continuous_exercise_intensity", + "continuous.exercise_intensity", + "connect_huawei_continuous_exercise_intensity", + ) { f, start, end, received -> + f.toContinuousExerciseIntensity(start, end, received) + }, + ) + add( + sampleSetDefinition( + "continuous_exercise_intensity_statistics", + "continuous.exercise_intensity.statistics", + "connect_huawei_continuous_exercise_intensity_statistics", + ) { f, start, end, received -> + f.toContinuousExerciseIntensity(start, end, received) + }, + ) + + add( + sampleSetDefinition( + "continuous_exercise_intensity_v2", + "continuous.exercise_intensity.v2", + "connect_huawei_continuous_exercise_intensity_v2", + ) { f, start, end, received -> + HuaweiContinuousExerciseIntensityV2.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + exerciseType = f.getInt("exercise_type") + }.build() + }, + ) + add( + sampleSetDefinition( + "continuous_exercise_intensity_v2_statistics", + "continuous.exercise_intensity.v2.statistics", + "connect_huawei_continuous_exercise_intensity_v2_statistics", + ) { f, start, end, received -> + HuaweiContinuousExerciseIntensityV2Statistics.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + // Huawei reports this as a single Map field ("intensity"), + // not per-heart-rate-zone durations - see FieldValues.getIntMap for the wire-shape + // caveat. + intensityMap = f.getIntMap("intensity") + }.build() + }, + ) + + add( + sampleSetDefinition( + "continuous_sleep_fragment", + "continuous.sleep.fragment", + "connect_huawei_continuous_sleep_fragment", + ) { f, start, end, received -> + HuaweiContinuousSleepFragment.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + sleepState = f.getInt("sleep_state") + }.build() + }, + ) + + add( + sampleSetDefinition( + "continuous_spo2_statistics", + "continuous.spo2.statistics", + "connect_huawei_continuous_spo2_statistics", + // Statistics variant is documented under "continuous.", but its underlying raw + // detailed data type is "com.huawei.instantaneous.spo2" - a different namespace, + // per the official "SpO2" data type reference. + queryDataTypeSuffix = "instantaneous.spo2", + useDailyPolymerize = true, + ) { f, start, end, received -> + HuaweiContinuousSpo2Statistics.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + saturationAvg = f.getDouble("saturation_avg") + saturationMax = f.getDouble("saturation_max") + saturationMin = f.getDouble("saturation_min") + saturationLast = f.getDouble("saturation_last") + }.build() + }, + ) + + add( + sampleSetDefinition( + "continuous_steps_delta", + "continuous.steps.delta", + "connect_huawei_continuous_steps_delta", + ) { f, start, end, received -> + HuaweiContinuousStepsDelta.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + stepsDelta = f.getInt("steps_delta") + }.build() + }, + ) + add( + sampleSetDefinition( + "continuous_steps_total", + "continuous.steps.total", + "connect_huawei_continuous_steps_total", + queryDataTypeSuffix = "continuous.steps.delta", + useDailyPolymerize = true, + ) { f, start, end, received -> + HuaweiContinuousStepsTotal.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + steps = f.getInt("steps") + duration = f.getInt("duration") + }.build() + }, + ) + + add( + sampleSetDefinition( + "emotion", + "emotion", + "connect_huawei_emotion", + ) { f, start, _, received -> + HuaweiEmotion.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + emotionStatus = f.getInt("emotionStatus") + }.build() + }, + ) + + add( + healthRecordDefinition( + "health_record_dynamic_bp", + "health.record.dynamic_bp", + "connect_huawei_health_record_dynamic_bp", + ) { f, start, end, received -> + f.toHealthRecordDynamicBp(start, end, received) + }, + ) + add( + healthRecordDefinition( + "health_record_bradycardia", + "health.record.bradycardia", + "connect_huawei_health_record_bradycardia", + ) { f, start, end, received -> + f.toHealthRecordHeartRateAlert(start, end, received) + }, + ) + add( + healthRecordDefinition( + "health_record_tachycardia", + "health.record.tachycardia", + "connect_huawei_health_record_tachycardia", + ) { f, start, end, received -> + f.toHealthRecordHeartRateAlert(start, end, received) + }, + ) + add( + healthRecordDefinition( + "health_record_hyperthermia", + "health.record.hyperthermia", + "connect_huawei_health_record_hyperthermia", + ) { f, start, end, received -> + HuaweiHealthRecordHyperthermia.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + highBodyTemperatureAlarm = f.getFloat("highBodyTemperatureAlarm") + }.build() + }, + ) + add( + healthRecordDefinition( + "health_record_low_spo2_alert", + "health.record.lowSpo2Alert", + "connect_huawei_health_record_low_spo2_alert", + ) { f, start, end, received -> + HuaweiHealthRecordLowSpo2Alert.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + threshold = f.getFloat("threshold") + maxSpO2 = f.getFloat("maxSpO2") + minSpO2 = f.getFloat("minSpO2") + }.build() + }, + ) + add( + healthRecordDefinition( + "health_record_menstrual_cycle", + "health.record.menstrual_cycle", + "connect_huawei_health_record_menstrual_cycle", + ) { f, start, end, received -> + HuaweiHealthRecordMenstrualCycle.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + recordday = f.getInt("recordDay") + status = f.getInt("status") + substatus = f.getInt("subStatus") + remarks = f.getString("remarks") + timezone = f.getString("timeZone") + }.build() + }, + ) + add( + healthRecordDefinition( + "health_record_sleep", + "health.record.sleep", + "connect_huawei_health_record_sleep", + ) { f, start, end, received -> + HuaweiHealthRecordSleep.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + fallAsleepTime = f.getLong("fall_asleep_time") + wakeupTime = f.getLong("wakeup_time") + lightSleepTime = f.getInt("light_sleep_time") + deepSleepTime = f.getInt("deep_sleep_time") + dreamTime = f.getInt("dream_time") + awakeTime = f.getInt("awake_time") + allSleepTime = f.getInt("all_sleep_time") + wakeupCount = f.getInt("wakeup_count") + deepSleepPart = f.getInt("deep_sleep_part") + sleepScore = f.getInt("sleep_score") + sleepLatency = f.getInt("sleep_latency") + sleepEfficiency = f.getInt("sleep_efficiency") + goBedTime = f.getLong("go_bed_time") + sleepType = f.getInt("sleep_type") + prepareSleepTime = f.getLong("prepare_sleep_time") + offBedTime = f.getLong("off_bed_time") + }.build() + }, + ) + + add( + sampleSetDefinition( + "heart_rate_variability", + "heart_rate_variability", + "connect_huawei_heart_rate_variability", + ) { f, start, _, received -> + HuaweiHeartRateVariability.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + // int, milliseconds, (0, 200], per the official "Heart Rate Variability" + // reference. + heartRateVariabilityRmssd = f.getInt("heartRateVariabilityRMSSD") + }.build() + }, + ) + + add( + sampleSetDefinition( + "resting_calories_statistics", + "resting_calories.statistics", + "connect_huawei_resting_calories_statistics", + ) { f, start, end, received -> + HuaweiRestingCaloriesStatistics.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + predictedCalories = f.getFloat("predictedCalories") + totalCalories = f.getFloat("totalCalories") + }.build() + }, + ) + + add( + sampleSetDefinition( + "sleep_on_off_bed_record", + "sleep.on_off_bed_record", + "connect_huawei_sleep_on_off_bed_record", + ) { f, start, _, received -> + HuaweiSleepOnOffBedRecord.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + onOffBedState = f.getInt("onOffBedState") + }.build() + }, + ) + + add( + sampleSetDefinition( + "sleep_respiratory_detail", + "sleep_respiratory_detail", + "connect_huawei_sleep_respiratory_detail", + ) { f, start, end, received -> + HuaweiSleepRespiratoryDetail.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + type = f.getInt("type") + value = f.getDouble("value") + }.build() + }, + ) + add( + sampleSetDefinition( + "sleep_respiratory_event", + "sleep_respiratory_event", + "connect_huawei_sleep_respiratory_event", + ) { f, start, end, received -> + HuaweiSleepRespiratoryEvent.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + eventname = f.getInt("eventName") + }.build() + }, + ) + + add( + sampleSetDefinition( + "vo2max", + "vo2max", + "connect_huawei_vo2max", + ) { f, start, _, received -> + HuaweiVo2Max.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + vo2max = f.getInt("vo2max") + }.build() + }, + ) + } + + /** + * The raw `com.huawei.active_hours` data type's only documented field is `isActive` (whether + * that hour had at least moderate-intensity activity) - it has no moderate/high intensity + * minute breakdown, unlike what [toActiveHoursStatistics] reads. + */ + private fun FieldValues.toRawActiveHours( + start: Instant, + end: Instant?, + received: Instant, + ): HuaweiActiveHours = HuaweiActiveHours.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + activeHours = getInt("isActive") + }.build() + + /** The `com.huawei.active_hours.statistics` data type's only documented field is `activeHours` + * (the number of active hours in the statistical period). */ + private fun FieldValues.toActiveHoursStatistics( + start: Instant, + end: Instant?, + received: Instant, + ): HuaweiActiveHours = HuaweiActiveHours.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + activeHours = getInt("activeHours") + }.build() + + private fun FieldValues.toContinuousActivityStatistics( + start: Instant, + end: Instant?, + received: Instant, + ): HuaweiContinuousActivityStatistics = HuaweiContinuousActivityStatistics.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + typeOfActivity = getInt("activity_type") + span = getInt("span") + fragments = getInt("fragments") + }.build() + + private fun FieldValues.toContinuousExerciseIntensity( + start: Instant, + end: Instant?, + received: Instant, + ): HuaweiContinuousExerciseIntensity = HuaweiContinuousExerciseIntensity.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + intensity = getDouble("intensity") + span = getInt("span") + }.build() + + private fun FieldValues.toHealthRecordHeartRateAlert( + start: Instant, + end: Instant?, + received: Instant, + ): HuaweiHealthRecordHeartRateAlert = HuaweiHealthRecordHeartRateAlert.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + threshold = getDouble("threshold") + avgHeartRate = getDouble("avg_heart_rate") + maxHeartRate = getDouble("max_heart_rate") + minHeartRate = getDouble("min_heart_rate") + }.build() + + /** + * The 24h ambulatory blood pressure monitoring record has ~80 numeric fields, all following + * the same `` naming (e.g. `avgSystolicBpAll`, `maxHeartRateWake`), which + * Huawei uses verbatim as its camelCase field keys. + */ + private fun FieldValues.toHealthRecordDynamicBp( + start: Instant, + end: Instant?, + received: Instant, + ): HuaweiHealthRecordDynamicBp { + val f = this + fun i(name: String) = f.getInt(name) + fun d(name: String) = f.getDouble(name) + fun l(name: String) = f.getLong(name) + return HuaweiHealthRecordDynamicBp.newBuilder().apply { + time = start.toEpoch() + timeReceived = received.toEpoch() + endTime = end?.toEpoch() + planId = f.getString("planId") + planStartTime = l("planStartTime") + planEndTime = l("planEndTime") + planActualTime = l("planActualTime") + planStatus = i("planStatus") + gasBagType = i("gasBagType") + sleepStartTime = l("sleepStartTime") + sleepEndTime = l("sleepEndTime") + + validCntAll = i("validCntAll") + cntAll = i("cntAll") + maxSystolicBpAll = i("maxSystolicBpAll") + maxDiastolicBpAll = i("maxDiastolicBpAll") + maxHeartRateAll = i("maxHeartRateAll") + midSystolicBpAll = i("midSystolicBpAll") + midDiastolicBpAll = i("midDiastolicBpAll") + midHeartRateAll = i("midHeartRateAll") + minSystolicBpAll = i("minSystolicBpAll") + minDiastolicBpAll = i("minDiastolicBpAll") + minHeartRateAll = i("minHeartRateAll") + avgSystolicBpAll = i("avgSystolicBpAll") + avgDiastolicBpAll = i("avgDiastolicBpAll") + avgHeartRateAll = i("avgHeartRateAll") + stdSystolicBpAll = i("stdSystolicBpAll") + stdDiastolicBpAll = i("stdDiastolicBpAll") + stdHeartRateAll = i("stdHeartRateAll") + coefSystolicBpAll = d("coefSystolicBpAll") + coefDiastolicBpAll = d("coefDiastolicBpAll") + coefHeartRateAll = d("coefHeartRateAll") + loadSystolicBpAll = d("loadSystolicBpAll") + loadDiastolicBpAll = d("loadDiastolicBpAll") + dropSystolicBpAll = d("dropSystolicBpAll") + dropDiastolicBpAll = d("dropDiastolicBpAll") + peakSystolicBpAll = i("peakSystolicBpAll") + peakDiastolicBpAll = i("peakDiastolicBpAll") + + validCntWake = i("validCntWake") + cntWake = i("cntWake") + maxSystolicBpWake = i("maxSystolicBpWake") + maxDiastolicBpWake = i("maxDiastolicBpWake") + maxHeartRateWake = i("maxHeartRateWake") + midSystolicBpWake = i("midSystolicBpWake") + midDiastolicBpWake = i("midDiastolicBpWake") + midHeartRateWake = i("midHeartRateWake") + minSystolicBpWake = i("minSystolicBpWake") + minDiastolicBpWake = i("minDiastolicBpWake") + minHeartRateWake = i("minHeartRateWake") + avgSystolicBpWake = i("avgSystolicBpWake") + avgDiastolicBpWake = i("avgDiastolicBpWake") + avgHeartRateWake = i("avgHeartRateWake") + stdSystolicBpWake = i("stdSystolicBpWake") + stdDiastolicBpWake = i("stdDiastolicBpWake") + stdHeartRateWake = i("stdHeartRateWake") + coefSystolicBpWake = d("coefSystolicBpWake") + coefDiastolicBpWake = d("coefDiastolicBpWake") + coefHeartRateWake = d("coefHeartRateWake") + loadSystolicBpWake = d("loadSystolicBpWake") + loadDiastolicBpWake = d("loadDiastolicBpWake") + + validCntSleep = i("validCntSleep") + cntSleep = i("cntSleep") + maxSystolicBpSleep = i("maxSystolicBpSleep") + maxDiastolicBpSleep = i("maxDiastolicBpSleep") + maxHeartRateSleep = i("maxHeartRateSleep") + midSystolicBpSleep = i("midSystolicBpSleep") + midDiastolicBpSleep = i("midDiastolicBpSleep") + midHeartRateSleep = i("midHeartRateSleep") + minSystolicBpSleep = i("minSystolicBpSleep") + minDiastolicBpSleep = i("minDiastolicBpSleep") + minHeartRateSleep = i("minHeartRateSleep") + avgSystolicBpSleep = i("avgSystolicBpSleep") + avgDiastolicBpSleep = i("avgDiastolicBpSleep") + avgHeartRateSleep = i("avgHeartRateSleep") + stdSystolicBpSleep = i("stdSystolicBpSleep") + stdDiastolicBpSleep = i("stdDiastolicBpSleep") + stdHeartRateSleep = i("stdHeartRateSleep") + coefSystolicBpSleep = d("coefSystolicBpSleep") + coefDiastolicBpSleep = d("coefDiastolicBpSleep") + coefHeartRateSleep = d("coefHeartRateSleep") + loadSystolicBpSleep = d("loadSystolicBpSleep") + loadDiastolicBpSleep = d("loadDiastolicBpSleep") + + validCntWakeTwo = i("validCntWakeTwo") + cntWakeTwo = i("cntWakeTwo") + maxSystolicBpWakeTwo = i("maxSystolicBpWakeTwo") + maxDiastolicBpWakeTwo = i("maxDiastolicBpWakeTwo") + maxHeartRateWakeTwo = i("maxHeartRateWakeTwo") + midSystolicBpWakeTwo = i("midSystolicBpWakeTwo") + midDiastolicBpWakeTwo = i("midDiastolicBpWakeTwo") + midHeartRateWakeTwo = i("midHeartRateWakeTwo") + minSystolicBpWakeTwo = i("minSystolicBpWakeTwo") + minDiastolicBpWakeTwo = i("minDiastolicBpWakeTwo") + minHeartRateWakeTwo = i("minHeartRateWakeTwo") + avgSystolicBpWakeTwo = i("avgSystolicBpWakeTwo") + avgDiastolicBpWakeTwo = i("avgDiastolicBpWakeTwo") + avgHeartRateWakeTwo = i("avgHeartRateWakeTwo") + stdSystolicBpWakeTwo = i("stdSystolicBpWakeTwo") + stdDiastolicBpWakeTwo = i("stdDiastolicBpWakeTwo") + stdHeartRateWakeTwo = i("stdHeartRateWakeTwo") + coefSystolicBpWakeTwo = d("coefSystolicBpWakeTwo") + coefDiastolicBpWakeTwo = d("coefDiastolicBpWakeTwo") + coefHeartRateWakeTwo = d("coefHeartRateWakeTwo") + + extendData = f.getString("extendData") + }.build() + } + + private fun sampleSetDefinition( + key: String, + dataTypeSuffix: String, + defaultTopic: String, + enabledByDefault: Boolean = true, + // Huawei's polymerize API rejects a groupByTime-aggregated query for at least some data + // types (confirmed live: "com.huawei.resting_calories does not support the query mode, + // please use dailyPolymerize API") - the day-aggregated ("*.statistics") variant of every + // data type must go through the dedicated sampleSet:dailyPolymerize endpoint instead. + // Likewise, "*.total" data types are never valid *request* dataTypeNames (confirmed live: + // "no default dataCollector found for: com.huawei.continuous.steps.total") - Huawei's own + // dailyPolymerize example queries the "*.delta" data type and gets a "*.total"-labelled + // response back, so a "*.total" route must override [queryDataTypeSuffix] to name its + // sibling raw/delta data type instead. + queryDataTypeSuffix: String = dataTypeSuffix.removeSuffix(".statistics"), + useDailyPolymerize: Boolean = dataTypeSuffix.endsWith(".statistics"), + buildRecord: ( + fields: FieldValues, + startTime: Instant, + endTime: Instant?, + timeReceived: Instant, + ) -> SpecificRecord, + ): HuaweiRouteDefinition = HuaweiRouteDefinition( + key, + defaultTopic, + enabledByDefault, + ) { repo, topic -> + val rawDataTypeName = VENDOR_PREFIX + queryDataTypeSuffix + if (useDailyPolymerize) { + HuaweiDailyPolymerizeRoute( + userRepository = repo, + dataTypeName = rawDataTypeName, + topic = topic, + buildRecord = buildRecord, + ) + } else { + HuaweiSampleSetRoute( + userRepository = repo, + dataTypeName = rawDataTypeName, + topic = topic, + buildRecord = buildRecord, + ) + } + } + + /** + * ECG voltage data of all measurement detail points associated with an ECG record: a single + * point's `voltage_datas` string as-is, or a JSON array of each point's value when there are + * several. + */ + private fun List.voltageData(): String? { + val segments = mapNotNull { it.getString("voltage_datas") } + return when (segments.size) { + 0 -> null + 1 -> segments.single() + else -> MAPPER.writeValueAsString(segments) + } + } + + private fun healthRecordDefinition( + key: String, + dataTypeName: String, + defaultTopic: String, + subDataTypes: List = emptyList(), + buildRecord: ( + fields: FieldValues, + startTime: Instant, + endTime: Instant?, + timeReceived: Instant, + ) -> SpecificRecord, + ): HuaweiRouteDefinition = HuaweiRouteDefinition(key, defaultTopic) { repo, topic -> + HuaweiHealthRecordRoute( + userRepository = repo, + dataTypeName = VENDOR_PREFIX + dataTypeName, + topic = topic, + subDataTypes = subDataTypes, + buildRecord = buildRecord, + ) + } +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiSampleSetRoute.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiSampleSetRoute.kt new file mode 100644 index 00000000..7ca1db1c --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/route/HuaweiSampleSetRoute.kt @@ -0,0 +1,91 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.route + +import com.fasterxml.jackson.databind.ObjectMapper +import org.apache.avro.specific.SpecificRecord +import org.radarbase.huawei.converter.FieldValues +import org.radarbase.huawei.converter.HuaweiDataConverter +import org.radarbase.huawei.converter.HuaweiSampleSetConverter +import org.radarbase.huawei.request.RestRequest +import org.radarbase.huawei.user.User +import org.radarbase.huawei.user.UserRepository +import java.time.Duration +import java.time.Instant + +/** + * Route backed by `POST /healthkit/v1/sampleSet:polymerize`, which covers the large majority of + * raw (non-`.statistics`) Huawei Health Kit data types (all `continuous.*`, `instantaneous.*`, + * `cgm_blood_glucose`, `active_hours`, `emotion`, + * `heart_rate_variability`, `vo2max`, `sleep.on_off_bed_record`, and `sleep_respiratory_*` types). + * Returns raw, un-aggregated sample points for [dataTypeName] over the requested time range. + * + * The day-aggregated `.statistics` variant of a data type is not queried through this route - + * see [HuaweiDailyPolymerizeRoute]. + * + * @author yatharthranjan + */ +open class HuaweiSampleSetRoute( + userRepository: UserRepository, + private val dataTypeName: String, + private val topic: String, + maxIntervalPerRequest: Duration = Duration.ofDays(30L), + buildRecord: ( + fields: FieldValues, + startTime: Instant, + endTime: Instant?, + timeReceived: Instant, + ) -> SpecificRecord, +) : HuaweiRoute(userRepository, maxIntervalPerRequest) { + + override val converters: List = + listOf(HuaweiSampleSetConverter(topic, buildRecord)) + + override fun toString(): String = "huawei_" + topic.removePrefix("connect_huawei_") + + override fun generateRequests( + user: User, + start: Instant, + end: Instant, + max: Int, + ): Sequence = chunkedRanges(start, end, max).map { (rangeStart, rangeEnd) -> + RestRequest( + request = createPostRequest( + user, + "sampleSet:polymerize", + buildRequestBody(rangeStart, rangeEnd), + ), + user = user, + route = this, + startDate = rangeStart, + endDate = rangeEnd, + ) + } + + private fun buildRequestBody(start: Instant, end: Instant): String { + val root = MAPPER.createObjectNode() + root.putArray("polymerizeWith").addObject().put("dataTypeName", dataTypeName) + root.put("startTime", start.toEpochMilli()) + root.put("endTime", end.toEpochMilli()) + return MAPPER.writeValueAsString(root) + } + + companion object { + private val MAPPER = ObjectMapper() + } +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/route/Route.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/route/Route.kt new file mode 100644 index 00000000..ba2ae506 --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/route/Route.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.route + +import org.radarbase.huawei.request.RestRequest +import org.radarbase.huawei.user.User +import java.time.Duration +import java.time.Instant + +/** + * @author yatharthranjan + */ +interface Route { + + fun generateRequests(user: User, start: Instant, end: Instant): Sequence + + fun generateRequests(user: User, start: Instant, end: Instant, max: Int): Sequence + + /** + * This is how it would appear in the offsets + */ + override fun toString(): String + + /** + * The duration of data to request in a single request of this route. + */ + val maxIntervalPerRequest: Duration +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/user/HuaweiUser.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/user/HuaweiUser.kt new file mode 100644 index 00000000..692120c8 --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/user/HuaweiUser.kt @@ -0,0 +1,48 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.user + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarcns.kafka.ObservationKey +import java.time.Instant + +/** + * @author yatharthranjan + */ +@JsonIgnoreProperties(ignoreUnknown = true) +data class HuaweiUser( + @JsonProperty("id") override val id: String, + @JsonProperty("createdAt") override val createdAt: Instant, + @JsonProperty("projectId") override val projectId: String, + @JsonProperty("userId") override val userId: String, + @JsonProperty("humanReadableUserId") override val humanReadableUserId: String?, + @JsonProperty("sourceId") override val sourceId: String, + @JsonProperty("externalId") override val externalId: String?, + @JsonProperty("isAuthorized") override val isAuthorized: Boolean, + @JsonProperty("startDate") override val startDate: Instant, + @JsonProperty("endDate") override val endDate: Instant? = null, + @JsonProperty("version") override val version: String? = null, + @JsonProperty("serviceUserId") override val serviceUserId: String? = null, +) : User { + override val observationKey: ObservationKey = ObservationKey(projectId, userId, sourceId) + override val versionedId: String = "$id${version?.let { "#$it" } ?: ""}" + + fun isComplete() = + isAuthorized && (endDate == null || startDate.isBefore(endDate)) && serviceUserId != null +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/user/User.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/user/User.kt new file mode 100644 index 00000000..e2092cda --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/user/User.kt @@ -0,0 +1,42 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.user + +import org.radarcns.kafka.ObservationKey +import java.time.Instant + +/** + * @author yatharthranjan + */ +interface User { + val id: String + val projectId: String + val userId: String + val sourceId: String + val externalId: String? + val startDate: Instant + val endDate: Instant? + val createdAt: Instant + val humanReadableUserId: String? + val serviceUserId: String? + val version: String? + val isAuthorized: Boolean + + val observationKey: ObservationKey + val versionedId: String +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/user/UserNotAuthorizedException.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/user/UserNotAuthorizedException.kt new file mode 100644 index 00000000..ace22016 --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/user/UserNotAuthorizedException.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.user + +/** + * @author yatharthranjan + */ +class UserNotAuthorizedException(message: String) : Exception(message) { + constructor(user: User) : this("User ${user.id} is not authorized") +} diff --git a/huawei-library/src/main/kotlin/org/radarbase/huawei/user/UserRepository.kt b/huawei-library/src/main/kotlin/org/radarbase/huawei/user/UserRepository.kt new file mode 100644 index 00000000..835b44cc --- /dev/null +++ b/huawei-library/src/main/kotlin/org/radarbase/huawei/user/UserRepository.kt @@ -0,0 +1,59 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.user + +import java.io.IOException + +/** User repository for Huawei Health Kit users. */ +/** + * @author yatharthranjan + */ +interface UserRepository { + /** + * Get specified user. + * + * @throws IOException if the user cannot be retrieved from the repository. + */ + @Throws(IOException::class) + operator fun get(key: String): User? + + /** + * Get all relevant users. + * + * @throws IOException if the list cannot be retrieved from the repository. + */ + @Throws(IOException::class) + fun stream(): Sequence + + /** + * Get the current access token of given user. + * + * @throws IOException if the new access token cannot be retrieved from the repository. + * @throws UserNotAuthorizedException if the refresh token is no longer valid. Manual action + * should be taken to get a new refresh token. + * @throws NoSuchElementException if the user does not exist in this repository. + */ + @Throws(IOException::class, UserNotAuthorizedException::class) + fun getAccessToken(user: User): String + + /** + * Discard any cached access token of given user, e.g. after the Huawei API rejected it with + * HTTP 401 before its advertised expiry, so the next [getAccessToken] call fetches a new one. + */ + fun invalidateAccessToken(user: User) {} +} diff --git a/huawei-library/src/test/kotlin/org/radarbase/huawei/converter/FieldValuesTest.kt b/huawei-library/src/test/kotlin/org/radarbase/huawei/converter/FieldValuesTest.kt new file mode 100644 index 00000000..c52d3e19 --- /dev/null +++ b/huawei-library/src/test/kotlin/org/radarbase/huawei/converter/FieldValuesTest.kt @@ -0,0 +1,122 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.converter + +import com.fasterxml.jackson.databind.ObjectMapper +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull + +/** + * @author yatharthranjan + */ +class FieldValuesTest { + private val mapper = ObjectMapper() + + @Test + fun `parses typed-value array shape`() { + val node = mapper.readTree( + """ + [ + {"fieldName": "steps", "integerValue": 123}, + {"fieldName": "distance", "floatValue": 4.5}, + {"fieldName": "note", "stringValue": "hello"} + ] + """.trimIndent(), + ) + val fields = FieldValues.from(node) + + assertEquals(123, fields.getInt("steps")) + assertEquals(4.5, fields.getDouble("distance")) + assertEquals("hello", fields.getString("note")) + assertNull(fields.getInt("missing")) + } + + @Test + fun `parses flattened object shape`() { + val node = mapper.readTree("""{"avg": 1.5, "max": 3, "min": null}""") + val fields = FieldValues.from(node) + + assertEquals(1.5, fields.getDouble("avg")) + assertEquals(3, fields.getInt("max")) + assertNull(fields.getInt("min")) + } + + @Test + fun `falls back to later candidate keys and serializes list values`() { + val node = mapper.readTree( + """ + [ + {"fieldName": "heartRateVariabilityRmssd", "integerValue": 42}, + {"fieldName": "voltage_datas", "value": [1.5, -2.0]}, + {"fieldName": "custom", "doubleValue": 7.5} + ] + """.trimIndent(), + ) + val fields = FieldValues.from(node) + + assertEquals(42, fields.getInt("heartRateVariabilityRMSSD", "heartRateVariabilityRmssd")) + assertEquals("[1.5,-2.0]", fields.getString("voltage_datas")) + assertNull(fields.getInt("voltage_datas")) + assertEquals(7.5, fields.getDouble("custom")) + } + + @Test + fun `non-numeric text is null rather than zero for numeric getters`() { + val node = mapper.readTree( + """ + [ + {"fieldName": "vendor", "stringValue": "HUAWEI"}, + {"fieldName": "count", "stringValue": " 42 "} + ] + """.trimIndent(), + ) + val fields = FieldValues.from(node) + + assertNull(fields.getInt("vendor")) + assertNull(fields.getDouble("vendor")) + assertEquals(42, fields.getInt("count")) + assertEquals(42L, fields.getLong("count")) + } + + @Test + fun `parses map-typed values`() { + val node = mapper.readTree( + """ + [ + {"fieldName": "plain", "mapValue": {"1": 10, "2": 20}}, + {"fieldName": "typed", "mapValue": {"1": {"integerValue": 5}}}, + {"fieldName": "entries", "mapValue": [{"key": "3", "value": {"integerValue": 7}}]} + ] + """.trimIndent(), + ) + val fields = FieldValues.from(node) + + assertEquals(mapOf("1" to 10, "2" to 20), fields.getIntMap("plain")) + assertEquals(mapOf("1" to 5), fields.getIntMap("typed")) + assertEquals(mapOf("3" to 7), fields.getIntMap("entries")) + } + + @Test + fun `handles missing or null root node`() { + val fields = FieldValues.from(null) + + assertNull(fields.getInt("anything")) + assertNull(fields.getString("anything")) + } +} diff --git a/huawei-library/src/test/kotlin/org/radarbase/huawei/converter/HuaweiSampleSetConverterTest.kt b/huawei-library/src/test/kotlin/org/radarbase/huawei/converter/HuaweiSampleSetConverterTest.kt new file mode 100644 index 00000000..1ee7855e --- /dev/null +++ b/huawei-library/src/test/kotlin/org/radarbase/huawei/converter/HuaweiSampleSetConverterTest.kt @@ -0,0 +1,97 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.converter + +import com.fasterxml.jackson.databind.ObjectMapper +import org.radarbase.huawei.user.HuaweiUser +import org.radarcns.connector.huawei.HuaweiContinuousStepsDelta +import java.time.Instant +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * @author yatharthranjan + */ +class HuaweiSampleSetConverterTest { + private val mapper = ObjectMapper() + private val user = HuaweiUser( + id = "u1", + createdAt = Instant.now(), + projectId = "p", + userId = "u", + humanReadableUserId = null, + sourceId = "s", + externalId = "ext", + isAuthorized = true, + startDate = Instant.parse("2024-01-01T00:00:00Z"), + ) + private val converter = HuaweiSampleSetConverter("topic") { f, start, end, received -> + HuaweiContinuousStepsDelta.newBuilder().apply { + time = start.toEpochMilli() / 1000.0 + timeReceived = received.toEpochMilli() / 1000.0 + endTime = end?.let { it.toEpochMilli() / 1000.0 } + stepsDelta = f.getInt("steps_delta") + }.build() + } + + @Test + fun `reads group-wrapped sample sets with nanosecond times`() { + val root = mapper.readTree( + """ + {"group": [{ + "startTime": 1704067200000, "endTime": 1704153600000, + "sampleSet": [{"samplePoints": [ + {"startTime": 1704067200000000000, "endTime": "1704067260000000000", + "value": [{"fieldName": "steps_delta", "integerValue": 12}]} + ]}] + }]} + """.trimIndent(), + ) + val record = converter.processRecords(root, user).single().getOrThrow() + val value = record.value as HuaweiContinuousStepsDelta + + assertEquals(1704067200L, record.offset) + assertEquals(1704067200.0, value.time) + assertEquals(1704067260.0, value.endTime) + assertEquals(12, value.stepsDelta) + } + + @Test + fun `reads top-level sample sets with millisecond times`() { + val root = mapper.readTree( + """ + {"sampleSet": [{"samplePoints": [ + {"startTime": 1704067200000, "endTime": 1704067260000, + "value": [{"fieldName": "steps_delta", "integerValue": 3}]} + ]}]} + """.trimIndent(), + ) + val record = converter.processRecords(root, user).single().getOrThrow() + + assertEquals(1704067200L, record.offset) + } + + @Test + fun `infers timestamp units from magnitude`() { + val expected = Instant.parse("2024-01-01T00:00:00.123Z") + assertEquals(expected, epochInstantOf(1704067200123L)) + assertEquals(expected, epochInstantOf(1704067200123000L)) + assertEquals(expected, epochInstantOf(1704067200123000000L)) + assertEquals(Instant.parse("2024-01-01T00:00:00Z"), epochInstantOf(1704067200L)) + } +} diff --git a/huawei-library/src/test/kotlin/org/radarbase/huawei/request/HuaweiRequestGeneratorTest.kt b/huawei-library/src/test/kotlin/org/radarbase/huawei/request/HuaweiRequestGeneratorTest.kt new file mode 100644 index 00000000..1723f39f --- /dev/null +++ b/huawei-library/src/test/kotlin/org/radarbase/huawei/request/HuaweiRequestGeneratorTest.kt @@ -0,0 +1,238 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.huawei.request + +import okhttp3.Protocol +import okhttp3.Request +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import org.radarbase.huawei.offset.Offset +import org.radarbase.huawei.route.HuaweiDailyPolymerizeRoute +import org.radarbase.huawei.route.HuaweiRoute +import org.radarbase.huawei.route.HuaweiSampleSetRoute +import org.radarbase.huawei.route.Route +import org.radarbase.huawei.user.HuaweiUser +import org.radarbase.huawei.user.User +import org.radarbase.huawei.user.UserNotAuthorizedException +import org.radarbase.huawei.user.UserRepository +import org.radarcns.connector.huawei.HuaweiContinuousStepsDelta +import java.time.Duration +import java.time.Instant +import java.time.temporal.ChronoUnit +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * @author yatharthranjan + */ +class HuaweiRequestGeneratorTest { + private val user: User = HuaweiUser( + id = "u1", + createdAt = Instant.now(), + projectId = "p", + userId = "u", + humanReadableUserId = null, + sourceId = "s", + externalId = "ext", + isAuthorized = true, + startDate = Instant.parse("2024-01-01T00:00:00Z"), + ) + + private var tokenError: Exception? = null + private var invalidated = 0 + private val repository = object : UserRepository { + override fun get(key: String): User = user + override fun stream(): Sequence = sequenceOf(user) + override fun getAccessToken(user: User): String = tokenError?.let { throw it } ?: "token" + override fun invalidateAccessToken(user: User) { + invalidated++ + } + } + + private val offsets = mutableMapOf() + private val offsetManager = object : HuaweiOffsetManager { + override fun getOffset(route: Route, user: User): Offset? = + offsets[route.toString()]?.let { Offset(user, route, it) } + + override fun updateOffsets(route: Route, user: User, offset: Instant) { + offsets[route.toString()] = offset + } + } + + private val route = HuaweiSampleSetRoute( + repository, + "com.huawei.continuous.steps.delta", + "topic", + ) { f, start, _, received -> + HuaweiContinuousStepsDelta.newBuilder().apply { + time = start.epochSecond.toDouble() + timeReceived = received.epochSecond.toDouble() + stepsDelta = f.getInt("steps_delta") + }.build() + } + + private val generator = HuaweiRequestGenerator(repository, offsetManager, listOf(route)) + + private fun request(route: HuaweiRoute, start: Instant, end: Instant) = RestRequest( + Request.Builder().url("https://example.com").build(), + user, + route, + start, + end, + ) + + private fun response(req: RestRequest, code: Int, body: String) = Response.Builder() + .request(req.request) + .protocol(Protocol.HTTP_1_1) + .code(code) + .message("msg") + .body(body.toResponseBody()) + .build() + + private fun pointsBody(vararg starts: Instant, hasMoreData: Boolean = false): String { + val points = starts.joinToString(",") { start -> + """{"startTime": ${start.toEpochMilli() * 1_000_000}, + "value": [{"fieldName": "steps_delta", "integerValue": 1}]}""" + } + return """{"hasMoreData": $hasMoreData, + "group": [{"sampleSet": [{"samplePoints": [$points]}]}]}""" + } + + @Test + fun `empty historic window advances to its end`() { + val start = Instant.parse("2024-01-01T00:00:00Z") + val end = Instant.parse("2024-01-31T00:00:00Z") + val req = request(route, start, end) + + val records = generator.requestSuccessful(req, response(req, 200, pointsBody())) + + assertTrue(records.isEmpty()) + assertEquals(end, offsets[route.toString()]) + } + + @Test + fun `empty recent window does not advance past the late sync window`() { + val now = Instant.now() + val start = now.minus(Duration.ofDays(20)) + offsets[route.toString()] = start + val req = request(route, start, now) + + generator.requestSuccessful(req, response(req, 200, pointsBody())) + + val offset = offsets.getValue(route.toString()) + val limit = Instant.now().minus(Duration.ofDays(7)) + assertTrue(offset <= limit, "offset $offset advanced too far") + assertTrue(offset > now.minus(Duration.ofDays(8))) + } + + @Test + fun `recent records advance to just after the latest record and drop already-seen ones`() { + val now = Instant.now().truncatedTo(ChronoUnit.SECONDS) + val offset = now.minus(Duration.ofHours(2)) + offsets[route.toString()] = offset + val req = request(route, offset, now) + val seen = offset.minusSeconds(30) + val latest = now.minus(Duration.ofHours(1)) + + val records = generator.requestSuccessful( + req, + response(req, 200, pointsBody(seen, offset, latest)), + ) + + assertEquals(listOf(offset.epochSecond, latest.epochSecond), records.map { it.offset }) + assertEquals(latest.plusSeconds(1), offsets[route.toString()]) + } + + @Test + fun `partial historic response continues after the latest record`() { + val start = Instant.parse("2024-01-01T00:00:00Z") + val latest = Instant.parse("2024-01-05T00:00:00Z") + val req = request(route, start, Instant.parse("2024-01-31T00:00:00Z")) + + generator.requestSuccessful(req, response(req, 200, pointsBody(latest, hasMoreData = true))) + + assertEquals(latest.plusSeconds(1), offsets[route.toString()]) + } + + @Test + fun `failed chunk stops the remaining chunks of that route`() { + val requests = generator.requests(route, user, 100).iterator() + val first = requests.next() + generator.handleResponse(first, response(first, 500, "boom")) + + assertTrue(!requests.hasNext()) + assertEquals(null, offsets[route.toString()]) + } + + @Test + fun `unauthorized user is backed off instead of throwing`() { + tokenError = UserNotAuthorizedException("revoked") + + assertTrue(generator.requests(route, user, 100).toList().isEmpty()) + + tokenError = null + assertTrue(generator.requests(route, user, 100).toList().isEmpty(), "not backed off") + } + + @Test + fun `no data collector error is treated as an empty response`() { + val start = Instant.parse("2024-01-01T00:00:00Z") + val end = Instant.parse("2024-01-31T00:00:00Z") + val req = request(route, start, end) + val body = """{"error":{"code":400,"message":"no default dataCollector found for: x."}}""" + + val result = generator.handleResponse(req, response(req, 400, body)) + + assertTrue(result is HuaweiResult.Success) + assertEquals(end, offsets[route.toString()]) + } + + @Test + fun `401 invalidates the cached access token`() { + val req = request(route, user.startDate, user.startDate.plus(Duration.ofDays(1))) + + generator.handleResponse(req, response(req, 401, "{}")) + + assertEquals(1, invalidated) + } + + @Test + fun `daily requests cover whole completed UTC days only`() { + val daily = HuaweiDailyPolymerizeRoute( + repository, + "com.huawei.continuous.steps.delta", + "daily", + ) { _, start, _, received -> + HuaweiContinuousStepsDelta.newBuilder().apply { + time = start.epochSecond.toDouble() + timeReceived = received.epochSecond.toDouble() + }.build() + } + val start = Instant.parse("2024-01-01T10:00:00Z") + val end = Instant.parse("2024-01-04T10:00:00Z") + + val requests = daily.generateRequests(user, start, end, 10).toList() + + assertEquals(1, requests.size) + assertEquals(Instant.parse("2024-01-02T00:00:00Z"), requests[0].startDate) + assertEquals(Instant.parse("2024-01-04T00:00:00Z"), requests[0].endDate) + val body = okio.Buffer().also { requests[0].request.body!!.writeTo(it) }.readUtf8() + assertTrue(body.contains("\"startDay\":\"20240102\""), body) + assertTrue(body.contains("\"endDay\":\"20240103\""), body) + } +} diff --git a/huawei-library/src/test/kotlin/org/radarbase/huawei/route/HuaweiRouteFactoryTest.kt b/huawei-library/src/test/kotlin/org/radarbase/huawei/route/HuaweiRouteFactoryTest.kt new file mode 100644 index 00000000..cd8e09ae --- /dev/null +++ b/huawei-library/src/test/kotlin/org/radarbase/huawei/route/HuaweiRouteFactoryTest.kt @@ -0,0 +1,266 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.huawei.route + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.node.ObjectNode +import org.apache.avro.Schema +import org.radarbase.huawei.user.HuaweiUser +import org.radarbase.huawei.user.User +import org.radarbase.huawei.user.UserRepository +import org.radarcns.connector.huawei.HuaweiContinuousEcgDetail +import org.radarcns.connector.huawei.HuaweiHealthRecordDynamicBp +import java.time.Instant +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * Exercises every [HuaweiRouteFactory] definition end to end: builds the route, feeds a + * generously-populated fixture payload shaped like the endpoint it targets (`sampleSet:polymerize`, + * `healthRecords`, or `activityRecords`), and asserts the converter produces exactly one record on + * the definition's own topic without throwing. This is the main regression test against typos in + * the ~90 hand-written Huawei field-value key strings (and the Avro builder calls around them). + * + * @author yatharthranjan + */ +class HuaweiRouteFactoryTest { + private val mapper = ObjectMapper() + private val fakeUser: User = HuaweiUser( + id = "u1", + createdAt = Instant.now(), + projectId = "p", + userId = "u", + humanReadableUserId = null, + sourceId = "s", + externalId = "ext", + isAuthorized = true, + startDate = Instant.parse("2024-01-01T00:00:00Z"), + ) + private val fakeUserRepository = object : UserRepository { + override fun get(key: String): User = fakeUser + override fun stream(): Sequence = sequenceOf(fakeUser) + override fun getAccessToken(user: User): String = "token" + } + + @Test + fun `definitions have unique keys and topics`() { + val keys = HuaweiRouteFactory.definitions.map { it.key } + val topics = HuaweiRouteFactory.definitions.map { it.defaultTopic } + + assertEquals(keys.size, keys.toSet().size, "duplicate route definition keys: $keys") + assertEquals(topics.size, topics.toSet().size, "duplicate route definition topics: $topics") + } + + @Test + fun `every definition converts a fixture payload without error`() { + val failures = mutableListOf() + + HuaweiRouteFactory.definitions.forEach { definition -> + val route = definition.build(fakeUserRepository, definition.defaultTopic) + try { + val payload = fixtureFor(route) + val records = route.converters.single().processRecords(payload, fakeUser).toList() + val successes = records.mapNotNull { it.getOrNull() } + + if (successes.size != 1) { + failures += "${definition.key}: expected 1 record, got ${successes.size} " + + "(errors: ${records.mapNotNull { it.exceptionOrNull() }})" + } else if (successes.first().topic != definition.defaultTopic) { + failures += "${definition.key}: unexpected topic ${successes.first().topic}" + } + } catch (e: Exception) { + failures += "${definition.key}: threw $e" + } + } + + assertTrue(failures.isEmpty(), "Failures:\n" + failures.joinToString("\n")) + } + + @Test + fun `ECG route reads record id, record fields and associated voltage data`() { + val definition = HuaweiRouteFactory.definitions.single { it.key == "continuous_ecg_detail" } + val route = definition.build(fakeUserRepository, definition.defaultTopic) + val payload = mapper.readTree( + """ + {"healthRecords": [{ + "startTime": $START_NANOS, "endTime": $END_NANOS, + "dataTypeName": "com.huawei.continuous.ecg_record", + "id": "ecg-1", + "value": [ + {"fieldName": "ecg_type", "integerValue": 1}, + {"fieldName": "avg_heart_rate", "floatValue": 75.6}, + {"fieldName": "ecg_arrhythmia_type", "longValue": 8}, + {"fieldName": "user_symptom", "longValue": 1022}, + {"fieldName": "sampling_frequency", "integerValue": 500}, + {"fieldName": "ecg_algorithm_version", "stringValue": "1.0"}, + {"fieldName": "ecg_data_sources", "stringValue": "HUAWEI"}, + {"fieldName": "ecg_data_length", "integerValue": 7500}, + {"fieldName": "package_name", "stringValue": "com.huawei.health"} + ], + "subDataDetails": [{ + "dataTypeName": "com.huawei.continuous.ecg_detail", + "samplePoints": [ + {"startTime": $START_NANOS, "value": [ + {"fieldName": "voltage_datas", "stringValue": "0.1,0.2"}]} + ] + }] + }]} + """.trimIndent(), + ) + + val record = route.converters.single().processRecords(payload, fakeUser) + .single().getOrThrow().value as HuaweiContinuousEcgDetail + + assertEquals("ecg-1", record.ecgRecordId) + assertEquals(1, record.ecgType) + assertEquals(75.6f, record.averageHeartRate) + assertEquals(8, record.ecgArrhythmiaType) + assertEquals(1022L, record.userSymptom) + assertEquals(500, record.samplingFrequency) + assertEquals("1.0", record.ecgAlgorithmVersion) + assertEquals("HUAWEI", record.ecgDataSources) + assertEquals(7500, record.ecgDataLength) + assertEquals("com.huawei.health", record.packageName) + assertEquals("0.1,0.2", record.voltageData) + } + + private fun fixtureFor(route: HuaweiRoute) = when (route) { + is HuaweiActivityRecordRoute -> activityRecordFixture() + is HuaweiHealthRecordRoute -> healthRecordFixture() + is HuaweiDailyPolymerizeRoute -> dailyPolymerizeFixture() + is HuaweiSampleSetRoute -> sampleSetFixture() + else -> error("Unknown route type: ${route::class}") + } + + private fun sampleSetFixture(): ObjectNode { + val root = mapper.createObjectNode() + val sampleSet = root.putArray("sampleSet") + val group = sampleSet.addObject() + val samplePoints = group.putArray("samplePoints") + val point = samplePoints.addObject() + point.put("startTime", START_MILLIS) + point.put("endTime", END_MILLIS) + point.set("value", genericValueArray()) + return root + } + + private fun dailyPolymerizeFixture(): ObjectNode { + val root = mapper.createObjectNode() + val groups = root.putArray("group") + val group = groups.addObject() + group.put("startTime", START_MILLIS) + group.put("endTime", END_MILLIS) + val sampleSet = group.putArray("sampleSet") + val collector = sampleSet.addObject() + val samplePoints = collector.putArray("samplePoints") + val point = samplePoints.addObject() + point.put("startTime", START_NANOS) + point.put("endTime", END_NANOS) + point.set("value", genericValueArray()) + return root + } + + private fun healthRecordFixture(): ObjectNode { + val root = mapper.createObjectNode() + val records = root.putArray("healthRecords") + val record = records.addObject() + record.put("startTime", START_NANOS) + record.put("endTime", END_NANOS) + record.set("value", genericValueArray()) + return root + } + + private fun activityRecordFixture(): ObjectNode { + val root = mapper.createObjectNode() + val records = root.putArray("activityRecord") + val record = records.addObject() + record.put("startTime", START_MILLIS) + record.put("endTime", END_MILLIS) + record.put("id", "activity-1") + record.put("name", "Run") + record.put("desc", "Morning run") + record.put("timeZone", "Europe/London") + record.put("activityType", "1") + record.put("activeTime", 1000L) + record.put("isKeepGoing", false) + val device = record.putObject("device") + device.put("manufacturer", "Huawei") + device.put("type", 1) + val summary = record.putObject("activitySummary") + summary.put("avgPace", 300.0) + summary.put("bestPace", 250.0) + summary.putObject("paceMap") + summary.putArray("dataSummary") + summary.putArray("sectionSummary") + return root + } + + /** One value entry per literal field-value key used across [HuaweiRouteFactory], plus every + * (snake-cased) field of [HuaweiHealthRecordDynamicBp] - covering both the ad hoc key names + * used for most data types and the mechanically-derived ones used for the 24h ABPM record. */ + private fun genericValueArray(): ArrayNode { + val array = mapper.createArrayNode() + (LITERAL_FIELD_KEYS + dynamicBpFieldKeys()).distinct().forEach { key -> + val entry = array.addObject() + entry.put("fieldName", key) + entry.put("integerValue", 1) + entry.put("floatValue", 1.5) + entry.put("stringValue", "test") + } + return array + } + + private fun dynamicBpFieldKeys(): List = + (HuaweiHealthRecordDynamicBp::class.java.getField("SCHEMA$").get(null) as Schema).fields + .map { it.name() } + .filterNot { it in setOf("time", "timeReceived", "endTime") } + + companion object { + private const val START_MILLIS = 1704067200000L // 2024-01-01T00:00:00Z + private const val END_MILLIS = 1704070800000L // 2024-01-01T01:00:00Z + private const val START_NANOS = START_MILLIS * 1_000_000L + private const val END_NANOS = END_MILLIS * 1_000_000L + + private val LITERAL_FIELD_KEYS = listOf( + "activeCalories", "activeCaloriesGoal", "activeHours", "activeHoursGoal", + "activity_type", "all_sleep_time", "ascent_total", "avg", "avg_body_fat_rate", + "avg_heart_rate", "avgBreatheRate", "awake_time", "calories", "calories_total", + "correlate_mealtime", "correlate_sleep", "count", "deep_sleep_part", "deep_sleep_time", + "descent_total", "diastolic_pressure_avg", "diastolic_pressure_max", + "diastolic_pressure_min", "distance", "distance_delta", "dream_time", "duration", + "ecg_algorithm_version", "ecg_arrhythmia_type", "ecg_data_length", "ecg_data_sources", + "ecg_type", "emotionStatus", "eventName", "exercise_type", "exerciseTime", + "exerciseTimeGoal", "extendData", "fall_asleep_time", "fragments", "go_bed_time", + "heartRateVariabilityRMSSD", "highBodyTemperatureAlarm", "intensity", "isActive", + "last", "level", "light_sleep_time", "max", "max_body_fat_rate", "max_heart_rate", + "maxBreatheRate", "maxBreathrateBaseline", "maxSpO2", "meal", "measure_count", "min", + "min_body_fat_rate", "min_heart_rate", "minBreatheRate", "minBreathrateBaseline", + "minSpO2", "off_bed_time", "onOffBedState", "package_name", "predictedCalories", + "prepare_sleep_time", "recordDay", "remarks", "sample_source", "sampling_frequency", + "saturation_avg", "saturation_last", "saturation_max", "saturation_min", + "sleep_efficiency", "sleep_latency", "sleep_score", "sleep_state", "sleep_type", "span", + "sphygmus_avg", "sphygmus_last", "sphygmus_max", "sphygmus_min", "status", "steps", + "steps_delta", "stepsGoal", "subStatus", "systolic_pressure_avg", + "systolic_pressure_max", "systolic_pressure_min", "temperature", "threshold", + "timeZone", "totalCalories", "type", "user_symptom", "value", "vo2max", "voltage_datas", + "wakeup_count", "wakeup_time", + ) + } +} diff --git a/kafka-connect-huawei-source/Dockerfile b/kafka-connect-huawei-source/Dockerfile new file mode 100644 index 00000000..5da90434 --- /dev/null +++ b/kafka-connect-huawei-source/Dockerfile @@ -0,0 +1,62 @@ +# Copyright 2026 Onsentia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM --platform=$BUILDPLATFORM gradle:8.14-jdk17 AS builder + +RUN mkdir /code +WORKDIR /code + +ENV GRADLE_USER_HOME=/code/.gradlecache \ + GRADLE_OPTS="-Dorg.gradle.vfs.watch=false -Djdk.lang.Process.launchMechanism=vfork" + +COPY ./gradle/libs.versions.toml /code/gradle/ +COPY ./build.gradle.kts ./settings.gradle.kts ./gradle.properties /code/ +COPY kafka-connect-huawei-source/build.gradle.kts /code/kafka-connect-huawei-source/ +COPY huawei-library/build.gradle /code/huawei-library/ + +RUN gradle downloadDependencies copyDependencies + +COPY ./kafka-connect-huawei-source/src/ /code/kafka-connect-huawei-source/src +COPY ./huawei-library/src/ /code/huawei-library/src + +RUN gradle jar + +FROM confluentinc/cp-kafka-connect-base:7.8.7 + +USER appuser + +LABEL org.opencontainers.image.authors="yatharth.ranjan@kcl.ac.uk, yatharthranjan@onsentia.com" + +LABEL description="Kafka Huawei Health Kit REST API Source connector" + +ENV CONNECT_PLUGIN_PATH="/usr/share/java/kafka-connect/plugins" \ + WAIT_FOR_KAFKA="1" + +# To isolate the classpath from the plugin path as recommended +COPY --from=builder /code/kafka-connect-huawei-source/build/third-party/*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-huawei-source/ +COPY --from=builder /code/huawei-library/build/third-party/*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-huawei-source/ + +COPY --from=builder /code/kafka-connect-huawei-source/build/libs/*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-huawei-source/ +COPY --from=builder /code/huawei-library/build/libs/*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-huawei-source/ + +# Load topics validator +COPY --chown=appuser:appuser ./docker/ensure /etc/confluent/docker/ensure + +# Load modified launcher +COPY --chown=appuser:appuser ./docker/launch /etc/confluent/docker/launch + +# Overwrite the log4j configuration to include Sentry monitoring. +COPY ./docker/log4j.properties.template /etc/confluent/docker/log4j.properties.template +# Copy Sentry monitoring jars. +COPY --from=builder /code/kafka-connect-huawei-source/build/third-party/sentry-* /etc/kafka-connect/jars diff --git a/kafka-connect-huawei-source/build.gradle.kts b/kafka-connect-huawei-source/build.gradle.kts new file mode 100644 index 00000000..90851ed4 --- /dev/null +++ b/kafka-connect-huawei-source/build.gradle.kts @@ -0,0 +1,38 @@ +description = "Kafka connector for Huawei Health Kit API source" + +dependencies { + + /* The entries in the block below are added here to force the version of + * transitive dependencies and mitigate reported vulnerabilities + */ + implementation(libs.netty.handler.proxy) + implementation(libs.netty.handler) + + api(project(":huawei-library")) + api(libs.kafka.connect.avro.converter) + api(libs.radar.schemas.commons.huawei) + implementation(libs.radar.commons.kotlin) + + api(libs.okhttp) + implementation(platform(libs.jackson.bom)) + implementation(libs.jackson.dataformat.yaml) + implementation(libs.jackson.datatype.jsr310) + implementation(libs.kotlin.stdlib) + + implementation(libs.ktor.client.auth) + implementation(libs.ktor.client.content.negotiation) + implementation(libs.ktor.serialization.jackson) + implementation(libs.ktor.client.cio) + implementation(libs.ktor.serialization.kotlinx.json) + implementation(libs.jackson.module.kotlin) + + // Included in connector runtime + compileOnly(libs.kafka.connect.api) + compileOnly(platform(libs.jackson.bom)) + compileOnly(libs.jackson.databind) + + testImplementation(libs.kafka.connect.api) + testImplementation(libs.wiremock) + testImplementation(libs.mockito.core) + testImplementation(libs.kotlin.test) +} diff --git a/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/AbstractRestSourceConnector.java b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/AbstractRestSourceConnector.java new file mode 100644 index 00000000..479ac97a --- /dev/null +++ b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/AbstractRestSourceConnector.java @@ -0,0 +1,60 @@ +package org.radarbase.connect.rest.huawei; + +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.kafka.connect.connector.Task; +import org.apache.kafka.connect.source.SourceConnector; +import org.radarbase.connect.rest.huawei.util.VersionUtil; + +/** + * @author yatharthranjan + */ +@SuppressWarnings("unused") +public abstract class AbstractRestSourceConnector extends SourceConnector { + protected HuaweiRestSourceConnectorConfig config; + + @Override + public String version() { + return VersionUtil.getVersion(); + } + + @Override + public Class taskClass() { + return HuaweiSourceTask.class; + } + + @Override + public List> taskConfigs(int maxTasks) { + return Collections.nCopies(maxTasks, new HashMap<>(config.originalsStrings())); + } + + @Override + public void start(Map props) { + config = getConfig(props); + } + + public abstract HuaweiRestSourceConnectorConfig getConfig(Map conf); + + @Override + public void stop() { + } +} diff --git a/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/HuaweiRestSourceConnectorConfig.kt b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/HuaweiRestSourceConnectorConfig.kt new file mode 100644 index 00000000..d797cce2 --- /dev/null +++ b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/HuaweiRestSourceConnectorConfig.kt @@ -0,0 +1,380 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.connect.rest.huawei + +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import org.apache.kafka.common.config.AbstractConfig +import org.apache.kafka.common.config.ConfigDef +import org.apache.kafka.common.config.ConfigDef.Importance +import org.apache.kafka.common.config.ConfigDef.NonEmptyString +import org.apache.kafka.common.config.ConfigDef.Type +import org.apache.kafka.common.config.ConfigDef.Width +import org.apache.kafka.common.config.ConfigException +import org.apache.kafka.connect.errors.ConnectException +import org.radarbase.connect.rest.huawei.user.HuaweiServiceUserRepository +import org.radarbase.connect.rest.huawei.user.HuaweiUserRepository +import org.radarbase.huawei.route.HuaweiRouteFactory +import java.net.MalformedURLException +import java.net.URL +import java.nio.file.Path +import java.nio.file.Paths +import java.time.Duration + +/** + * Kafka Connect configuration for the Huawei Health Kit source connector. + * + * Every data type registered in [HuaweiRouteFactory.definitions] gets a `huawei..enabled` + * boolean and a `huawei..topic` string config, generated from that single shared registry + * instead of ~110 hand-duplicated `ConfigDef` entries (one connector, one config, one canonical + * list of Huawei data types). + * + * @author yatharthranjan + */ +class HuaweiRestSourceConnectorConfig( + config: ConfigDef, + parsedConfig: MutableMap, + doLog: Boolean, +) : AbstractConfig(config, parsedConfig, doLog) { + + constructor(parsedConfig: MutableMap, doLog: Boolean) : this( + conf(), + parsedConfig, + doLog, + ) + + private var userRepository: HuaweiUserRepository? = null + + fun getHuaweiUsers(): List = getList(HUAWEI_USERS_CONFIG) + + fun getHuaweiClient(): String = getString(HUAWEI_API_CLIENT_CONFIG) + + fun getHuaweiClientSecret(): String = getPassword(HUAWEI_API_SECRET_CONFIG).value() + + fun getUserRepository(reuse: HuaweiUserRepository?): HuaweiUserRepository { + val configuredClass = getClass(HUAWEI_USER_REPOSITORY_CONFIG) + val repo = if (reuse != null && reuse.javaClass == configuredClass) { + reuse + } else { + createUserRepository() + } + repo.initialize(this) + userRepository = repo + return repo + } + + fun getUserRepository(): HuaweiUserRepository { + val repo = checkNotNull(userRepository) { "User repository has not been initialized" } + repo.initialize(this) + return repo + } + + @Suppress("UNCHECKED_CAST") + private fun createUserRepository(): HuaweiUserRepository = try { + (getClass(HUAWEI_USER_REPOSITORY_CONFIG) as Class) + .getDeclaredConstructor() + .newInstance() + } catch (e: ReflectiveOperationException) { + throw ConnectException("Invalid class. $e") + } + + /** + * Directory containing per-user YAML credential files, for the file-based + * [org.radarbase.connect.rest.huawei.user.HuaweiYamlUserRepository]. Only used if that + * repository is configured via [HUAWEI_USER_REPOSITORY_CONFIG]. + */ + fun getHuaweiUserCredentialsPath(): Path = + Paths.get(getString(HUAWEI_USER_CREDENTIALS_DIR_CONFIG)) + + fun getHuaweiUserRepositoryUrl(): HttpUrl { + var urlString = getString(HUAWEI_USER_REPOSITORY_URL_CONFIG).trim() + if (urlString.isNotEmpty() && urlString.last() != '/') { + urlString += "/" + } + return urlString.toHttpUrlOrNull() + ?: throw ConfigException( + HUAWEI_USER_REPOSITORY_URL_CONFIG, + urlString, + "User repository URL $urlString cannot be parsed as URL.", + ) + } + + fun getPollIntervalPerUser(): Duration = Duration.ofSeconds( + getInt(HUAWEI_USER_POLL_INTERVAL_CONFIG).toLong(), + ) + + fun getHuaweiUserRepositoryClientId(): String = getString( + HUAWEI_USER_REPOSITORY_CLIENT_ID_CONFIG, + ) + + fun getHuaweiUserRepositoryClientSecret(): String = + getPassword(HUAWEI_USER_REPOSITORY_CLIENT_SECRET_CONFIG).value() + + fun getHuaweiUserRepositoryTokenUrl(): URL? { + val value = getString(HUAWEI_USER_REPOSITORY_TOKEN_URL_CONFIG) + if (value.isNullOrEmpty()) { + return null + } + return try { + URL(value) + } catch (e: MalformedURLException) { + throw ConfigException("Huawei user repository token URL is invalid.") + } + } + + /** + * The (config key -> Kafka topic) pairs of every Huawei data type that is enabled in this + * configuration. + */ + fun enabledTopics(): Map = + HuaweiRouteFactory.definitions + .filter { getBoolean(enabledKey(it.key)) } + .associate { it.key to getString(topicKey(it.key)) } + + companion object { + private const val SOURCE_POLL_INTERVAL_CONFIG = "rest.source.poll.interval.ms" + private const val SOURCE_POLL_INTERVAL_DOC = "How often to poll the source URL." + private const val SOURCE_POLL_INTERVAL_DISPLAY = "Polling interval" + private const val SOURCE_POLL_INTERVAL_DEFAULT = 60000L + + const val SOURCE_URL_CONFIG = "rest.source.base.url" + private const val SOURCE_URL_DOC = "Base URL for REST source connector." + private const val SOURCE_URL_DISPLAY = "Base URL for REST source connector." + const val SOURCE_URL_DEFAULT = "https://health-api.cloud.huawei.com/healthkit/v1" + + const val HUAWEI_USERS_CONFIG = "huawei.users" + private const val HUAWEI_USERS_DOC = + "The user ID of Huawei users to include in polling, separated by commas. " + + "Non existing user names will be ignored. " + + "If empty, all users in the user directory will be used." + private const val HUAWEI_USERS_DISPLAY = "Huawei users" + + const val HUAWEI_API_CLIENT_CONFIG = "huawei.api.client" + private const val HUAWEI_API_CLIENT_DOC = "Client ID for the Huawei Health Kit API" + private const val HUAWEI_API_CLIENT_DISPLAY = "Huawei API client ID" + + const val HUAWEI_API_SECRET_CONFIG = "huawei.api.secret" + private const val HUAWEI_API_SECRET_DOC = + "Secret for the Huawei API client set in huawei.api.client." + private const val HUAWEI_API_SECRET_DISPLAY = "Huawei API client secret" + + const val HUAWEI_USER_REPOSITORY_CONFIG = "huawei.user.repository.class" + private const val HUAWEI_USER_REPOSITORY_DOC = + "Class for managing users and authentication." + private const val HUAWEI_USER_REPOSITORY_DISPLAY = "User repository class" + + const val HUAWEI_USER_POLL_INTERVAL_CONFIG = "huawei.user.poll.interval" + private const val HUAWEI_USER_POLL_INTERVAL_DOC = + "Polling interval per Huawei user per request route in seconds." + private const val HUAWEI_USER_POLL_INTERVAL_DEFAULT = 150 + private const val HUAWEI_USER_POLL_INTERVAL_DISPLAY = "Per-user per-route polling interval." + + const val HUAWEI_USER_CREDENTIALS_DIR_CONFIG = "huawei.user.dir" + private const val HUAWEI_USER_CREDENTIALS_DIR_DOC = + "Directory containing Huawei user information and credentials. Only used if a " + + "file-based user repository is configured." + private const val HUAWEI_USER_CREDENTIALS_DIR_DISPLAY = "User directory" + private const val HUAWEI_USER_CREDENTIALS_DIR_DEFAULT = + "/var/lib/kafka-connect-huawei-source/users" + + const val HUAWEI_USER_REPOSITORY_URL_CONFIG = "huawei.user.repository.url" + private const val HUAWEI_USER_REPOSITORY_URL_DOC = + "URL for webservice containing user credentials. Only used if a webservice-based " + + "user repository is configured." + private const val HUAWEI_USER_REPOSITORY_URL_DISPLAY = "User repository URL" + private const val HUAWEI_USER_REPOSITORY_URL_DEFAULT = "" + + const val HUAWEI_USER_REPOSITORY_CLIENT_ID_CONFIG = "huawei.user.repository.client.id" + private const val HUAWEI_USER_REPOSITORY_CLIENT_ID_DOC = + "Client ID for connecting to the service repository." + private const val HUAWEI_USER_REPOSITORY_CLIENT_ID_DISPLAY = + "Client ID for user repository." + + const val HUAWEI_USER_REPOSITORY_CLIENT_SECRET_CONFIG = + "huawei.user.repository.client.secret" + private const val HUAWEI_USER_REPOSITORY_CLIENT_SECRET_DOC = + "Client secret for connecting to the service repository." + private const val HUAWEI_USER_REPOSITORY_CLIENT_SECRET_DISPLAY = + "Client Secret for user repository." + + const val HUAWEI_USER_REPOSITORY_TOKEN_URL_CONFIG = + "huawei.user.repository.oauth2.token.url" + private const val HUAWEI_USER_REPOSITORY_TOKEN_URL_DOC = + "OAuth 2.0 token url for retrieving client credentials." + private const val HUAWEI_USER_REPOSITORY_TOKEN_URL_DISPLAY = "OAuth 2.0 token URL." + + private fun enabledKey(key: String) = "huawei.$key.enabled" + private fun topicKey(key: String) = "huawei.$key.topic" + + @JvmStatic + fun conf(): ConfigDef { + val group = "Huawei" + var order = 0 + + val def = ConfigDef() + .define( + SOURCE_POLL_INTERVAL_CONFIG, + Type.LONG, + SOURCE_POLL_INTERVAL_DEFAULT, + Importance.LOW, + SOURCE_POLL_INTERVAL_DOC, + group, + ++order, + Width.SHORT, + SOURCE_POLL_INTERVAL_DISPLAY, + ) + .define( + SOURCE_URL_CONFIG, + Type.STRING, + SOURCE_URL_DEFAULT, + Importance.HIGH, + SOURCE_URL_DOC, + group, + ++order, + Width.SHORT, + SOURCE_URL_DISPLAY, + ) + .define( + HUAWEI_USERS_CONFIG, + Type.LIST, + emptyList(), + Importance.HIGH, + HUAWEI_USERS_DOC, + group, + ++order, + Width.SHORT, + HUAWEI_USERS_DISPLAY, + ) + .define( + HUAWEI_API_CLIENT_CONFIG, + Type.STRING, + ConfigDef.NO_DEFAULT_VALUE, + NonEmptyString(), + Importance.HIGH, + HUAWEI_API_CLIENT_DOC, + group, + ++order, + Width.SHORT, + HUAWEI_API_CLIENT_DISPLAY, + ) + .define( + HUAWEI_API_SECRET_CONFIG, + Type.PASSWORD, + ConfigDef.NO_DEFAULT_VALUE, + Importance.HIGH, + HUAWEI_API_SECRET_DOC, + group, + ++order, + Width.SHORT, + HUAWEI_API_SECRET_DISPLAY, + ) + .define( + HUAWEI_USER_POLL_INTERVAL_CONFIG, + Type.INT, + HUAWEI_USER_POLL_INTERVAL_DEFAULT, + Importance.MEDIUM, + HUAWEI_USER_POLL_INTERVAL_DOC, + group, + ++order, + Width.SHORT, + HUAWEI_USER_POLL_INTERVAL_DISPLAY, + ) + .define( + HUAWEI_USER_REPOSITORY_CONFIG, + Type.CLASS, + HuaweiServiceUserRepository::class.java, + Importance.MEDIUM, + HUAWEI_USER_REPOSITORY_DOC, + group, + ++order, + Width.SHORT, + HUAWEI_USER_REPOSITORY_DISPLAY, + ) + .define( + HUAWEI_USER_CREDENTIALS_DIR_CONFIG, + Type.STRING, + HUAWEI_USER_CREDENTIALS_DIR_DEFAULT, + Importance.LOW, + HUAWEI_USER_CREDENTIALS_DIR_DOC, + group, + ++order, + Width.SHORT, + HUAWEI_USER_CREDENTIALS_DIR_DISPLAY, + ) + .define( + HUAWEI_USER_REPOSITORY_URL_CONFIG, + Type.STRING, + HUAWEI_USER_REPOSITORY_URL_DEFAULT, + Importance.LOW, + HUAWEI_USER_REPOSITORY_URL_DOC, + group, + ++order, + Width.SHORT, + HUAWEI_USER_REPOSITORY_URL_DISPLAY, + ) + .define( + HUAWEI_USER_REPOSITORY_CLIENT_ID_CONFIG, + Type.STRING, + "", + Importance.MEDIUM, + HUAWEI_USER_REPOSITORY_CLIENT_ID_DOC, + group, + ++order, + Width.SHORT, + HUAWEI_USER_REPOSITORY_CLIENT_ID_DISPLAY, + ) + .define( + HUAWEI_USER_REPOSITORY_CLIENT_SECRET_CONFIG, + Type.PASSWORD, + "", + Importance.MEDIUM, + HUAWEI_USER_REPOSITORY_CLIENT_SECRET_DOC, + group, + ++order, + Width.SHORT, + HUAWEI_USER_REPOSITORY_CLIENT_SECRET_DISPLAY, + ) + .define( + HUAWEI_USER_REPOSITORY_TOKEN_URL_CONFIG, + Type.STRING, + "", + Importance.MEDIUM, + HUAWEI_USER_REPOSITORY_TOKEN_URL_DOC, + group, + ++order, + Width.SHORT, + HUAWEI_USER_REPOSITORY_TOKEN_URL_DISPLAY, + ) + + HuaweiRouteFactory.definitions.forEach { d -> + val label = d.key.replace('_', ' ') + def.define( + enabledKey(d.key), Type.BOOLEAN, d.enabledByDefault, Importance.LOW, + "Enable or disable Huawei $label", group, ++order, Width.SHORT, + "Huawei $label enabled", + ) + def.define( + topicKey(d.key), Type.STRING, d.defaultTopic, Importance.LOW, + "Kafka topic for Huawei $label", group, ++order, Width.SHORT, + "Huawei $label topic", + ) + } + + return def + } + } +} diff --git a/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/HuaweiSourceConnector.java b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/HuaweiSourceConnector.java new file mode 100644 index 00000000..75e8397c --- /dev/null +++ b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/HuaweiSourceConnector.java @@ -0,0 +1,145 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.huawei; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.config.ConfigException; +import org.radarbase.huawei.user.User; +import org.radarbase.connect.rest.huawei.user.HuaweiUserRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import kotlin.sequences.SequencesKt; +import kotlin.sequences.Sequence; +import kotlin.streams.jdk8.StreamsKt; + +import static org.radarbase.connect.rest.huawei.HuaweiRestSourceConnectorConfig.HUAWEI_USERS_CONFIG; + +/** + * @author yatharthranjan + */ +public class HuaweiSourceConnector extends AbstractRestSourceConnector { + + private static final Logger logger = LoggerFactory.getLogger(HuaweiSourceConnector.class); + private ScheduledExecutorService executor; + private Set configuredUsers; + private HuaweiUserRepository repository; + + @Override + public void start(Map props) { + logger.info("Starting Huawei source connector"); + super.start(props); + executor = Executors.newSingleThreadScheduledExecutor(); + + executor.scheduleAtFixedRate(() -> { + if (repository.hasPendingUpdates()) { + try { + logger.info("Requesting latest user details..."); + repository.applyPendingUpdates(); + Set newUsers = + SequencesKt.toSet(getConfig(props, false).getUserRepository(repository).stream()); + if (configuredUsers != null && !newUsers.equals(configuredUsers)) { + logger.info("User info mismatch found. Requesting reconfiguration..."); + reconfigure(); + } + } catch (IOException e) { + logger.warn("Failed to refresh users: {}", e.toString()); + } + } else { + logger.info("No pending updates found. Not attempting to refresh users."); + } + }, 0, 5, TimeUnit.MINUTES); + } + + @Override + public void stop() { + super.stop(); + executor.shutdown(); + + configuredUsers = null; + } + + private HuaweiRestSourceConnectorConfig getConfig(Map conf, boolean doLog) { + return new HuaweiRestSourceConnectorConfig(conf, doLog); + } + + @Override + public HuaweiRestSourceConnectorConfig getConfig(Map conf) { + HuaweiRestSourceConnectorConfig connectorConfig = getConfig(conf, true); + repository = connectorConfig.getUserRepository(repository); + return connectorConfig; + } + + @Override + public ConfigDef config() { + return HuaweiRestSourceConnectorConfig.conf(); + } + + @Override + public List> taskConfigs(int maxTasks) { + return configureTasks(maxTasks); + } + + private List> configureTasks(int maxTasks) { + Map baseConfig = config.originalsStrings(); + HuaweiRestSourceConnectorConfig huaweiConfig = getConfig(baseConfig); + if (repository == null) { + repository = huaweiConfig.getUserRepository(null); + } + // Divide the users over tasks + try { + Sequence ids = SequencesKt.map(huaweiConfig.getUserRepository(repository).stream(), User::getVersionedId); + List> userTasks = StreamsKt.asStream(ids) + // group users based on their hashCode, in principle, this allows for more efficient + // reconfigurations for a fixed number of tasks, since that allows existing tasks to + // only handle small modifications users to handle. + .collect(Collectors.groupingBy( + u -> Math.abs(u.hashCode()) % maxTasks, + Collectors.joining(","))) + .values().stream() + .map(u -> { + Map taskConfig = new HashMap<>(baseConfig); + taskConfig.put(HUAWEI_USERS_CONFIG, u); + return taskConfig; + }) + .collect(Collectors.toList()); + this.configuredUsers = SequencesKt.toSet(huaweiConfig.getUserRepository().stream()); + logger.info("Received userTask Configs {}", userTasks); + return userTasks; + } catch (Exception ex) { + throw new ConfigException("Cannot read users", ex); + } + } + + public void reconfigure() { + new Thread(() -> { + logger.info("Requesting reconfiguration"); + context.requestTaskReconfiguration(); + logger.info("Requested reconfiguration"); + }).start(); + } +} diff --git a/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/HuaweiSourceTask.java b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/HuaweiSourceTask.java new file mode 100644 index 00000000..62cfffa1 --- /dev/null +++ b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/HuaweiSourceTask.java @@ -0,0 +1,221 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.huawei; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.kafka.connect.data.SchemaAndValue; +import org.apache.kafka.connect.errors.ConnectException; +import org.apache.kafka.connect.source.SourceRecord; +import org.apache.kafka.connect.source.SourceTask; +import org.apache.kafka.connect.storage.OffsetStorageReader; +import org.radarbase.connect.rest.huawei.offset.KafkaOffsetManager; +import org.radarbase.connect.rest.huawei.user.HuaweiUserRepository; +import org.radarbase.connect.rest.huawei.util.VersionUtil; +import org.radarbase.huawei.converter.TopicData; +import org.radarbase.huawei.request.HuaweiRequestGenerator; +import org.radarbase.huawei.request.HuaweiResult; +import org.radarbase.huawei.request.HuaweiResult.Success; +import org.radarbase.huawei.request.HuaweiResult.Error; +import org.radarbase.huawei.request.HuaweiErrorBase; +import org.radarbase.huawei.request.RestRequest; +import org.radarbase.huawei.route.HuaweiRouteDefinition; +import org.radarbase.huawei.route.HuaweiRouteFactory; +import org.radarbase.huawei.route.Route; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.radarbase.huawei.user.User; +import io.confluent.connect.avro.AvroData; +import kotlin.streams.jdk8.StreamsKt; +import okhttp3.OkHttpClient; +import okhttp3.Response; + +/** + * @author yatharthranjan + */ +public class HuaweiSourceTask extends SourceTask { + private static final Logger logger = LoggerFactory.getLogger(HuaweiSourceTask.class); + + private OkHttpClient baseClient; + private HuaweiUserRepository userRepository; + private List routes; + private HuaweiRequestGenerator huaweiRequestGenerator; + private final AvroData avroData = new AvroData(20); + private KafkaOffsetManager offsetManager; + private static final String TIMESTAMP_OFFSET_KEY = "timestamp"; + private static final long TIMEOUT = 60000L; + private int routeStartIndex = 0; + private final CountDownLatch stopLatch = new CountDownLatch(1); + private boolean lastPollHadRecords = false; + + public void initialize(HuaweiRestSourceConnectorConfig config, OffsetStorageReader offsetStorageReader) { + this.baseClient = new OkHttpClient(); + + this.userRepository = config.getUserRepository(); + this.offsetManager = new KafkaOffsetManager(offsetStorageReader); + this.routes = getRoutes(config); + this.huaweiRequestGenerator = new HuaweiRequestGenerator(this.userRepository, this.offsetManager, this.routes); + this.offsetManager.initialize(getPartitions()); + } + + private List getRoutes(HuaweiRestSourceConnectorConfig config) { + Map enabledTopics = config.enabledTopics(); + List result = new ArrayList<>(); + for (HuaweiRouteDefinition definition : HuaweiRouteFactory.INSTANCE.getDefinitions()) { + String topic = enabledTopics.get(definition.getKey()); + if (topic != null) { + result.add(definition.getBuild().invoke(userRepository, topic)); + } + } + return result; + } + + public List> getPartitions() { + try { + return StreamsKt.asStream(userRepository.stream()) + .flatMap(u -> this.routes.stream().map(r -> getPartition(r.toString(), u))) + .collect(Collectors.toList()); + } catch (Exception e) { + logger.warn("Failed to initialize user partitions.."); + return Collections.emptyList(); + } + } + + public Map getPartition(String route, User user) { + Map partition = new HashMap<>(4); + partition.put("user", user.getVersionedId()); + partition.put("route", route); + return partition; + } + + public Stream requests() { + if (this.routes == null || this.routes.isEmpty()) { + return Stream.empty(); + } + + // Rotate routes so that all routes are requested in a round-robin manner + List rotatedRoutes = getRotatedRoutes(); + return rotatedRoutes.stream() + .flatMap((Route r) -> StreamsKt.asStream(huaweiRequestGenerator.requests(r, 100))); + } + + private List getRotatedRoutes() { + List rotatedRoutes = new ArrayList<>(this.routes); + Collections.rotate(rotatedRoutes, routeStartIndex % this.routes.size()); + routeStartIndex = (routeStartIndex + 1) % this.routes.size(); + return rotatedRoutes; + } + + public Stream handleRequest(RestRequest req) throws IOException { + try (Response response = baseClient.newCall(req.getRequest()).execute()) { + HuaweiResult> result = this.huaweiRequestGenerator.handleResponse(req, response); + if (result instanceof HuaweiResult.Success) { + Success> success = (Success>) result; + return success.getValue().stream().map(r -> { + SchemaAndValue avro = avroData.toConnectData(r.getValue().getSchema(), r.getValue()); + SchemaAndValue key = avroData.toConnectData(r.getKey().getSchema(), r.getKey()); + Map partition = getPartition(req.getRoute().toString(), req.getUser()); + // Stored offsets mark where to resume: just after this record's start time (seconds). + Map offset = Collections.singletonMap(TIMESTAMP_OFFSET_KEY, r.getOffset() + 1); + + return new SourceRecord(partition, offset, r.getTopic(), + key.schema(), key.value(), avro.schema(), avro.value()); + }); + } else { + HuaweiErrorBase e = (HuaweiErrorBase) ((HuaweiResult.Error) result).getError(); + logger.warn("Failed to make request: {} {} {}", e.getMessage(), e.getCause(), e.getCode()); + return Stream.empty(); + } + } + } + + @Override + public void start(Map map) { + HuaweiRestSourceConnectorConfig connectorConfig; + try { + Class connector = Class.forName(map.get("connector.class")); + Object connectorInst = connector.getConstructor().newInstance(); + connectorConfig = ((HuaweiSourceConnector) connectorInst).getConfig(map); + } catch (ClassNotFoundException e) { + throw new ConnectException("Connector " + map.get("connector.class") + " not found", e); + } catch (ReflectiveOperationException e) { + throw new ConnectException("Connector " + map.get("connector.class") + + " could not be instantiated", e); + } + this.initialize(connectorConfig, context.offsetStorageReader()); + } + + @Override + public List poll() throws InterruptedException { + // Only wait between polls when there is nothing left to catch up on, and wake up as soon + // as the task is stopped. + if (!lastPollHadRecords && stopLatch.await(TIMEOUT, TimeUnit.MILLISECONDS)) { + return null; + } + + long requestsGenerated = 0; + List sourceRecords = Collections.emptyList(); + + try { + Iterator requestIterator = this.requests().iterator(); + + while (sourceRecords.isEmpty() && stopLatch.getCount() > 0 && requestIterator.hasNext()) { + RestRequest request = requestIterator.next(); + + logger.info("Requesting for user {}, url: {}", request.getUser().getUserId(), request.getRequest().url()); + requestsGenerated++; + + try { + sourceRecords = this.handleRequest(request) + .collect(Collectors.toList()); + } catch (IOException | RuntimeException ex) { + logger.warn("Failed to make request: {}", ex.toString()); + } + } + } catch (Exception ex) { + // Never let a failure to list users or build requests kill the task; retry next poll. + logger.error("Failed to generate Huawei requests: {}", ex.toString(), ex); + } + + lastPollHadRecords = !sourceRecords.isEmpty(); + logger.info("Processed {} records from {} URLs", sourceRecords.size(), requestsGenerated); + + return sourceRecords; + } + + @Override + public void stop() { + logger.debug("Stopping source task"); + stopLatch.countDown(); + } + + @Override + public String version() { + return VersionUtil.getVersion(); + } +} diff --git a/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/offset/KafkaOffsetManager.java b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/offset/KafkaOffsetManager.java new file mode 100644 index 00000000..7c84198d --- /dev/null +++ b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/offset/KafkaOffsetManager.java @@ -0,0 +1,75 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.huawei.offset; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import static java.time.temporal.ChronoUnit.NANOS; +import org.apache.kafka.connect.storage.OffsetStorageReader; +import org.radarbase.huawei.offset.Offset; +import org.radarbase.huawei.request.HuaweiOffsetManager; +import org.radarbase.huawei.route.Route; +import org.radarbase.huawei.user.User; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author yatharthranjan + */ +public class KafkaOffsetManager implements HuaweiOffsetManager { + private static final Logger logger = LoggerFactory.getLogger(KafkaOffsetManager.class); + private static final String TIMESTAMP_OFFSET_KEY = "timestamp"; + private static final Duration ONE_NANO = NANOS.getDuration(); + + private final OffsetStorageReader offsetStorageReader; + private final Map offsets = new ConcurrentHashMap<>(); + + public KafkaOffsetManager(OffsetStorageReader offsetStorageReader) { + this.offsetStorageReader = offsetStorageReader; + } + + public void initialize(List> partitions) { + if (this.offsetStorageReader != null) { + this.offsetStorageReader.offsets(partitions).entrySet().stream() + .filter(e -> e.getValue() != null && e.getValue().get(TIMESTAMP_OFFSET_KEY) instanceof Number) + .forEach(e -> offsets.put( + e.getKey().get("user") + "-" + e.getKey().get("route"), + Instant.ofEpochSecond(((Number) e.getValue().get(TIMESTAMP_OFFSET_KEY)).longValue()))); + } else { + logger.warn("Offset storage reader is null, will resume from an empty state."); + } + } + + @Override + public Offset getOffset(Route route, User user) { + Instant offset = offsets.getOrDefault(getOffsetKey(route, user), user.getStartDate().minus(ONE_NANO)); + return new Offset(user, route, offset); + } + + @Override + public void updateOffsets(Route route, User user, Instant offset) { + offsets.put(getOffsetKey(route, user), offset); + } + + private String getOffsetKey(Route route, User user) { + return user.getVersionedId() + "-" + route.toString(); + } +} diff --git a/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/HttpResponseException.java b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/HttpResponseException.java new file mode 100644 index 00000000..4a8573a9 --- /dev/null +++ b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/HttpResponseException.java @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.huawei.user; + +import java.io.IOException; + +/** + * @author yatharthranjan + */ +public class HttpResponseException extends IOException { + private final int statusCode; + + public HttpResponseException(String message, int statusCode) { + super(message); + this.statusCode = statusCode; + } + + public int getStatusCode() { + return statusCode; + } +} diff --git a/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/HuaweiLocalUser.kt b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/HuaweiLocalUser.kt new file mode 100644 index 00000000..b3f15b06 --- /dev/null +++ b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/HuaweiLocalUser.kt @@ -0,0 +1,128 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.connect.rest.huawei.user + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonProperty +import org.radarbase.huawei.user.User +import org.radarcns.kafka.ObservationKey +import java.time.Instant +import java.util.Objects + +/** + * A single user's Huawei Health Kit credentials, read from (and written back to) a local YAML + * file by [HuaweiYamlUserRepository]. Mirrors Fitbit's `LocalUser`. See + * `docker/huawei-user.yml.template` for the expected file format. + * + * @author yatharthranjan + */ +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@JsonIgnoreProperties(ignoreUnknown = true) +class HuaweiLocalUser : User { + @JsonProperty("id") + override var id: String = "" + + @JsonProperty("projectId") + override var projectId: String = "" + + @JsonProperty("userId") + override var userId: String = "" + + @JsonProperty("sourceId") + override var sourceId: String = "" + + @JsonProperty("externalUserId") + override var externalId: String? = null + + @JsonProperty("startDate") + override var startDate: Instant = Instant.parse("2017-01-01T00:00:00Z") + + @JsonProperty("endDate") + override var endDate: Instant? = Instant.parse("9999-12-31T23:59:59.999Z") + + @JsonProperty("createdAt") + override var createdAt: Instant = Instant.now() + + @JsonProperty("humanReadableUserId") + override var humanReadableUserId: String? = null + + @JsonProperty("serviceUserId") + override var serviceUserId: String? = null + + @JsonProperty("version") + override var version: String? = null + + @JsonProperty("oauth2") + var oauth2Credentials: OAuth2UserCredentials = OAuth2UserCredentials() + + @JsonProperty("authorized") + var isAuthorizedOverride: Boolean? = null + + override val isAuthorized: Boolean + get() = isAuthorizedOverride + ?: (!oauth2Credentials.isAccessTokenExpired || oauth2Credentials.hasRefreshToken()) + + override val observationKey: ObservationKey + get() = ObservationKey(projectId, userId, sourceId) + + override val versionedId: String + get() = "$id${version?.let { "#$it" } ?: ""}" + + fun copy(): HuaweiLocalUser { + val copy = HuaweiLocalUser() + copy.id = id + copy.projectId = projectId + copy.userId = userId + copy.sourceId = sourceId + copy.externalId = externalId + copy.startDate = startDate + copy.endDate = endDate + copy.createdAt = createdAt + copy.humanReadableUserId = humanReadableUserId + copy.serviceUserId = serviceUserId + copy.version = version + copy.oauth2Credentials = oauth2Credentials + copy.isAuthorizedOverride = isAuthorizedOverride + return copy + } + + /** + * Equality covers the user's identity and polling configuration, but not its OAuth2 tokens or + * [createdAt] (which defaults to the read time when absent from the file): the connector + * compares user sets to decide whether tasks need reconfiguring, and a token refresh must + * not trigger that. + */ + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is HuaweiLocalUser) return false + return id == other.id && + projectId == other.projectId && + userId == other.userId && + sourceId == other.sourceId && + externalId == other.externalId && + startDate == other.startDate && + endDate == other.endDate && + serviceUserId == other.serviceUserId && + version == other.version && + isAuthorizedOverride == other.isAuthorizedOverride + } + + override fun hashCode(): Int = Objects.hash(id, projectId, userId, sourceId, version) + + override fun toString(): String = "HuaweiLocalUser(id='$id', versionedId='$versionedId')" +} diff --git a/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/HuaweiServiceUserRepository.kt b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/HuaweiServiceUserRepository.kt new file mode 100644 index 00000000..002ea1d4 --- /dev/null +++ b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/HuaweiServiceUserRepository.kt @@ -0,0 +1,320 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.connect.rest.huawei.user + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.kotlin.readValue +import com.fasterxml.jackson.module.kotlin.registerKotlinModule +import io.ktor.client.HttpClient +import io.ktor.client.engine.cio.CIO +import io.ktor.client.plugins.HttpTimeout +import io.ktor.client.plugins.auth.Auth +import io.ktor.client.plugins.auth.providers.BasicAuthCredentials +import io.ktor.client.plugins.auth.providers.basic +import io.ktor.client.plugins.contentnegotiation.ContentNegotiation +import io.ktor.client.plugins.defaultRequest +import io.ktor.client.request.HttpRequestBuilder +import io.ktor.client.request.request +import io.ktor.client.request.setBody +import io.ktor.client.request.url +import io.ktor.client.statement.bodyAsText +import io.ktor.client.statement.request +import io.ktor.http.ContentType +import io.ktor.http.HttpMethod +import io.ktor.http.HttpStatusCode +import io.ktor.http.URLBuilder +import io.ktor.http.Url +import io.ktor.http.contentLength +import io.ktor.http.contentType +import io.ktor.http.isSuccess +import io.ktor.http.takeFrom +import io.ktor.serialization.jackson.jackson +import io.ktor.serialization.kotlinx.json.json +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.Json +import org.radarbase.connect.rest.huawei.HuaweiRestSourceConnectorConfig +import org.radarbase.huawei.user.HuaweiUser +import org.radarbase.huawei.user.User +import org.radarbase.huawei.user.UserNotAuthorizedException +import org.radarbase.kotlin.coroutines.CacheConfig +import org.radarbase.kotlin.coroutines.CachedSet +import org.radarbase.kotlin.coroutines.CachedValue +import org.radarbase.ktor.auth.ClientCredentialsConfig +import org.radarbase.ktor.auth.clientCredentials +import org.slf4j.LoggerFactory +import java.io.IOException +import java.util.concurrent.ConcurrentHashMap +import kotlin.streams.asSequence +import kotlin.time.Duration.Companion.days +import kotlin.time.Duration.Companion.hours +import kotlin.time.Duration.Companion.minutes +import kotlin.time.Duration.Companion.seconds + +/** + * User repository backed by the RADAR-base "rest-source-authorizer" webservice, mirroring + * [org.radarbase.connect.rest.oura.user.OuraServiceUserRepository]. Retrieves the list of Huawei + * users configured for a study (`GET users?source-type=Huawei`) and their Huawei Health Kit OAuth2 + * access/refresh tokens (`users//token`). + * + * @author yatharthranjan + */ +@Suppress("unused") +class HuaweiServiceUserRepository : HuaweiUserRepository() { + private lateinit var userCache: CachedSet + private lateinit var client: HttpClient + private val credentialCaches = ConcurrentHashMap>() + private val credentialCacheConfig = + CacheConfig(refreshDuration = 1.days, retryDuration = 1.minutes) + private val mapper = ObjectMapper().registerKotlinModule().registerModule(JavaTimeModule()) + + @Throws(IOException::class) + override fun get(key: String): User = + runBlocking(Dispatchers.Default) { + makeRequest { url("users/$key") } + } + + override fun initialize(config: HuaweiRestSourceConnectorConfig) { + val containedUsers = config.getHuaweiUsers().toHashSet() + val tokenUrl = config.getHuaweiUserRepositoryTokenUrl() + ?.let { URLBuilder(it.toString()).build() } + + client = + createClient( + baseUrl = URLBuilder(config.getHuaweiUserRepositoryUrl().toString()).build(), + tokenUrl = tokenUrl, + clientId = config.getHuaweiUserRepositoryClientId(), + clientSecret = config.getHuaweiUserRepositoryClientSecret(), + scope = "SUBJECT.READ MEASUREMENT.CREATE", + audience = "res_restAuthorizer", + ) + + userCache = + CachedSet( + CacheConfig(refreshDuration = 1.hours, retryDuration = 1.minutes), + ) { + makeRequest { url("users?source-type=Huawei") } + .users + .toHashSet() + .filterTo(HashSet()) { u -> + u.isComplete() && + (containedUsers.isEmpty() || u.versionedId in containedUsers) + } + } + } + + private fun createClient( + baseUrl: Url, + tokenUrl: Url?, + clientId: String?, + clientSecret: String?, + scope: String?, + audience: String?, + ): HttpClient = + HttpClient(CIO) { + if (tokenUrl != null) { + install(Auth) { + clientCredentials( + ClientCredentialsConfig( + tokenUrl.toString(), + clientId, + clientSecret, + scope, + audience, + ).copyWithEnv("MANAGEMENT_PORTAL"), + baseUrl.host, + ) + } + install(ContentNegotiation) { + json( + Json { + ignoreUnknownKeys = true + }, + ) + } + } else if (clientId != null && clientSecret != null) { + install(Auth) { + basic { + credentials { + BasicAuthCredentials(username = clientId, password = clientSecret) + } + realm = "Access to the '/' path" + sendWithoutRequest { + it.url.host == baseUrl.host + } + } + } + } + + defaultRequest { + url.takeFrom(baseUrl) + } + + install(ContentNegotiation) { + jackson { + registerModule(JavaTimeModule()) // support java.time.* types + } + } + + install(HttpTimeout) { + connectTimeoutMillis = 60.seconds.inWholeMilliseconds + requestTimeoutMillis = 90.seconds.inWholeMilliseconds + } + } + + override fun stream(): Sequence = + runBlocking(Dispatchers.Default) { + val valueInCache = + userCache.getFromCache() + .takeIf { it is CachedValue.CacheValue } + ?.getOrThrow() + + (valueInCache ?: userCache.get()) + .stream() + .filter { it.isComplete() } + .asSequence() + } + + @Throws(IOException::class, UserNotAuthorizedException::class) + override fun getAccessToken(user: User): String { + if (!user.isAuthorized) { + throw UserNotAuthorizedException("User is not authorized") + } + return runBlocking(Dispatchers.Default) { + credentialCache(user) + .get { !it.isAccessTokenExpired } + .value + .accessToken + } + } + + override fun invalidateAccessToken(user: User) { + credentialCaches -= user.id + } + + @Throws(IOException::class, UserNotAuthorizedException::class) + override fun refreshAccessToken(user: User): String { + if (!user.isAuthorized) { + throw UserNotAuthorizedException("User is not authorized") + } + return runBlocking(Dispatchers.Default) { + val token = + requestAccessToken(user) { + url("users/${user.id}/token") + method = HttpMethod.Post + setBody("{}") + contentType(ContentType.Application.Json) + } + credentialCache(user).set(token) + token.accessToken + } + } + + private suspend fun credentialCache(user: User): CachedValue = + credentialCaches.computeIfAbsent(user.id) { + CachedValue(credentialCacheConfig) { + requestAccessToken(user) { url("users/${user.id}/token") } + } + } + + @Throws(UserNotAuthorizedException::class, IOException::class) + private suspend fun requestAccessToken( + user: User, + builder: HttpRequestBuilder.() -> Unit, + ): OAuth2UserCredentials = + try { + makeRequest(builder) + } catch (ex: HttpResponseException) { + if (ex.statusCode == 407) { + credentialCaches -= user.id + throw UserNotAuthorizedException(ex.message ?: "User is not authorized") + } + throw ex + } + + override fun hasPendingUpdates(): Boolean = + runBlocking(Dispatchers.Default) { + userCache.isStale(1.hours) + } + + @Throws(IOException::class) + override fun applyPendingUpdates() { + logger.info("Requesting user information from webservice") + + runBlocking(Dispatchers.Default) { + userCache.get() + } + } + + private suspend inline fun makeRequest( + crossinline builder: HttpRequestBuilder.() -> Unit, + ): T = + withContext(Dispatchers.IO) { + val requestBuilder = HttpRequestBuilder() + builder(requestBuilder) + logger.info("Making HTTP request: ${requestBuilder.method} ${requestBuilder.url}") + + val response = client.request(builder) + logger.info("Response status: ${response.status}") + val contentLength = response.contentLength() + val transferEncoding = response.headers["Transfer-Encoding"] + val hasBody = (contentLength != null && contentLength > 0) || + (transferEncoding != null && transferEncoding.contains("chunked")) + val responseBody = try { + response.bodyAsText() + } catch (e: Exception) { + "Error reading body: ${e.message}" + } + + if (response.status == HttpStatusCode.NotFound) { + logger.error("HTTP 404 Not Found: ${response.request.url}") + throw NoSuchElementException("URL " + response.request.url + " does not exist") + } else if (!response.status.isSuccess()) { + val message = "HTTP ${response.status.value} error: $responseBody" + logger.error(message) + throw HttpResponseException(message, response.status.value) + } else if (!hasBody) { + logger.warn( + "HTTP ${response.status.value} OK but no body content. Returning empty result.", + ) + @Suppress("UNCHECKED_CAST") + return@withContext when (T::class) { + String::class -> "" as T + List::class -> emptyList() as T + else -> mapper.readValue("{}") + } + } + + try { + val result = mapper.readValue(responseBody) + logger.info("Successfully parsed response as ${T::class.simpleName}") + result + } catch (e: Exception) { + logger.error( + "Failed to parse response body as ${T::class.simpleName}: ${e.message}", + ) + logger.error("Response body that failed to parse: $responseBody") + throw e + } + } + + companion object { + private val logger = LoggerFactory.getLogger(HuaweiServiceUserRepository::class.java) + } +} diff --git a/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/HuaweiUserRepository.kt b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/HuaweiUserRepository.kt new file mode 100644 index 00000000..44efe62e --- /dev/null +++ b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/HuaweiUserRepository.kt @@ -0,0 +1,39 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.connect.rest.huawei.user + +import org.radarbase.connect.rest.huawei.HuaweiRestSourceConnectorConfig +import org.radarbase.huawei.user.User +import org.radarbase.huawei.user.UserNotAuthorizedException +import org.radarbase.huawei.user.UserRepository +import java.io.IOException + +/** + * @author yatharthranjan + */ +@Suppress("unused") +abstract class HuaweiUserRepository : UserRepository { + abstract fun initialize(config: HuaweiRestSourceConnectorConfig) + + @Throws(IOException::class, UserNotAuthorizedException::class) + abstract fun refreshAccessToken(user: User): String + + @Throws(IOException::class) + abstract fun applyPendingUpdates() + + abstract fun hasPendingUpdates(): Boolean +} diff --git a/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/HuaweiUsers.java b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/HuaweiUsers.java new file mode 100644 index 00000000..db225b9a --- /dev/null +++ b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/HuaweiUsers.java @@ -0,0 +1,43 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.huawei.user; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.ArrayList; +import java.util.List; +import org.radarbase.huawei.user.HuaweiUser; + +/** + * @author yatharthranjan + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class HuaweiUsers { + private final List users; + + @JsonCreator + public HuaweiUsers(@JsonProperty("users") List users) { + this.users = new ArrayList<>(users); + } + + public List getUsers() { + return users; + } +} diff --git a/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/HuaweiYamlUserRepository.kt b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/HuaweiYamlUserRepository.kt new file mode 100644 index 00000000..adde2b86 --- /dev/null +++ b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/HuaweiYamlUserRepository.kt @@ -0,0 +1,283 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.connect.rest.huawei.user + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.SerializationFeature +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.kotlin.registerKotlinModule +import okhttp3.FormBody +import okhttp3.OkHttpClient +import okhttp3.Request +import org.radarbase.connect.rest.huawei.HuaweiRestSourceConnectorConfig +import org.radarbase.huawei.user.User +import org.radarbase.huawei.user.UserNotAuthorizedException +import org.slf4j.LoggerFactory +import java.io.IOException +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardCopyOption +import java.time.Duration +import java.time.Instant +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicReference +import java.util.concurrent.locks.ReentrantLock +import java.util.stream.Collectors + +/** + * User repository that reads (and writes refreshed tokens back to) YAML files in a local + * directory, one file per user - mirrors Fitbit's `YamlUserRepository`. This is the easiest way + * to run this connector locally without standing up a rest-source-authorizer webservice: register + * a Huawei Health Kit OAuth2 app, obtain one user's access/refresh token by hand (e.g. via + * Huawei's OAuth 2.0 authorization code flow), and drop them into a file under the directory + * configured by `huawei.user.dir` - see `docker/huawei-user.yml.template`. + * + * @author yatharthranjan + */ +@Suppress("unused") +class HuaweiYamlUserRepository : HuaweiUserRepository() { + private val client = OkHttpClient() + private val users = ConcurrentHashMap() + private val nextFetch = AtomicReference(Instant.EPOCH) + private lateinit var credentialsDir: Path + private lateinit var clientId: String + private lateinit var clientSecret: String + private var containedUsers: Set = emptySet() + + override fun initialize(config: HuaweiRestSourceConnectorConfig) { + credentialsDir = config.getHuaweiUserCredentialsPath() + // Each task is assigned a subset of users (by versioned ID); empty means all users. + containedUsers = config.getHuaweiUsers().toHashSet() + Files.createDirectories(credentialsDir) + clientId = config.getHuaweiClient() + clientSecret = config.getHuaweiClientSecret() + } + + override operator fun get(key: String): User? { + updateUsers() + return users[key]?.locked { it.copy() } + } + + override fun stream(): Sequence { + if (nextFetch.get() == Instant.EPOCH) { + applyPendingUpdates() + } + return users.values.asSequence() + .filter { + it.locked { u -> + u.oauth2Credentials.hasRefreshToken() && + (containedUsers.isEmpty() || u.versionedId in containedUsers) + } + } + .map { it.locked { u -> u.copy() } } + } + + @Throws(IOException::class, UserNotAuthorizedException::class) + override fun getAccessToken(user: User): String { + updateUsers() + val actual = users[user.id] + ?: throw NoSuchElementException("User $user is not present in this user repository.") + val current = actual.locked { u -> + if (!u.oauth2Credentials.isAccessTokenExpired) u.oauth2Credentials.accessToken else null + } + return current ?: refreshAccessToken(user) + } + + override fun invalidateAccessToken(user: User) { + users[user.id]?.update { it.oauth2Credentials.invalidateAccessToken() } + } + + @Throws(IOException::class, UserNotAuthorizedException::class) + override fun refreshAccessToken(user: User): String { + val actual = users[user.id] + ?: throw NoSuchElementException("User $user is not present in this user repository.") + val refreshToken = actual.locked { it.oauth2Credentials.refreshToken } + val node = requestAccessToken(refreshToken) + + val expiresIn = node["expires_in"]?.asLong() + val accessToken = node["access_token"]?.asText() + ?: throw UserNotAuthorizedException("Did not get an access token") + val newRefreshToken = node["refresh_token"]?.asText() ?: refreshToken + + actual.update { u -> + u.oauth2Credentials = OAuth2UserCredentials(newRefreshToken, accessToken, expiresIn) + store(actual, u) + } + return accessToken + } + + override fun hasPendingUpdates(): Boolean = Instant.now().isAfter(nextFetch.get()) + + @Throws(IOException::class) + override fun applyPendingUpdates() { + forceUpdateUsers() + nextFetch.set(Instant.now().plus(FETCH_THRESHOLD)) + } + + private fun updateUsers() { + val next = nextFetch.get() + val now = Instant.now() + if (!now.isAfter(next) || !nextFetch.compareAndSet(next, now.plus(FETCH_THRESHOLD))) { + return + } + forceUpdateUsers() + } + + private fun forceUpdateUsers() { + val paths = try { + Files.walk(credentialsDir).use { walker -> + walker + .filter { + Files.isRegularFile(it) && + it.fileName.toString().lowercase().endsWith(".yml") + } + .sorted() + .collect(Collectors.toList()) + } + } catch (ex: IOException) { + logger.error("Failed to read user directory: {}", ex.toString()) + return + } + val newUsers = LinkedHashMap() + paths.forEach { path -> + // A single malformed or unreadable file must not hide every other user. + val lockedUser = try { + LockedUser( + YAML_READER.readValue(path.toFile(), HuaweiLocalUser::class.java), + path, + Files.getLastModifiedTime(path).toInstant(), + ) + } catch (ex: IOException) { + logger.error("Failed to read user file {}: {}", path, ex.toString()) + return@forEach + } + val id = lockedUser.user.id + val existing = newUsers.putIfAbsent(id, lockedUser) + if (existing != null) { + logger.warn( + "Ignoring {}: user ID {} already defined in {}", + path, + id, + existing.path, + ) + } + } + users.keys.retainAll(newUsers.keys) + // Keep in-memory state (e.g. freshly refreshed tokens) unless the file was edited since it + // was last read or written here, e.g. to add a new refresh token. + newUsers.forEach { (id, u) -> + users.merge(id, u) { old, new -> + if (new.modifiedAt > old.modifiedAt || new.path != old.path) new else old + } + } + } + + private fun requestAccessToken(refreshToken: String?): JsonNode { + if (refreshToken.isNullOrEmpty()) { + throw UserNotAuthorizedException("Refresh token is not set") + } + val request = Request.Builder() + .url(HUAWEI_TOKEN_URL) + .post( + // Huawei's OAuth 2.0 token endpoint takes the client credentials as form + // parameters, not as HTTP Basic authentication. + FormBody.Builder() + .add("grant_type", "refresh_token") + .add("client_id", clientId) + .add("client_secret", clientSecret) + .add("refresh_token", refreshToken) + .build(), + ) + .build() + + client.newCall(request).execute().use { response -> + val body = response.body?.string() + return when { + response.isSuccessful && body != null -> JSON_READER.readTree(body) + response.code == 400 || response.code == 401 -> + throw UserNotAuthorizedException( + "Refresh token was rejected (HTTP ${response.code}): $body", + ) + else -> throw IOException( + "Failed to request refresh token, HTTP status ${response.code}" + + (body?.let { " and content $it" } ?: ""), + ) + } + } + } + + private fun store(lockedUser: LockedUser, user: HuaweiLocalUser) { + val path = lockedUser.path + try { + // Temp file next to the target (not ending in .yml, so it is never read as a user), + // so the move is a same-directory rename. + val temp = Files.createTempFile(path.parent, ".${user.id}", ".tmp") + try { + Files.newOutputStream(temp).use { out -> YAML_WRITER.writeValue(out, user) } + Files.move(temp, path, StandardCopyOption.REPLACE_EXISTING) + lockedUser.modifiedAt = Files.getLastModifiedTime(path).toInstant() + } finally { + Files.deleteIfExists(temp) + } + } catch (ex: IOException) { + logger.error("Failed to store user file: {}", ex.toString()) + } + } + + /** Guards a mutable [HuaweiLocalUser] against concurrent read/refresh/store. */ + private class LockedUser( + val user: HuaweiLocalUser, + val path: Path, + @Volatile var modifiedAt: Instant, + ) { + private val lock = ReentrantLock() + + fun locked(block: (HuaweiLocalUser) -> V): V { + lock.lock() + try { + return block(user) + } finally { + lock.unlock() + } + } + + fun update(block: (HuaweiLocalUser) -> Unit) { + lock.lock() + try { + block(user) + } finally { + lock.unlock() + } + } + } + + companion object { + private val logger = LoggerFactory.getLogger(HuaweiYamlUserRepository::class.java) + private const val HUAWEI_TOKEN_URL = "https://oauth-login.cloud.huawei.com/oauth2/v3/token" + private val FETCH_THRESHOLD = Duration.ofHours(1L) + private val YAML_MAPPER = ObjectMapper(YAMLFactory()).apply { + registerKotlinModule() + registerModule(JavaTimeModule()) + configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false) + } + private val YAML_READER = YAML_MAPPER.reader() + private val YAML_WRITER = YAML_MAPPER.writerFor(HuaweiLocalUser::class.java) + private val JSON_READER = ObjectMapper().registerModule(JavaTimeModule()).reader() + } +} diff --git a/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/OAuth2UserCredentials.java b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/OAuth2UserCredentials.java new file mode 100644 index 00000000..7c0ecc16 --- /dev/null +++ b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/user/OAuth2UserCredentials.java @@ -0,0 +1,87 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.huawei.user; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSetter; +import java.time.Duration; +import java.time.Instant; + +/** + * @author yatharthranjan + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class OAuth2UserCredentials { + private static final Duration DEFAULT_EXPIRY = Duration.ofHours(1); + private static final Duration EXPIRY_TIME_MARGIN = Duration.ofMinutes(5); + + @JsonProperty + private String accessToken; + @JsonProperty + private String refreshToken; + @JsonProperty + private Instant expiresAt; + + public OAuth2UserCredentials() { + } + + public OAuth2UserCredentials(String refreshToken, String accessToken, Long expiresIn) { + this.refreshToken = refreshToken; + this.accessToken = accessToken; + this.expiresAt = getExpiresAt(expiresIn != null && expiresIn > 0L + ? Duration.ofSeconds(expiresIn) : DEFAULT_EXPIRY); + } + + public String getAccessToken() { + return accessToken; + } + + @JsonSetter + public void setAccessToken(String accessToken) { + this.accessToken = accessToken; + if (expiresAt == null) { + expiresAt = getExpiresAt(DEFAULT_EXPIRY); + } + } + + public boolean hasRefreshToken() { + return refreshToken != null && !refreshToken.isEmpty(); + } + + public String getRefreshToken() { + return refreshToken; + } + + protected static Instant getExpiresAt(Duration expiresIn) { + return Instant.now() + .plus(expiresIn) + .minus(EXPIRY_TIME_MARGIN); + } + + /** Mark the access token as expired, so it is refreshed before its next use. */ + public void invalidateAccessToken() { + expiresAt = Instant.EPOCH; + } + + @JsonIgnore + public boolean isAccessTokenExpired() { + return expiresAt == null || Instant.now().isAfter(expiresAt); + } +} diff --git a/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/util/VersionUtil.java b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/util/VersionUtil.java new file mode 100644 index 00000000..1c74b19f --- /dev/null +++ b/kafka-connect-huawei-source/src/main/java/org/radarbase/connect/rest/huawei/util/VersionUtil.java @@ -0,0 +1,35 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.huawei.util; + +/** + * @author yatharthranjan + */ +public final class VersionUtil { + private VersionUtil() { + // utility class + } + + public static String getVersion() { + try { + return VersionUtil.class.getPackage().getImplementationVersion(); + } catch (Exception ex) { + return "0.0.0.0"; + } + } +} diff --git a/kafka-connect-huawei-source/src/test/java/org/radarbase/connect/rest/huawei/HuaweiRestSourceConnectorConfigTest.kt b/kafka-connect-huawei-source/src/test/java/org/radarbase/connect/rest/huawei/HuaweiRestSourceConnectorConfigTest.kt new file mode 100644 index 00000000..6ce94da9 --- /dev/null +++ b/kafka-connect-huawei-source/src/test/java/org/radarbase/connect/rest/huawei/HuaweiRestSourceConnectorConfigTest.kt @@ -0,0 +1,66 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.huawei + +import org.junit.jupiter.api.Test +import org.radarbase.huawei.route.HuaweiRouteFactory +import kotlin.test.assertEquals + +/** + * @author yatharthranjan + */ +class HuaweiRestSourceConnectorConfigTest { + + @Test + fun conf() { + println(HuaweiRestSourceConnectorConfig.conf().toHtmlTable()) + } + + @Test + fun `enabled topics default to every data type enabled by default`() { + val config = HuaweiRestSourceConnectorConfig( + mutableMapOf( + "huawei.api.client" to "client", + "huawei.api.secret" to "secret", + ), + false, + ) + + val enabled = config.enabledTopics() + + HuaweiRouteFactory.definitions.forEach { definition -> + val expected = definition.defaultTopic.takeIf { definition.enabledByDefault } + assertEquals(expected, enabled[definition.key], definition.key) + } + } + + @Test + fun `a data type can be disabled via config`() { + val definition = HuaweiRouteFactory.definitions.first() + val config = HuaweiRestSourceConnectorConfig( + mutableMapOf( + "huawei.api.client" to "client", + "huawei.api.secret" to "secret", + "huawei.${definition.key}.enabled" to "false", + ), + false, + ) + + assertEquals(null, config.enabledTopics()[definition.key]) + } +} diff --git a/kafka-connect-huawei-source/src/test/java/org/radarbase/connect/rest/huawei/user/HuaweiYamlUserRepositoryTest.kt b/kafka-connect-huawei-source/src/test/java/org/radarbase/connect/rest/huawei/user/HuaweiYamlUserRepositoryTest.kt new file mode 100644 index 00000000..94538f1c --- /dev/null +++ b/kafka-connect-huawei-source/src/test/java/org/radarbase/connect/rest/huawei/user/HuaweiYamlUserRepositoryTest.kt @@ -0,0 +1,116 @@ +/* + * Copyright 2026 Onsentia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.huawei.user + +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import org.radarbase.connect.rest.huawei.HuaweiRestSourceConnectorConfig +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.attribute.FileTime +import java.time.Instant +import kotlin.test.assertEquals + +/** + * @author yatharthranjan + */ +class HuaweiYamlUserRepositoryTest { + @TempDir + lateinit var dir: Path + + private fun writeUser(id: String, refreshToken: String, accessToken: String = "access") { + Files.writeString( + dir.resolve("$id.yml"), + """ + id: $id + projectId: p + userId: $id + sourceId: s + startDate: 2024-01-01T00:00:00Z + oauth2: + accessToken: $accessToken + refreshToken: $refreshToken + expiresAt: 2999-01-01T00:00:00Z + """.trimIndent(), + ) + } + + private fun repository(users: String = ""): HuaweiYamlUserRepository { + val config = HuaweiRestSourceConnectorConfig( + mutableMapOf( + "huawei.api.client" to "client", + "huawei.api.secret" to "secret", + "huawei.user.dir" to dir.toString(), + "huawei.users" to users, + ), + false, + ) + return HuaweiYamlUserRepository().apply { initialize(config) } + } + + @Test + fun `streams only users assigned to the task and skips unreadable files`() { + writeUser("a", "refresh-a") + writeUser("b", "refresh-b") + writeUser("c", "") + Files.writeString(dir.resolve("broken.yml"), "id: [unterminated") + + assertEquals(setOf("a", "b"), repository().stream().map { it.id }.toSet()) + assertEquals(setOf("b"), repository("b").stream().map { it.id }.toSet()) + } + + @Test + fun `users are equal regardless of tokens and read time`() { + writeUser("a", "refresh-a") + val first = repository().stream().single() + writeUser("a", "refresh-a2", accessToken = "other") + val second = repository().stream().single() + + assertEquals(first, second) + assertEquals(first.hashCode(), second.hashCode()) + } + + @Test + fun `invalidated access token is no longer returned as valid`() { + writeUser("a", "refresh-a") + val repository = repository() + val user = repository.stream().single() + assertEquals("access", repository.getAccessToken(user)) + + repository.invalidateAccessToken(user) + + val credentials = (repository["a"] as HuaweiLocalUser).oauth2Credentials + assertEquals(true, credentials.isAccessTokenExpired) + } + + @Test + fun `edited user files are reloaded`() { + writeUser("a", "refresh-a", accessToken = "old") + val repository = repository() + assertEquals("old", repository.getAccessToken(repository.stream().single())) + + writeUser("a", "refresh-a", accessToken = "new") + Files.setLastModifiedTime( + dir.resolve("a.yml"), + FileTime.from(Instant.now().plusSeconds(60)), + ) + repository.applyPendingUpdates() + + assertEquals("new", repository.getAccessToken(repository.stream().single())) + } +} diff --git a/scripts/huawei-api-probe.sh b/scripts/huawei-api-probe.sh new file mode 100755 index 00000000..caa96fd6 --- /dev/null +++ b/scripts/huawei-api-probe.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash +# +# Copyright 2026 Onsentia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# @author yatharthranjan +# +# Queries every Huawei Health Kit endpoint exactly as the RADAR Huawei connector does, saving +# each response (with its HTTP status) to $OUT_DIR/.json for review. +# +# Authenticate with EITHER a refresh token plus your Huawei app's client credentials (a new +# access token is fetched first): +# HUAWEI_CLIENT_ID= HUAWEI_CLIENT_SECRET= \ +# HUAWEI_REFRESH_TOKEN= ./huawei-api-probe.sh +# OR a still-valid access token: +# HUAWEI_TOKEN= ./huawei-api-probe.sh +# +# Instead of HUAWEI_REFRESH_TOKEN you can pass USER_FILE=docker/users/.yml to read +# oauth2.refreshToken from the connector's user file (the file is not modified). +# +# The route list below mirrors org.radarbase.huawei.route.HuaweiRouteFactory; keep it in sync when +# routes are added or changed. Responses contain health data: don't commit or share them untrimmed. +# +# Options: +# DAYS=30 ./huawei-api-probe.sh # window size, max 30 (default 7) +# ./huawei-api-probe.sh continuous_ecg_detail emotion # only these routes +set -euo pipefail + +TOKEN_URL=https://oauth-login.cloud.huawei.com/oauth2/v3/token + +json_field() { # json key -> string/number value of a top-level key, or empty + if command -v python3 >/dev/null; then + python3 -c 'import json,sys; v=json.loads(sys.argv[1]).get(sys.argv[2]); print("" if v is None else v)' "$1" "$2" + elif command -v jq >/dev/null; then + jq -r --arg k "$2" '.[$k] // empty' <<<"$1" + else + sed -n "s/.*\"$2\"[[:space:]]*:[[:space:]]*\"\{0,1\}\([^\",}]*\).*/\1/p" <<<"$1" + fi +} + +if [ -z "${HUAWEI_REFRESH_TOKEN:-}" ] && [ -n "${USER_FILE:-}" ]; then + # First non-empty "refreshToken:" value in the YAML file, without surrounding quotes. + HUAWEI_REFRESH_TOKEN=$(sed -n 's/^[[:space:]]*refreshToken:[[:space:]]*//p' "$USER_FILE" \ + | head -n 1 | sed -e 's/[[:space:]]*$//' -e 's/^["'\'']//' -e 's/["'\'']$//') + [ -n "$HUAWEI_REFRESH_TOKEN" ] || { echo "No refreshToken found in $USER_FILE" >&2; exit 1; } +fi + +if [ -n "${HUAWEI_REFRESH_TOKEN:-}" ]; then + : "${HUAWEI_CLIENT_ID:?Set HUAWEI_CLIENT_ID (your Huawei app ID, huawei.api.client in the connector config)}" + : "${HUAWEI_CLIENT_SECRET:?Set HUAWEI_CLIENT_SECRET (huawei.api.secret in the connector config)}" + echo "Refreshing access token..." + TOKEN_RESPONSE=$(curl -sS -w '\n%{http_code}' -X POST "$TOKEN_URL" \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + --data-urlencode grant_type=refresh_token \ + --data-urlencode "client_id=$HUAWEI_CLIENT_ID" \ + --data-urlencode "client_secret=$HUAWEI_CLIENT_SECRET" \ + --data-urlencode "refresh_token=$HUAWEI_REFRESH_TOKEN") + TOKEN_STATUS=${TOKEN_RESPONSE##*$'\n'} + TOKEN_BODY=${TOKEN_RESPONSE%$'\n'*} + if [ "$TOKEN_STATUS" != 200 ]; then + echo "Token refresh failed (HTTP $TOKEN_STATUS): $TOKEN_BODY" >&2 + exit 1 + fi + HUAWEI_TOKEN=$(json_field "$TOKEN_BODY" access_token) + [ -n "$HUAWEI_TOKEN" ] || { echo "No access_token in token response: $TOKEN_BODY" >&2; exit 1; } + echo "Got access token (expires in $(json_field "$TOKEN_BODY" expires_in)s, scopes: $(json_field "$TOKEN_BODY" scope))" + NEW_REFRESH_TOKEN=$(json_field "$TOKEN_BODY" refresh_token) + if [ -n "$NEW_REFRESH_TOKEN" ] && [ "$NEW_REFRESH_TOKEN" != "$HUAWEI_REFRESH_TOKEN" ]; then + echo >&2 + echo "WARNING: Huawei issued a NEW refresh token. The old one may no longer work, so put this" >&2 + echo "one in oauth2.refreshToken of your user file before (re)starting the connector:" >&2 + echo "$NEW_REFRESH_TOKEN" >&2 + echo >&2 + fi +fi + +: "${HUAWEI_TOKEN:?Set HUAWEI_REFRESH_TOKEN (+ HUAWEI_CLIENT_ID/SECRET), USER_FILE, or HUAWEI_TOKEN}" +DAYS="${DAYS:-7}" +OUT_DIR="${OUT_DIR:-huawei-responses}" +V1=https://health-api.cloud.huawei.com/healthkit/v1 +V2=https://health-api.cloud.huawei.com/healthkit/v2 + +# Window: the last DAYS complete UTC days, [START, END). +END_S=$(( $(date -u +%s) / 86400 * 86400 )) +START_S=$(( END_S - DAYS * 86400 )) +utc_day() { date -u -d "@$1" +%Y%m%d 2>/dev/null || date -u -r "$1" +%Y%m%d; } # GNU || BSD/macOS +START_MS=${START_S}000 +END_MS=${END_S}000 +START_NS=${START_S}000000000 +END_NS=${END_S}000000000 +START_DAY=$(utc_day "$START_S") +END_DAY=$(utc_day "$(( END_S - 86400 ))") # endDay is inclusive + +mkdir -p "$OUT_DIR" +echo "Window: $START_DAY .. $END_DAY (UTC), saving to $OUT_DIR/" + +wanted() { [ $# -eq 0 ] && return 0; local k; for k in $ONLY; do [ "$k" = "$1" ] && return 0; done; return 1; } +ONLY="$*" + +call() { # key method url [json body] + local key=$1 method=$2 url=$3 body=${4:-} + [ -n "$ONLY" ] && ! wanted "$key" && return 0 + local args=(-sS -o "$OUT_DIR/$key.json" -w '%{http_code}' -X "$method" "$url" + -H "Authorization: Bearer $HUAWEI_TOKEN" -H 'Content-Type: application/json;charset=UTF-8') + [ -n "$body" ] && args+=(--data "$body") + local status; status=$(curl "${args[@]}" || echo "curl-failed") + if [ "$status" = 200 ]; then + printf '%-45s %s\n' "$key" "$status" + else + # Error bodies contain no health data; show them inline for easy sharing. + printf '%-45s %s %s\n' "$key" "$status" "$(head -c 300 "$OUT_DIR/$key.json" 2>/dev/null | tr -d '\n')" + fi + sleep 1 # stay well clear of rate limits +} + +polymerize() { # key dataTypeName + local body + body=$(printf '{"polymerizeWith":[{"dataTypeName":"%s"}],"startTime":%s,"endTime":%s}' \ + "$2" "$START_MS" "$END_MS") + call "$1" POST "$V1/sampleSet:polymerize" "$body" +} + +daily() { # key dataTypeName + local body + body=$(printf '{"dataTypes":["%s"],"startDay":"%s","endDay":"%s","timeZone":"+0000"}' \ + "$2" "$START_DAY" "$END_DAY") + call "$1" POST "$V2/sampleSet:dailyPolymerize" "$body" +} + +health_record() { # key dataType [subDataType] + local url="$V2/healthRecords?dataType=$2&startTime=$START_NS&endTime=$END_NS" + [ -n "${3:-}" ] && url="$url&subDataType=$3" + call "$1" GET "$url" +} + +call activity_record GET "$V2/activityRecords?startTime=$START_MS&endTime=$END_MS" + + +# Raw sample points: POST v1 sampleSet:polymerize (times in ms) +polymerize cgm_blood_glucose com.huawei.cgm_blood_glucose +polymerize active_hours com.huawei.active_hours +polymerize continuous_activity_fragment com.huawei.continuous.activity.fragment +polymerize instantaneous_body_temperature com.huawei.instantaneous.body.temperature +polymerize instantaneous_skin_temperature com.huawei.instantaneous.skin.temperature +polymerize continuous_calories_burnt com.huawei.continuous.calories.burnt +polymerize continuous_calories_consumed com.huawei.continuous.calories.consumed # disabled by default +polymerize continuous_distance_delta com.huawei.continuous.distance.delta +polymerize continuous_exercise_intensity com.huawei.continuous.exercise_intensity +polymerize continuous_exercise_intensity_v2 com.huawei.continuous.exercise_intensity.v2 +polymerize continuous_sleep_fragment com.huawei.continuous.sleep.fragment +polymerize continuous_steps_delta com.huawei.continuous.steps.delta +polymerize emotion com.huawei.emotion +polymerize heart_rate_variability com.huawei.heart_rate_variability +polymerize sleep_on_off_bed_record com.huawei.sleep.on_off_bed_record +polymerize sleep_respiratory_detail com.huawei.sleep_respiratory_detail +polymerize sleep_respiratory_event com.huawei.sleep_respiratory_event +polymerize vo2max com.huawei.vo2max + +# Daily statistics: POST v2 sampleSet:dailyPolymerize (the raw type is queried) +daily cgm_blood_glucose_statistics com.huawei.cgm_blood_glucose +daily daily_activity_summary com.huawei.daily_activity_summary +daily active_hours_statistics com.huawei.active_hours +daily continuous_activity_statistics com.huawei.continuous.activity +daily continuous_altitude_statistics com.huawei.instantaneous.altitude +daily continuous_blood_glucose_statistics com.huawei.instantaneous.blood_glucose +daily continuous_breathe_rate_statistics com.huawei.instantaneous.breathe_rate +daily continuous_body_blood_pressure_statistics com.huawei.instantaneous.blood_pressure +daily continuous_body_temperature_rest_statistics com.huawei.continuous.body.temperature.rest # disabled by default +daily continuous_calories_bmr_statistics com.huawei.continuous.calories.bmr +daily continuous_exercise_heart_rate_statistics com.huawei.continuous.exercise_heart_rate +daily continuous_power_statistics com.huawei.continuous.power +daily continuous_speed_statistics com.huawei.continuous.speed +daily continuous_steps_rate_statistics com.huawei.continuous.steps.rate +daily continuous_stroke_rate_statistics com.huawei.continuous.stroke_rate +daily instantaneous_resting_heart_rate_statistics com.huawei.instantaneous.resting_heart_rate +daily instantaneous_stress_statistics com.huawei.instantaneous.stress +daily vo2max_statistics com.huawei.vo2max +daily continuous_heart_rate_statistics com.huawei.instantaneous.heart_rate +daily continuous_body_temperature_statistics com.huawei.instantaneous.body.temperature +daily continuous_skin_temperature_statistics com.huawei.instantaneous.skin.temperature +daily continuous_body_fat_rate_statistics com.huawei.instantaneous.body_weight +daily continuous_calories_burnt_total com.huawei.continuous.calories.burnt +daily continuous_distance_total com.huawei.continuous.distance.delta +daily continuous_exercise_intensity_statistics com.huawei.continuous.exercise_intensity +daily continuous_exercise_intensity_v2_statistics com.huawei.continuous.exercise_intensity.v2 +daily continuous_spo2_statistics com.huawei.instantaneous.spo2 +daily continuous_steps_total com.huawei.continuous.steps.delta +daily resting_calories_statistics com.huawei.resting_calories + +# Health records: GET v2 healthRecords (times in ns) +health_record continuous_ecg_detail com.huawei.continuous.ecg_record com.huawei.continuous.ecg_detail +health_record health_record_dynamic_bp com.huawei.health.record.dynamic_bp +health_record health_record_bradycardia com.huawei.health.record.bradycardia +health_record health_record_tachycardia com.huawei.health.record.tachycardia +health_record health_record_hyperthermia com.huawei.health.record.hyperthermia +health_record health_record_low_spo2_alert com.huawei.health.record.lowSpo2Alert +health_record health_record_menstrual_cycle com.huawei.health.record.menstrual_cycle +health_record health_record_sleep com.huawei.health.record.sleep diff --git a/settings.gradle.kts b/settings.gradle.kts index 64f23944..260feaed 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -3,6 +3,8 @@ include(":kafka-connect-fitbit-source") include(":kafka-connect-rest-source") include(":kafka-connect-oura-source") include(":oura-library") +include(":kafka-connect-huawei-source") +include(":huawei-library") pluginManagement { repositories {