diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..fdb096e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +* +docker/ +**/* diff --git a/deploy/Chart.yaml b/deploy/Chart.yaml index cffa1be..bfbe085 100644 --- a/deploy/Chart.yaml +++ b/deploy/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -name: biothings-annotator -description: A Helm chart for Biothings/pending.api-annotator +name: node-normalization +description: A Helm chart for NodeNormalizationAPI # A chart can be either an 'application' or a 'library' chart. # diff --git a/deploy/Jenkinsfile b/deploy/Jenkinsfile index 7e8335f..244fff4 100644 --- a/deploy/Jenkinsfile +++ b/deploy/Jenkinsfile @@ -15,7 +15,7 @@ pipeline { pollSCM('H/5 * * * *') } environment { - DOCKER_REPO_NAME = "853771734544.dkr.ecr.us-east-1.amazonaws.com/translator-bte-pending-api-annotator" + DOCKER_REPO_NAME = "853771734544.dkr.ecr.us-east-1.amazonaws.com/translator-nodenormalization-api" KUBERNETES_BLUE_CLUSTER_NAME = "translator-eks-ci-blue-cluster" KUBERNETES_GREEN_CLUSTER_NAME = "translator-eks-ci-green-cluster" PACKAGE_DIR = "docker" diff --git a/deploy/deploy.sh b/deploy/deploy.sh index de2ad12..1796392 100644 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -1,6 +1,6 @@ #!/bin/bash -projectName="biothings-annotator" +projectName="node-normalization" namespace="bte" #set +x diff --git a/deploy/templates/NOTES.txt b/deploy/templates/NOTES.txt index c987cb3..02818b6 100644 --- a/deploy/templates/NOTES.txt +++ b/deploy/templates/NOTES.txt @@ -6,16 +6,16 @@ {{- end }} {{- end }} {{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "pendingapi.fullname" . }}) + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "node-normalization.fullname" . }}) export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") echo http://$NODE_IP:$NODE_PORT {{- else if contains "LoadBalancer" .Values.service.type }} NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "pendingapi.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "pendingapi.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "node-normalization.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "node-normalization.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") echo http://$SERVICE_IP:{{ .Values.service.port }} {{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "pendingapi.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "node-normalization.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") echo "Visit http://127.0.0.1:8080 to use your application" kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT diff --git a/deploy/templates/_helpers.tpl b/deploy/templates/_helpers.tpl index 7cde9dc..5bc42c6 100644 --- a/deploy/templates/_helpers.tpl +++ b/deploy/templates/_helpers.tpl @@ -1,7 +1,7 @@ {{/* Expand the name of the chart. */}} -{{- define "biothings-annotator.name" -}} +{{- define "node-normalization.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} @@ -10,7 +10,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "biothings-annotator.fullname" -}} +{{- define "node-normalization.fullname" -}} {{- if .Values.fullnameOverride }} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} {{- else }} @@ -26,16 +26,16 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "biothings-annotator.chart" -}} +{{- define "node-normalization.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Common labels */}} -{{- define "biothings-annotator.labels" -}} -helm.sh/chart: {{ include "biothings-annotator.chart" . }} -{{ include "biothings-annotator.selectorLabels" . }} +{{- define "node-normalization.labels" -}} +helm.sh/chart: {{ include "node-normalization.chart" . }} +{{ include "node-normalization.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} @@ -45,7 +45,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{/* Selector labels */}} -{{- define "biothings-annotator.selectorLabels" -}} -app.kubernetes.io/name: {{ include "biothings-annotator.name" . }} +{{- define "node-normalization.selectorLabels" -}} +app.kubernetes.io/name: {{ include "node-normalization.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} diff --git a/deploy/templates/deployment.yaml b/deploy/templates/deployment.yaml index c3471c8..ee22f7b 100644 --- a/deploy/templates/deployment.yaml +++ b/deploy/templates/deployment.yaml @@ -1,9 +1,9 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "biothings-annotator.fullname" . }} + name: {{ include "node-normalization.fullname" . }} labels: - {{- include "biothings-annotator.labels" . | nindent 4 }} + {{- include "node-normalization.labels" . | nindent 4 }} spec: strategy: type: {{ .Values.deployment.strategy.type }} @@ -15,11 +15,11 @@ spec: replicas: {{ .Values.deployment.replicaCount }} selector: matchLabels: - {{- include "biothings-annotator.selectorLabels" . | nindent 6 }} + {{- include "node-normalization.selectorLabels" . | nindent 6 }} template: metadata: labels: - {{- include "biothings-annotator.selectorLabels" . | nindent 8 }} + {{- include "node-normalization.selectorLabels" . | nindent 8 }} {{- toYaml .Values.ncats.labels | nindent 8 }} spec: containers: @@ -30,11 +30,9 @@ spec: - name: ES_HOST value: {{ .Values.containers.es_host }} - name: OPENTELEMETRY_ENABLED - value: "{{ .Values.containers.OPENTELEMETRY_ENABLED_VALUE }}" - - name: OPENTELEMETRY_JAEGER_HOST - value: "{{ .Values.containers.OPENTELEMETRY_JAEGER_HOST_VALUE }}" - - name: OPENTELEMETRY_JAEGER_PORT - value: "{{ .Values.containers.OPENTELEMETRY_JAEGER_PORT_VALUE }}" + value: "{{ .Values.env.OPENTELEMETRY_ENABLED_VALUE }}" + - name: PORT + value: "{{ .Values.containers.port }}" ports: - name: http containerPort: {{ .Values.containers.port }} diff --git a/deploy/templates/ingress.yaml b/deploy/templates/ingress.yaml index ae538cc..a3f85e6 100644 --- a/deploy/templates/ingress.yaml +++ b/deploy/templates/ingress.yaml @@ -1,12 +1,12 @@ {{- if .Values.ingress.enabled -}} -{{- $fullName := include "biothings-annotator.fullname" . -}} +{{- $fullName := include "node-normalization.fullname" . -}} {{- $svcPort := .Values.service.port -}} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: {{ $fullName }} labels: - {{- include "biothings-annotator.labels" . | nindent 4 }} + {{- include "node-normalization.labels" . | nindent 4 }} {{- with .Values.ingress.annotations }} annotations: {{- toYaml . | nindent 4 }} @@ -27,7 +27,7 @@ spec: pathType: ImplementationSpecific backend: service: - name: {{ include "biothings-annotator.fullname" . }} + name: {{ include "node-normalization.fullname" . }} port: number: 80 - host: {{ .Values.ingress.annotatorHost | quote }} @@ -37,7 +37,7 @@ spec: pathType: ImplementationSpecific backend: service: - name: {{ include "biothings-annotator.fullname" . }} + name: {{ include "node-normalization.fullname" . }} port: number: 80 {{- end }} diff --git a/deploy/templates/service.yaml b/deploy/templates/service.yaml index 39dea79..25d1364 100644 --- a/deploy/templates/service.yaml +++ b/deploy/templates/service.yaml @@ -1,9 +1,9 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "biothings-annotator.fullname" . }} + name: {{ include "node-normalization.fullname" . }} labels: - {{- include "biothings-annotator.labels" . | nindent 4 }} + {{- include "node-normalization.labels" . | nindent 4 }} spec: type: {{ .Values.service.type }} ports: @@ -12,4 +12,4 @@ spec: protocol: TCP name: http selector: - {{- include "biothings-annotator.selectorLabels" . | nindent 4 }} + {{- include "node-normalization.selectorLabels" . | nindent 4 }} diff --git a/deploy/values.yaml b/deploy/values.yaml index aea4391..43eb3db 100644 --- a/deploy/values.yaml +++ b/deploy/values.yaml @@ -1,7 +1,10 @@ -# Default values for biothings-annotator. +# Default values for node-normalization. # This is a YAML-formatted file. # Declare variables to be passed into your templates. +# Override the full deployment name to prevent release name prefix +fullnameOverride: "node-normalization" + deployment: replicaCount: 1 strategy: @@ -14,15 +17,15 @@ deployment: # A new pod will only be created after an old pod is taken down. maxSurge: 0 image: - repository: 853771734544.dkr.ecr.us-east-1.amazonaws.com/translator-bte-pending-api-annotator + repository: 853771734544.dkr.ecr.us-east-1.amazonaws.com/translator-nodenormalization-api pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. tag: DOCKER_VERSION_VALUE containers: - name: biothingsannotator + name: node-normalization es_host: ES_HOST_VALUE - port: 9000 + port: 8000 env: OPENTELEMETRY_ENABLED_VALUE: True @@ -44,13 +47,13 @@ ingress: alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' alb.ingress.kubernetes.io/success-codes: '200' alb.ingress.kubernetes.io/load-balancer-attributes: idle_timeout.timeout_seconds=600 - host: BTEPA_HOSTNAME_VALUE + host: NODE_NORM_HOSTNAME_VALUE ncats: labels: gov.nih.ncats.appenv: appenv - gov.nih.ncats.appname: bte-biothings-annotator - gov.nih.ncats.appentry: BTEPA_HOSTNAME_VALUE + gov.nih.ncats.appname: bte-node-normalization + gov.nih.ncats.appentry: NODE_NORM_HOSTNAME_VALUE gov.nih.ncats.appentrytype: https gov.nih.ncats.appentryport: tcp gov.nih.ncats.appconnnum: '1' diff --git a/docker/Dockerfile b/docker/Dockerfile index 39b690c..45c5bee 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.12 AS builder -ARG NODENORM_REPO=https://github.com/biothings/NodeNormalizationAPI.git +ARG NODENORM_REPO=https://github.com/chevvak2/NodeNormalizationAPI.git ARG NODENORM_BRANCH=main WORKDIR /build/nodenorm @@ -16,22 +16,36 @@ RUN apt update -q -y && apt install -y --no-install-recommends git supervisor cu RUN useradd -m nodenorm && usermod -aG sudo nodenorm USER nodenorm WORKDIR /home/nodenorm/ +# Set PyStow home directory to a writable location +ENV PYSTOW_HOME=/home/nodenorm/.pystow +RUN mkdir -p /home/nodenorm/.pystow RUN python -m venv /home/nodenorm/venv COPY --from=builder --chown=nodenorm:nodenorm /build/wheels /home/nodenorm/wheels RUN /home/nodenorm/venv/bin/pip install /home/nodenorm/wheels/*.whl && rm -rf /home/nodenorm/wheels +# Apply patches to fix NodeNormalization compatibility issues +COPY --chown=nodenorm:nodenorm patches/health.py /home/nodenorm/venv/lib/python3.12/site-packages/nodenorm/handlers/health.py + WORKDIR /home/nodenorm/configuration COPY --from=builder --chown=nodenorm:nodenorm /build/nodenorm/src/nodenorm/config/config.default.json /home/nodenorm/configuration/default.json COPY --from=builder --chown=nodenorm:nodenorm /build/nodenorm/version.txt /home/nodenorm/configuration/version.txt -COPY --chown=nodenorm:nodenorm docker/configuration/supervisord.conf /etc/supervisor/supervisord.conf -COPY --chown=nodenorm:nodenorm docker/configuration/Caddyfile /etc/caddy/Caddyfile +# Override with corrected configuration +COPY --chown=nodenorm:nodenorm configuration/config.json /home/nodenorm/configuration/config.json -COPY --from=caddy_builder --chown=nodenorm:nodenorm /usr/bin/caddy /usr/bin/caddy +# Switch back to root for system-level operations +USER root +COPY configuration/supervisord.conf /etc/supervisor/supervisord.conf +COPY configuration/Caddyfile /etc/caddy/Caddyfile +COPY --from=caddy_builder /usr/bin/caddy /usr/bin/caddy +# Switch back to nodenorm user for runtime +USER nodenorm STOPSIGNAL SIGINT -EXPOSE 9000 +EXPOSE 8000 +# Run supervisord as root to manage processes +USER root ENTRYPOINT ["supervisord"] CMD ["-c", "/etc/supervisor/supervisord.conf"] diff --git a/docker/configuration/Caddyfile b/docker/configuration/Caddyfile index d490b99..73599df 100644 --- a/docker/configuration/Caddyfile +++ b/docker/configuration/Caddyfile @@ -1,4 +1,4 @@ -:9000 { +:8000 { reverse_proxy localhost:9001 encode zstd gzip log { diff --git a/docker/configuration/config.json b/docker/configuration/config.json new file mode 100644 index 0000000..c46ab5f --- /dev/null +++ b/docker/configuration/config.json @@ -0,0 +1,32 @@ +{ + "webserver": { + "HOST": "localhost", + "PORT": 8000, + "ENABLE_CURL_CLIENT": true, + "SETTINGS": { + "debug": true, + "autoreload": true + } + }, + "api": { + "API_PREFIX": "nodenorm", + "API_VERSION": "" + }, + "elasticsearch": { + "ES_HOST": "http://elasticsearch.es-core-components.svc.cluster.local:9200", + "ES_INDEX": "nodenorm", + "ES_ALIAS": "nodenorm", + "ES_DOC_TYPE": "node", + "ES_INDICES": {}, + "ES_ARGS": { + "sniff": false, + "request_timeout": 60 + } + }, + "telemetry": { + "OPENTELEMETRY_ENABLED": false, + "OPENTELEMETRY_SERVICE_NAME": "NodeNorm", + "OPENTELEMETRY_JAEGER_HOST": "http://localhost", + "OPENTELEMETRY_JAEGER_PORT": 6831 + } +} \ No newline at end of file diff --git a/docker/configuration/supervisord.conf b/docker/configuration/supervisord.conf index 97aaad9..d50dad9 100644 --- a/docker/configuration/supervisord.conf +++ b/docker/configuration/supervisord.conf @@ -17,11 +17,13 @@ stderr_logfile_maxbytes=0 stderr_logfile_backups=0 [program:python_app] -command=/home/nodenorm/venv/bin/python -m nodenorm --port=9001 +command=/home/nodenorm/venv/bin/python -m nodenorm --conf=/home/nodenorm/configuration/config.json --port=9001 +user=nodenorm +environment=PYSTOW_HOME=/home/nodenorm/.pystow autorestart=true stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stdout_logfile_backups=0 stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 -stderr_logfile_backups=0 +stderr_logfile_backups=0 \ No newline at end of file diff --git a/docker/patches/health.py b/docker/patches/health.py new file mode 100644 index 0000000..f8df770 --- /dev/null +++ b/docker/patches/health.py @@ -0,0 +1,70 @@ +from urllib.parse import urlparse + +from elasticsearch import AsyncElasticsearch + +from biothings.web.handlers import BaseHandler + +from nodenorm.biolink import BIOLINK_MODEL_VERSION + + +class NodeNormHealthHandler(BaseHandler): + """ + Important Endpoints + * /_cat/nodes + + Patched version to handle missing metadata gracefully and fix ES API compatibility + """ + + name = "health" + + async def get(self): + async_client: AsyncElasticsearch = self.biothings.elasticsearch.async_client + search_indices = self.biothings.elasticsearch.indices + + try: + # Fixed: Use keyword argument for ES 8.x compatibility + biothings_metadata = await async_client.indices.get(index=search_indices) + + # Use default babel version since metadata access may not be available + babel_version = "1.9" # Default fallback version + babel_markdown = f"https://github.com/ncatstranslator/Babel/blob/master/releases/{babel_version}.md" + + try: + attributes = [ + "name", + "cpu", + "disk.avail", + "disk.total", + "disk.used", + "disk.used_percent", + "heap.current", + "heap.max", + "load_1m", + "load_5m", + "load_15m", + "uptime,version", + ] + h_string = ",".join(attributes) + cat_nodes_response = await async_client.cat.nodes(format="json", h=h_string) + nodes_status = {node["name"]: node for node in cat_nodes_response} + nodes = {"elasticsearch": {"nodes": nodes_status}} + except Exception: + # Fallback if ES cluster info fails + nodes = {"elasticsearch": {"status": "connected"}} + + status_response = { + "status": "running", + "babel_version": babel_version, + "babel_version_url": babel_markdown, + "biolink_model_toolkit_version": BIOLINK_MODEL_VERSION, + **nodes, + } + + except Exception as e: + status_response = { + "status": "error", + "error": str(e), + "babel_version": "unknown" + } + + self.finish(status_response) \ No newline at end of file