diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5a89f46 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,51 @@ +# Python +__pycache__/ +*.py[cod] +*.pyo +*.pyd + +# Virtual environments +env/ +venv/ +.venv/ + +# Editor & OS files +.DS_Store +.vscode/ +.idea/ + +# Git +.git +.gitignore + +# Build/dist +build/ +dist/ +*.egg-info + +# Static/local settings +media/ +static/ +local_settings.py +*.sqlite3 +*.log + +# Node +node_modules/ + +# Docker +Dockerfile +docker-compose*.yml + +# Data directory +data/ + +# Backups +backups/ + +# Vagrant +Vagrantfile +.vagrant/ + +# SQL files +*.sql \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3e28fd5..19dae96 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ landmapper/app/testing_files landmapper/app/static/landmapper/shapefiles/ backups/* src/* +.env +*.sql \ No newline at end of file diff --git a/README.md b/README.md index 20b1915..082ae3a 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,40 @@ Simple Woodland Discovery For Land Owners ## Installation -Under construction - Coming Soon! + +### Docker + +To run locally with Docker follow these steps (for Oregon): + +Pre-requisites: +- Docker +- Docker compose +1. Clone the repo +``` +git clone https://github.com/Ecotrust/landmapper.git +``` +2. Check out the `dockerize` branch +``` +git checkout dockerize +``` +3. Clone dependent repos in a location next to the landmapper repo +``` +git clone https://github.com/Ecotrust/madrona-features.git +git clone https://github.com/Ecotrust/madrona-manipulators.git +git clone https://github.com/Ecotrust/p97-nursery.git +``` +4. Add the following files to the `docker/` directory. Ask an Ecotrust Software team member for these files: +- OR_TAXLOTS.sql +- OR_POPULATION_2021.sql +- FOREST_TYPES_2021.sql +- OR_SOIL_2021.sql + +5. Start the docker containers using docker compose +``` +docker compose up -d --build +``` + +6. Application should be live at `https://localhost:8000/` --- diff --git a/docker/.env.example b/docker/.env.example new file mode 100644 index 0000000..60b017e --- /dev/null +++ b/docker/.env.example @@ -0,0 +1,16 @@ +ALLOWED_HOSTS=localhost,127.0.0.1,[::1] +DEBUG=1 +POSTGRES_ENGINE=django.contrib.gis.db.backends.postgis +POSTGRES_HOST=db +POSTGRES_PORT=5432 +POSTGRES_DB=landmapper +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres_password_change_me +SECRET_KEY=foobar +MAPBOX_TOKEN= +GOOGLE_API_KEY= +GEOAPIFY_API_KEY= +RECAPTCHA_PUBLIC_KEY= +RECAPTCHA_PRIVATE_KEY= +ARCGIS_DEVELOPER_KEY= +GOOGLE_ANALYTICS_KEY= \ No newline at end of file diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 0000000..7158e0f --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,88 @@ +services: + db: + container_name: db + image: postgis/postgis:18-3.6 + restart: unless-stopped + platform: linux/amd64 + environment: + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + volumes: + - landmapper_db_data:/var/lib/postgresql + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB} -h localhost -p ${POSTGRES_PORT}"] + interval: 10s + timeout: 5s + retries: 5 + redis: + container_name: redis + image: redis:alpine + ports: + - "6379:6379" + landmapper: + container_name: landmapper + build: + context: ../landmapper/ + dockerfile: ../landmapper/Dockerfile + restart: unless-stopped + env_file: + - path: .env + required: true + ports: + - "8000:8000" + healthcheck: + test: ["CMD-SHELL", "python manage.py check --deploy 2>/dev/null || python -c 'import urllib.request; urllib.request.urlopen(\"http://localhost:8000/\")' 2>/dev/null"] + interval: 15s + timeout: 10s + retries: 5 + start_period: 30s + volumes: + - type: bind + source: ../p97-nursery + target: /usr/local/apps/landmapper/apps/p97-nursery + bind: + create_host_path: false + - type: bind + source: ../madrona-features + target: /usr/local/apps/landmapper/apps/madrona-features + bind: + create_host_path: false + - type: bind + source: ../madrona-manipulators + target: /usr/local/apps/landmapper/apps/madrona-manipulators + bind: + create_host_path: false + - static_volume:/usr/src/app/static + - media_volume:/usr/src/app/media + - ../landmapper:/usr/src/app + - ./OR_TAXLOTS.sql:/tmp/OR_TAXLOTS.sql:ro + - ./OR_POPULATION_2021.sql:/tmp/OR_POPULATION_2021.sql:ro + - ./FOREST_TYPES_2021.sql:/tmp/FOREST_TYPES_2021.sql:ro + - ./OR_SOIL_2021.sql:/tmp/OR_SOIL_2021.sql:ro + depends_on: + db: + condition: service_healthy + redis: + condition: service_started + environment: + ALLOWED_HOSTS: ${ALLOWED_HOSTS} + DEBUG: ${DEBUG} + POSTGRES_ENGINE: ${POSTGRES_ENGINE} + POSTGRES_HOST: ${POSTGRES_HOST} + POSTGRES_PORT: ${POSTGRES_PORT} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + SECRET_KEY: ${SECRET_KEY} +volumes: + landmapper_db_data: + static_volume: + media_volume: + + + + + diff --git a/landmapper/Dockerfile b/landmapper/Dockerfile new file mode 100644 index 0000000..46a108c --- /dev/null +++ b/landmapper/Dockerfile @@ -0,0 +1,65 @@ +FROM python:3.10-slim + +# Prevent Python from writing .pyc files and enable unbuffered stdout/stderr +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +ENV PIP_NO_CACHE_DIR=1 + +# from doc: +# sudo apt install python3-pip postgresql postgresql-contrib postgresql-server-dev-14 postgis python3-gdal libgdal-dev redis-server -y +# sudo apt install pdftk + +# Install system dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client \ + gdal-bin \ + libgdal-dev \ + libgeos-dev \ + pdftk \ + git \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +ENV CPLUS_INCLUDE_PATH=/usr/include/gdal +ENV C_INCLUDE_PATH=/usr/include/gdal + +ENV PYTHONPATH="/usr/local/apps/landmapper/apps/madrona-features:/usr/local/apps/landmapper/apps/p97-nursery:/usr/local/apps/landmapper/apps/madrona-manipulators:/usr/src/app" + +# Set working directory +WORKDIR /usr/src/app + +# Copy requirements first (cache pip install step when dependencies don't change) +COPY requirements.txt /usr/src/app/ + +# COPY src/madrona-features /usr/local/apps/madrona-features +# COPY src/madrona-manipulators /usr/local/apps/madrona-manipulators +# COPY src/p97-nursery /usr/local/apps/p97-nursery + +# Upgrade pip and install Python dependencies +RUN pip install --upgrade pip \ + && pip install -r requirements.txt + +# After installing gdal-bin and libgdal-dev, install matching version of GDAL Python bindings. This is required for Django to use GDAL. +RUN GDAL_VERSION=`gdal-config --version` && \ + pip install GDAL==$GDAL_VERSION + +# Copy the application code +COPY . /usr/src/app + +# Copy and make entrypoint executable. The repository contains `docker/entrypoint.sh` +# which runs collectstatic, migrations and launches uWSGI. +COPY entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh + +# Create directory for PDF reports +RUN rm -rf /usr/src/app/app/static/landmapper/report_pdf \ + && mkdir -p /usr/src/app/app/static/landmapper/report_pdf + +# Expose the port the app runs on (entrypoint starts django development server or uWSGI on 8000) +EXPOSE 8000 + +# Default settings module (can be overridden at runtime) +ENV DJANGO_SETTINGS_MODULE=landmapper.settings + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] \ No newline at end of file diff --git a/landmapper/app/migrations/0001_initial.py b/landmapper/app/migrations/0001_initial.py index dde6813..fe3f480 100644 --- a/landmapper/app/migrations/0001_initial.py +++ b/landmapper/app/migrations/0001_initial.py @@ -1,11 +1,11 @@ # Generated by Django 4.2b1 on 2023-03-17 00:18 import app.models -import ckeditor_uploader.fields from django.conf import settings import django.contrib.gis.db.models.fields from django.db import migrations, models import django.db.models.deletion +import tinymce.models class Migration(migrations.Migration): @@ -38,7 +38,7 @@ class Migration(migrations.Migration): ('order', models.SmallIntegerField(default=10)), ('staff_only', models.BooleanField(default=False)), ('header', models.CharField(blank=True, default=None, max_length=255, null=True, verbose_name='Popup Header')), - ('content', ckeditor_uploader.fields.RichTextUploadingField(blank=True, default=None, null=True, verbose_name='Popup Body')), + ('content', tinymce.models.HTMLField(blank=True, default=None, null=True, verbose_name='Popup Body')), ], ), migrations.CreateModel( diff --git a/landmapper/app/static/landmapper/report_pdf b/landmapper/app/static/landmapper/report_pdf deleted file mode 120000 index eddb1cf..0000000 --- a/landmapper/app/static/landmapper/report_pdf +++ /dev/null @@ -1 +0,0 @@ -/usr/local/apps/pdf/ \ No newline at end of file diff --git a/landmapper/entrypoint.sh b/landmapper/entrypoint.sh new file mode 100644 index 0000000..e33c684 --- /dev/null +++ b/landmapper/entrypoint.sh @@ -0,0 +1,114 @@ +#!/bin/sh + +# Exit on errors +set -e + +# If a SQL_HOST is provided, wait for Postgres to become available before running +# migrations. This prevents race conditions when using docker-compose where the +# web container starts before the DB is ready. +if [ -n "$SQL_HOST" ]; then + echo "Waiting for database at ${SQL_HOST}:${SQL_PORT:-5432}..." + # pg_isready is available after installing postgresql-client in the image + until pg_isready -h "$SQL_HOST" -p "${SQL_PORT:-5432}" >/dev/null 2>&1; do + echo "Postgres is unavailable - sleeping" + sleep 1 + done + echo "Postgres is up" +fi + +echo "Collecting static files..." +python manage.py collectstatic --noinput +echo "Applying database migrations..." +python manage.py migrate --noinput + +echo "Configuring Site object..." +python manage.py shell << 'EOF' +from django.contrib.sites.models import Site +from django.conf import settings + +site_id = getattr(settings, 'SITE_ID', 1) +domain = getattr(settings, 'ALLOWED_HOSTS', ['localhost'])[0] +if domain == '*': + domain = 'localhost' + +site, created = Site.objects.get_or_create( + id=site_id, + defaults={'domain': domain, 'name': 'Landmapper'} +) +if not created and (site.domain != domain or site.name != 'Landmapper'): + site.domain = domain + site.name = 'Landmapper' + site.save() + print(f"Updated Site: {site.name} ({site.domain})") +else: + print(f"Site configured: {site.name} ({site.domain})") +EOF + +# Load default lookups only if no lookups exist. Use MenuPage as the check. +echo "Checking for existing initial data..." +if [ "$(python manage.py shell -c 'from app.models import MenuPage; print(MenuPage.objects.count())' 2>/dev/null | tail -1)" = "0" ]; then + echo "No initial data found, loading default initial data fixture..." + python manage.py loaddata app/fixtures/initial_data.json +else + echo "Initial data already exists, skipping fixture load." +fi + +# Load geodata SQL dumps if tables are empty +echo "Checking for existing taxlot data..." +if [ "$(python manage.py shell -c 'from app.models import Taxlot; print(Taxlot.objects.count())' 2>/dev/null | tail -1)" = "0" ]; then + echo "No taxlot data found, loading SQL dumps..." + + # Check if SQL files exist and load them + TAXLOTS_FILE="/tmp/OR_TAXLOTS.sql" + if [ -f "$TAXLOTS_FILE" ]; then + echo "Loading taxlots from $TAXLOTS_FILE..." + PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$POSTGRES_HOST" -U "$POSTGRES_USER" -d "$POSTGRES_DB" -f "$TAXLOTS_FILE" + else + echo "WARNING: $TAXLOTS_FILE not found" + fi +else + echo "Taxlot data already exists, skipping SQL dump load." +fi + +echo "Checking for population data..." +if [ "$(python manage.py shell -c 'from app.models import PopulationPoint; print(PopulationPoint.objects.count())' 2>/dev/null | tail -1)" = "0" ]; then + echo "No population data found, loading SQL dump..." + + POPULATION_FILE="/tmp/OR_POPULATION_2021.sql" + if [ -f "$POPULATION_FILE" ]; then + echo "Loading population data from $POPULATION_FILE..." + PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$POSTGRES_HOST" -U "$POSTGRES_USER" -d "$POSTGRES_DB" -f "$POPULATION_FILE" + else + echo "WARNING: $POPULATION_FILE not found" + fi +else + echo "Population data already exists, skipping SQL dump load." +fi + +echo "Checking for Forest Types data..." +if [ "$(python manage.py shell -c 'from app.models import ForestType; print(ForestType.objects.count())' 2>/dev/null | tail -1)" = "0" ]; then + echo "No Forest Types data found, loading SQL dump..." + FOREST_FILE="/tmp/FOREST_TYPES_2021.sql" + if [ -f "$FOREST_FILE" ]; then + echo "Loading Forest Types data from $FOREST_FILE..." + PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$POSTGRES_HOST" -U "$POSTGRES_USER" -d "$POSTGRES_DB" -f "$FOREST_FILE" + else + echo "WARNING: $FOREST_FILE not found" + fi + +fi + +echo "Checking for Soil Types data..." +if [ "$(python manage.py shell -c 'from app.models import SoilType; print(SoilType.objects.count())' 2>/dev/null | tail -1)" = "0" ]; then + echo "No Soil Types data found, loading SQL dump..." + SOIL_FILE="/tmp/OR_SOIL_2021.sql" + if [ -f "$SOIL_FILE" ]; then + echo "Loading Soil Types data from $SOIL_FILE..." + PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$POSTGRES_HOST" -U "$POSTGRES_USER" -d "$POSTGRES_DB" -f "$SOIL_FILE" + else + echo "WARNING: $SOIL_FILE not found" + fi +fi + +echo "Starting python development server on :8000" +python manage.py runserver 0.0.0.0:8000 \ No newline at end of file diff --git a/landmapper/landmapper/settings.py b/landmapper/landmapper/settings.py index 24ef8e3..61f3a46 100644 --- a/landmapper/landmapper/settings.py +++ b/landmapper/landmapper/settings.py @@ -23,14 +23,10 @@ # See https://docs.djangoproject.com/en/dev/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'django-insecure-%v#dz7)z_dt@_*8swh3=-+8w4mg8*w2^wb$zofhqj*-p2b4jb_' +SECRET_KEY = os.environ.get("SECRET_KEY", default="set in .env file") # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = [] - -SITE_ID = 1 +DEBUG = bool(int(os.environ.get("DEBUG", default=0))) # Application definition @@ -104,18 +100,18 @@ # Database # https://docs.djangoproject.com/en/dev/ref/settings/#databases -DATABASES = { - 'default': { - 'ENGINE': "django.contrib.gis.db.backends.postgis", - 'NAME': "landmapper", - 'USER': 'postgres', - } -} +# DATABASES = { +# 'default': { +# 'ENGINE': "django.contrib.gis.db.backends.postgis", +# 'NAME': "landmapper", +# 'USER': 'postgres', +# } +# } CACHES = { 'default': { 'BACKEND': 'django_redis.cache.RedisCache', - 'LOCATION': 'redis://127.0.0.1:6379/1', + 'LOCATION': 'redis://redis:6379/1', 'OPTIONS': { 'CLIENT_CLASS': 'django_redis.client.DefaultClient' } @@ -162,7 +158,7 @@ LANGUAGE_CODE = 'en-us' -TIME_ZONE = 'US/Pacific' +TIME_ZONE = 'America/Los_Angeles' USE_I18N = True @@ -1520,7 +1516,8 @@ ########################################### FALLBACK_PROPERTY_REPORT_PDF_TEMPLATE = APP_DIR + '/pdf_templates/LM_Form.pdf' FALLBACK_PROPERTY_REPORT_PDF_TEMPLATE_SANS_FOREST_TYPES = APP_DIR + '/pdf_templates/LM_Form_sans_forests.pdf' -PROPERTY_REPORT_PDF_DIR = APP_DIR + '/static/landmapper/report_pdf/' +# PROPERTY_REPORT_PDF_DIR = APP_DIR + '/static/landmapper/report_pdf/' +PROPERTY_REPORT_PDF_DIR= '/tmp/' # For creating individual PDF exports PDF_PAGE_LOOKUP = { @@ -1553,10 +1550,50 @@ ########## OVERRIDES ################ STUDY_REGION_ID = 'OR' # Default to OR -try: - from .local_settings import * -except Exception as e: - pass +GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY", default="set in .env file") +MAPBOX_TOKEN = os.environ.get("MAPBOX_TOKEN", default="set in .env file") +GEOAPIFY_API_KEY = os.environ.get("GEOAPIFY_API_KEY", default="set in .env file") +RECAPTCHA_PUBLIC_KEY = os.environ.get("RECAPTCHA_PUBLIC_KEY", default="set in .env file") +RECAPTCHA_PRIVATE_KEY = os.environ.get("RECAPTCHA_PRIVATE_KEY", default="set in .env file") +ARCGIS_DEVELOPER_KEY = os.environ.get("ARCGIS_DEVELOPER_KEY", default="set in .env file") +GOOGLE_ANALYTICS_KEY = os.environ.get("GOOGLE_ANALYTICS_KEY", default="set in .env file") + +EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' + +LOG_FILE = "/usr/local/apps/log.txt" + +STATIC_ROOT = '/usr/local/apps/landmapper/landmapper/app/static' + +STUDY_REGION_ID = 'OR' +STREAMS_SOURCE = "ECOTRUST_{}".format(STUDY_REGION_ID) +SOIL_SOURCE = "ECOTRUST_{}".format(STUDY_REGION_ID) +TAXLOTS_SOURCE = "ECOTRUST_{}".format(STUDY_REGION_ID) +FOREST_TYPES_SOURCE = "ECOTRUST_{}".format(STUDY_REGION_ID) +FOREST_SIZE_SOURCE = "ECOTRUST_{}".format(STUDY_REGION_ID) +FOREST_DENSITY_SOURCE = "ECOTRUST_{}".format(STUDY_REGION_ID) +FOREST_CANOPY_SOURCE = "ECOTRUST_{}".format(STUDY_REGION_ID) + +TAXLOT_IMPORT_TIMESTAMP = 1702590468 +ENFORCE_TIMESTAMP = True + +DATABASES = { + 'default': { + 'ENGINE': 'django.contrib.gis.db.backends.postgis', + 'NAME': os.environ.get("POSTGRES_DB", default="landmapper"), + 'USER': os.environ.get("POSTGRES_USER", default="postgres"), + 'PASSWORD': os.environ.get("POSTGRES_PASSWORD", default="landmapper_password"), + 'HOST': os.environ.get("POSTGRES_HOST", default="db"), + 'PORT': os.environ.get("POSTGRES_PORT", default="5432"), + } +} + +ALLOWED_HOSTS = [] +ALLOWED_HOSTS_ENV = os.environ.get("ALLOWED_HOSTS") +if ALLOWED_HOSTS_ENV: + ALLOWED_HOSTS.extend(ALLOWED_HOSTS_ENV.split(",")) + +SITE_ID = 1 + STUDY_REGION = STUDY_REGIONS[STUDY_REGION_ID] # Test PDF template diff --git a/requirements.txt b/landmapper/requirements.txt similarity index 53% rename from requirements.txt rename to landmapper/requirements.txt index 904140e..a48f9c5 100644 --- a/requirements.txt +++ b/landmapper/requirements.txt @@ -6,12 +6,13 @@ django-flatblocks django-redis # Django-Registration 3.4 does not appear to be sending password reset emails... django-registration<=3.3 -# -e git+https://github.com/Ecotrust/madrona-features.git@main#egg=madrona_features --e /usr/local/apps/madrona-features -# -e git+https://github.com/Ecotrust/p97-nursery.git@main#egg=p97-nursery --e /usr/local/apps/p97-nursery -# -e git+https://github.com/Ecotrust/madrona-manipulators.git@main#egg=madrona_manipulators --e /usr/local/apps/madrona-manipulators +# TODO: figure out if these are needed here, or if having them in the bound volumes is sufficient +-e git+https://github.com/Ecotrust/madrona-features.git@main#egg=madrona_features +# -e /usr/local/apps/landmapper/apps/madrona-features +-e git+https://github.com/Ecotrust/p97-nursery.git@main#egg=p97-nursery +# -e /usr/local/apps/landmapper/apps/p97-nursery +-e git+https://github.com/Ecotrust/madrona-manipulators.git@main#egg=madrona_manipulators +# -e /usr/local/apps/landmapper/apps/madrona-manipulators django-tinymce geopy ipdb @@ -38,17 +39,12 @@ geocoder geopandas imageio scikit-image -pygeos +# pygeos matplotlib # matplotlib_scalebar pypdf2<3.0 PdfJinja -######## -# Georefrence PDFs -######## -GDAL<3.4.2 - ######## # Exporting ########