Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions python/PiFinder/catalog_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,10 +77,14 @@ def __init__(
desc: str,
max_sequence: int = 0,
sort=catalog_base_sequence_sort,
name: Optional[str] = None,
):
self.catalog_code = catalog_code
self.max_sequence = max_sequence
self.desc = desc
# Readable catalog name (e.g. "Collinder" for code "Col"); may be None
# for virtual catalogs or legacy rows, where the code is used instead.
self.name = name
self.sort = sort
self.__objects: List = []
self.id_to_pos: Dict[int, int] = {}
Expand Down
2 changes: 1 addition & 1 deletion python/PiFinder/catalog_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

# Bump when CompositeObject shape, _create_full_composite_object output, or
# the pickled payload structure changes.
CACHE_VERSION = 1
CACHE_VERSION = 2

CACHE_DIR = data_dir / "cache" / "catalogs"
PICKLE_PATH = CACHE_DIR / "composite_objects.pkl"
Expand Down
2 changes: 1 addition & 1 deletion python/PiFinder/catalog_imports/bright_stars_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ def load_bright_stars():
catalog = "Str"
conn, _ = objects_db.get_conn_cursor()
delete_catalog_from_database(catalog)
insert_catalog(catalog, Path(utils.astro_data_dir, "Str.desc"))
insert_catalog(catalog, Path(utils.astro_data_dir, "Str.desc"), "Bright Stars")

bstr = Path(utils.astro_data_dir, "bright_stars.csv")

Expand Down
2 changes: 1 addition & 1 deletion python/PiFinder/catalog_imports/caldwell_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ def load_caldwell():
catalog = "C"
conn, _ = objects_db.get_conn_cursor()
delete_catalog_from_database(catalog)
insert_catalog(catalog, Path(utils.astro_data_dir, "caldwell.desc"))
insert_catalog(catalog, Path(utils.astro_data_dir, "caldwell.desc"), "Caldwell")
data = Path(utils.astro_data_dir, "caldwell.dat")

# Prepare objects for batch insertion
Expand Down
10 changes: 7 additions & 3 deletions python/PiFinder/catalog_imports/catalog_import_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -242,11 +242,15 @@ def delete_catalog_from_database(catalog_code: str):
conn.commit()


def insert_catalog(catalog_name, description_path):
"""Insert a catalog description into the database"""
def insert_catalog(catalog_name, description_path, display_name=None):
"""Insert a catalog description into the database.

``display_name`` is the readable catalog name (e.g. "Collinder" for code
"Col") surfaced in the UI; it is stored alongside the code.
"""
with open(description_path, "r") as desc:
description = "".join(desc.readlines())
objects_db.insert_catalog(catalog_name, -1, description)
objects_db.insert_catalog(catalog_name, -1, description, display_name)


def insert_catalog_max_sequence(catalog_name):
Expand Down
2 changes: 1 addition & 1 deletion python/PiFinder/catalog_imports/harris_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -405,7 +405,7 @@ def load_harris() -> None:
delete_catalog_from_database(catalog)

# Path to file that describes the catalog
insert_catalog(catalog, Path(utils.astro_data_dir) / "harris/ReadMe")
insert_catalog(catalog, Path(utils.astro_data_dir) / "harris/ReadMe", "Harris")

# Read the catalog data
data = read_harris_catalog(data_path)
Expand Down
2 changes: 1 addition & 1 deletion python/PiFinder/catalog_imports/herschel_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def load_herschel400():
catalog = "H"
conn, _ = objects_db.get_conn_cursor()
delete_catalog_from_database(catalog)
insert_catalog(catalog, Path(utils.astro_data_dir, "herschel400.desc"))
insert_catalog(catalog, Path(utils.astro_data_dir, "herschel400.desc"), "Herschel")

hcat = Path(utils.astro_data_dir, "herschel400.tsv")
sequence = 0
Expand Down
2 changes: 1 addition & 1 deletion python/PiFinder/catalog_imports/lynga_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -549,7 +549,7 @@ def load_lynga() -> None:
delete_catalog_from_database(catalog)

# Path to file that describes the catalog
insert_catalog(catalog, Path(utils.astro_data_dir) / "lynga/ReadMe")
insert_catalog(catalog, Path(utils.astro_data_dir) / "lynga/ReadMe", "Lynga")

# Read the catalog data
data = read_lynga_catalog(data_path)
Expand Down
6 changes: 3 additions & 3 deletions python/PiFinder/catalog_imports/sac_loaders.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ def load_sac_asterisms():
catalog = "SaA"
conn, _ = objects_db.get_conn_cursor()
delete_catalog_from_database(catalog)
insert_catalog(catalog, Path(utils.astro_data_dir, "sac.desc"))
insert_catalog(catalog, Path(utils.astro_data_dir, "sac.desc"), "SAC Asterisms")

saca = Path(utils.astro_data_dir, "SAC_Asterisms_Ver32_Fence.txt")
sequence = 0
Expand Down Expand Up @@ -140,7 +140,7 @@ def load_sac_multistars():
conn, _ = objects_db.get_conn_cursor()
delete_catalog_from_database(catalog)
sam_path = Path(utils.astro_data_dir, "SAC_Multistars_Ver40")
insert_catalog(catalog, sam_path / "sacm.desc")
insert_catalog(catalog, sam_path / "sacm.desc", "SAC Doubles")
saca = sam_path / "SAC_DBL40_Fence.txt"
sequence = 0

Expand Down Expand Up @@ -245,7 +245,7 @@ def load_sac_redstars():
delete_catalog_from_database(catalog)

sam_path = Path(utils.astro_data_dir, "SAC_RedStars_Ver20")
insert_catalog(catalog, sam_path / "sacr.desc")
insert_catalog(catalog, sam_path / "sacr.desc", "SAC Red Stars")
sac = sam_path / "SAC_RedStars_ver20_FENCE.TXT"
sequence = 0

Expand Down
22 changes: 13 additions & 9 deletions python/PiFinder/catalog_imports/specialized_loaders.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,11 @@ def load_egc():
conn, _db_c = objects_db.get_conn_cursor()
delete_catalog_from_database(catalog)

insert_catalog(catalog, Path(utils.astro_data_dir, "EGC.desc"))
insert_catalog(
catalog,
Path(utils.astro_data_dir, "EGC.desc"),
"Extragalactic Globular Clusters",
)
egc = Path(utils.astro_data_dir, "EGC.tsv")

# Create shared ObjectFinder to avoid recreating for each object
Expand Down Expand Up @@ -102,7 +106,7 @@ def load_collinder():
catalog = "Col"
conn, _db_c = objects_db.get_conn_cursor()
delete_catalog_from_database(catalog)
insert_catalog(catalog, Path(utils.astro_data_dir, "collinder.desc"))
insert_catalog(catalog, Path(utils.astro_data_dir, "collinder.desc"), "Collinder")
coll = Path(utils.astro_data_dir, "collinder.txt")
Collinder = namedtuple(
"Collinder",
Expand Down Expand Up @@ -229,7 +233,7 @@ def load_taas200():
catalog = "Ta2"
conn, _ = objects_db.get_conn_cursor()
delete_catalog_from_database(catalog)
insert_catalog(catalog, Path(utils.astro_data_dir, "taas200.desc"))
insert_catalog(catalog, Path(utils.astro_data_dir, "taas200.desc"), "TAAS 200")
data = Path(utils.astro_data_dir, "TAAS_200.csv")
sequence = 0

Expand Down Expand Up @@ -341,7 +345,7 @@ def load_rasc_double_Stars():
conn, _ = objects_db.get_conn_cursor()
path = Path(utils.astro_data_dir, "RASC_DoubleStars")
delete_catalog_from_database(catalog)
insert_catalog(catalog, path / "rasc_ds.desc")
insert_catalog(catalog, path / "rasc_ds.desc", "RASC Double Stars")
data = path / "rasc_double_stars.csv"

# Create shared ObjectFinder to avoid recreating for each object
Expand Down Expand Up @@ -410,7 +414,7 @@ def load_barnard():
conn, _ = objects_db.get_conn_cursor()
path = Path(utils.astro_data_dir, "barnard")
delete_catalog_from_database(catalog)
insert_catalog(catalog, path / "barnard.desc")
insert_catalog(catalog, path / "barnard.desc", "Barnard")
data = path / "barnard.dat"
data_notes = path / "notes.dat"
barn_dict = defaultdict(str)
Expand Down Expand Up @@ -487,7 +491,7 @@ def load_sharpless():
conn, _ = objects_db.get_conn_cursor()
path = Path(utils.astro_data_dir, "sharpless")
delete_catalog_from_database(catalog)
insert_catalog(catalog, path / "sharpless.desc")
insert_catalog(catalog, path / "sharpless.desc", "Sharpless")
data = path / "catalog.dat"
akas = path / "akas.csv"
descriptions = path / "galaxymap_descriptions.csv"
Expand Down Expand Up @@ -597,7 +601,7 @@ def load_arp():
catalog = "Arp"
path = Path(utils.astro_data_dir, "arp")
delete_catalog_from_database(catalog)
insert_catalog(catalog, path / "arp.desc")
insert_catalog(catalog, path / "arp.desc", "Arp")
comments = path / "arp_comments.csv"

def expand(name):
Expand Down Expand Up @@ -704,7 +708,7 @@ def load_tlk_90_vars():
conn, _ = objects_db.get_conn_cursor()
path = Path(utils.astro_data_dir, "variables/TLK_90_vars")
delete_catalog_from_database(catalog)
insert_catalog(catalog, path / "v90.desc")
insert_catalog(catalog, path / "v90.desc", "TLK Variable Stars")
data = path / "v90.csv"

# Create shared ObjectFinder to avoid recreating for each object
Expand Down Expand Up @@ -767,7 +771,7 @@ def load_abell():
conn, _ = objects_db.get_conn_cursor()
data = Path(utils.astro_data_dir, "abell.tsv")
delete_catalog_from_database(catalog)
insert_catalog(catalog, Path(utils.astro_data_dir) / "abell.desc")
insert_catalog(catalog, Path(utils.astro_data_dir) / "abell.desc", "Abell")

# Create shared ObjectFinder to avoid recreating for each object
from .catalog_import_utils import ObjectFinder
Expand Down
12 changes: 9 additions & 3 deletions python/PiFinder/catalog_imports/steinicke_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -350,9 +350,15 @@ def load_ngc_catalog():
delete_catalog_from_database("M")

# Insert catalog descriptions
insert_catalog("NGC", Path(utils.astro_data_dir, "ngc_ic_m/ngc2000", "ngc.desc"))
insert_catalog("IC", Path(utils.astro_data_dir, "ngc_ic_m/", "ic.desc"))
insert_catalog("M", Path(utils.astro_data_dir, "ngc_ic_m/messier", "messier.desc"))
insert_catalog(
"NGC", Path(utils.astro_data_dir, "ngc_ic_m/ngc2000", "ngc.desc"), "NGC"
)
insert_catalog("IC", Path(utils.astro_data_dir, "ngc_ic_m/", "ic.desc"), "IC")
insert_catalog(
"M",
Path(utils.astro_data_dir, "ngc_ic_m/messier", "messier.desc"),
"Messier",
)

conn, _db_c = objects_db.get_conn_cursor()

Expand Down
4 changes: 3 additions & 1 deletion python/PiFinder/catalog_imports/wds_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,9 @@ def load_wds():

data_path = Path(utils.astro_data_dir, "WDS/wds_precise.txt")
delete_catalog_from_database(catalog)
insert_catalog(catalog, Path(utils.astro_data_dir) / "WDS/wds.desc")
insert_catalog(
catalog, Path(utils.astro_data_dir) / "WDS/wds.desc", "Washington Double Star"
)
data = read_wds_catalog(data_path)

def parse_coordinates_2000(coord):
Expand Down
11 changes: 9 additions & 2 deletions python/PiFinder/catalogs.py
Original file line number Diff line number Diff line change
Expand Up @@ -280,8 +280,14 @@ def apply(self, objects: List[CompositeObject]):
class Catalog(CatalogBase):
"""Extends the CatalogBase with filtering"""

def __init__(self, catalog_code: str, desc: str, max_sequence: int = 0):
super().__init__(catalog_code, desc, max_sequence)
def __init__(
self,
catalog_code: str,
desc: str,
max_sequence: int = 0,
name: Optional[str] = None,
):
super().__init__(catalog_code, desc, max_sequence, name=name)
self.catalog_filter: Union[CatalogFilter, None] = None
self.filtered_objects: List[CompositeObject] = self.get_objects()
self.filtered_objects_seq: List[int] = self._filtered_objects_to_seq()
Expand Down Expand Up @@ -1108,6 +1114,7 @@ def _get_catalogs(
catalog_code,
desc=catalog_info["desc"],
max_sequence=catalog_info["max_sequence"],
name=catalog_info.get("name"),
)
catalog.add_objects(composite_dict.get(catalog_code, []))
catalog_list.append(catalog)
Expand Down
81 changes: 75 additions & 6 deletions python/PiFinder/db/objects_db.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,38 @@
import PiFinder.utils as utils
from sqlite3 import Connection, Cursor
from sqlite3 import Connection, Cursor, OperationalError
from typing import Tuple, DefaultDict, List, Dict
from PiFinder.db.db import Database
from collections import defaultdict
import logging
import time

# Readable display names per catalog code, used both by the import loaders
# (fresh build) and by the startup migration that backfills the `name` column
# on already-shipped, pre-built databases. Keep these two paths in agreement.
CATALOG_DISPLAY_NAMES: Dict[str, str] = {
"NGC": "NGC",
"IC": "IC",
"M": "Messier",
"C": "Caldwell",
"H": "Herschel",
"Col": "Collinder",
"Ta2": "TAAS 200",
"SaA": "SAC Asterisms",
"SaM": "SAC Doubles",
"SaR": "SAC Red Stars",
"Str": "Bright Stars",
"EGC": "Extragalactic Globular Clusters",
"RDS": "RASC Double Stars",
"B": "Barnard",
"Sh2": "Sharpless",
"Abl": "Abell",
"Arp": "Arp",
"TLK": "TLK Variable Stars",
"WDS": "Washington Double Star",
"Har": "Harris",
"Lyn": "Lynga",
}


class ObjectsDatabase(Database):
def __init__(self, db_path=utils.pifinder_db):
Expand All @@ -26,6 +53,10 @@ def __init__(self, db_path=utils.pifinder_db):
self.conn.commit()
self.bulk_mode = False # Flag to disable commits during bulk operations

# One-time, idempotent backfill of the catalogs.name column for
# pre-built databases shipped without it.
self._migrate_catalog_names()

def create_tables(self):
# Create objects table
self.cursor.execute(
Expand Down Expand Up @@ -76,7 +107,8 @@ def create_tables(self):
CREATE TABLE IF NOT EXISTS catalogs (
catalog_code TEXT PRIMARY KEY,
max_sequence INT,
desc TEXT
desc TEXT,
name TEXT
);
"""
)
Expand Down Expand Up @@ -111,6 +143,43 @@ def create_tables(self):
# Commit changes to the database
self.conn.commit()

def _migrate_catalog_names(self) -> None:
"""Add and backfill the catalogs.name column on legacy databases.

The shipped pifinder_objects.db is a pre-built binary that predates the
name column, so a full re-import is not normally run on the device. This
adds the column if missing and populates it by catalog_code, leaving any
unmapped catalogs with a NULL name (the UI falls back to the code).
Safe to run on every startup: it is a no-op once the column exists.
"""
# The catalogs table may not exist yet during a fresh import build.
self.cursor.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name='catalogs'"
)
if self.cursor.fetchone() is None:
return

self.cursor.execute("PRAGMA table_info(catalogs)")
columns = [row["name"] for row in self.cursor.fetchall()]
if "name" in columns:
return

logging.info("Migrating catalogs table: adding and backfilling 'name'")
try:
self.cursor.execute("ALTER TABLE catalogs ADD COLUMN name TEXT")
except OperationalError as e:
# Another process won the race and already added the column.
if "duplicate column" in str(e).lower():
return
raise

for catalog_code, display_name in CATALOG_DISPLAY_NAMES.items():
self.cursor.execute(
"UPDATE catalogs SET name = ? WHERE catalog_code = ?;",
(display_name, catalog_code),
)
self.conn.commit()

def get_pifinder_database(self) -> Tuple[Connection, Cursor]:
return self.get_database(utils.pifinder_db)

Expand Down Expand Up @@ -235,13 +304,13 @@ def get_name_to_object_id(self, id_to_names_dict=None) -> Dict[str, int]:

# ---- CATALOGS methods ----

def insert_catalog(self, catalog_code, max_sequence, desc):
def insert_catalog(self, catalog_code, max_sequence, desc, name=None):
self.cursor.execute(
"""
INSERT INTO catalogs (catalog_code, max_sequence, desc)
VALUES (?, ?, ?);
INSERT INTO catalogs (catalog_code, max_sequence, desc, name)
VALUES (?, ?, ?, ?);
""",
(catalog_code, max_sequence, desc),
(catalog_code, max_sequence, desc, name),
)
self.conn.commit()

Expand Down
Loading