diff --git a/src/vfbquery/cached_functions.py b/src/vfbquery/cached_functions.py index 01de475..74a7aaf 100644 --- a/src/vfbquery/cached_functions.py +++ b/src/vfbquery/cached_functions.py @@ -668,7 +668,7 @@ def get_lineage_clones_in_cached(short_form: str, return_dataframe=True, limit: """ return _original_get_lineage_clones_in(short_form=short_form, return_dataframe=return_dataframe, limit=limit, force_refresh=force_refresh) -def get_images_neurons_cached(short_form: str, return_dataframe=True, limit: int = -1, force_refresh: bool = False): +def get_images_neurons_cached(short_form: str, return_dataframe=True, limit: int = -1, offset: int = 0, force_refresh: bool = False): """ Enhanced get_images_neurons with SOLR caching. @@ -680,7 +680,7 @@ def get_images_neurons_cached(short_form: str, return_dataframe=True, limit: int Returns: Images of the specified neuron """ - return _original_get_images_neurons(short_form=short_form, return_dataframe=return_dataframe, limit=limit, force_refresh=force_refresh) + return _original_get_images_neurons(short_form=short_form, return_dataframe=return_dataframe, limit=limit, offset=offset, force_refresh=force_refresh) def get_images_that_develop_from_cached(short_form: str, return_dataframe=True, limit: int = -1, force_refresh: bool = False): """ diff --git a/src/vfbquery/ha_api.py b/src/vfbquery/ha_api.py index 15e4a01..341856b 100644 --- a/src/vfbquery/ha_api.py +++ b/src/vfbquery/ha_api.py @@ -702,7 +702,7 @@ async def handle_get_term_info(request): # Query functions that accept offset/limit paging (id ASC pages, <=10000 rows). -PAGED_QUERY_FUNCS = {"get_all_aligned_images"} +PAGED_QUERY_FUNCS = {"get_all_aligned_images", "get_images_neurons"} # Hard safety cap on rows returned to the client for ANY query. Broad terms # make many queries huge or unbounded (e.g. AllAlignedImages ~528k, and diff --git a/src/vfbquery/vfb_queries.py b/src/vfbquery/vfb_queries.py index b32a852..67289b3 100644 --- a/src/vfbquery/vfb_queries.py +++ b/src/vfbquery/vfb_queries.py @@ -10,6 +10,7 @@ import json import numpy as np from urllib.parse import unquote +import hashlib from .solr_result_cache import with_solr_cache, solr_caching_disabled import time import threading @@ -2561,7 +2562,7 @@ def get_instances(short_form: str, return_dataframe=True, limit: int = -1): MATCH (i:Individual:has_image)-[:INSTANCEOF]->(p:Class), (i)<-[:depicts]-(tc:Individual)-[r:in_register_with]->(tct:Template)-[:depicts]->(templ:Template), (i)-[:has_source]->(ds:DataSet) - WHERE p.short_form IN {class_ids!r} + WHERE p.short_form IN {class_ids!r} AND NOT i:Deprecated OPTIONAL MATCH (i)-[rx:database_cross_reference]->(site:Site) OPTIONAL MATCH (ds)-[:has_license|license]->(lic:License) RETURN i.short_form as id, @@ -2589,7 +2590,7 @@ def get_instances(short_form: str, return_dataframe=True, limit: int = -1): MATCH (i:Individual:has_image)-[:INSTANCEOF]->(p:Class), (i)<-[:depicts]-(tc:Individual)-[r:in_register_with]->(tct:Template)-[:depicts]->(templ:Template), (i)-[:has_source]->(ds:DataSet) - WHERE p.short_form IN {class_ids!r} + WHERE p.short_form IN {class_ids!r} AND NOT i:Deprecated OPTIONAL MATCH (i)-[rx:database_cross_reference]->(site:Site) OPTIONAL MATCH (ds)-[:has_license|license]->(lic:License) RETURN count(*) AS total_count @@ -4576,7 +4577,7 @@ def get_flybase_combo_pubs(short_form: str, return_dataframe=True, limit: int = @with_solr_cache('images_neurons') -def get_images_neurons(short_form: str, return_dataframe=True, limit: int = -1): +def get_images_neurons(short_form: str, return_dataframe=True, limit: int = -1, offset: int = 0): """ Retrieves individual neuron images with parts in the specified synaptic neuropil. @@ -4595,7 +4596,7 @@ def get_images_neurons(short_form: str, return_dataframe=True, limit: int = -1): """ owl_query = f" and some <{_short_form_to_iri(short_form)}>" return _owlery_query_to_results(owl_query, short_form, return_dataframe, limit, - solr_field='anat_image_query', query_by_label=False, query_instances=True) + solr_field='anat_image_query', query_by_label=False, query_instances=True, offset=offset) @with_solr_cache('images_that_develop_from') @@ -4737,10 +4738,26 @@ def _get_standard_query_headers(): } +@with_solr_cache('owlery_ids') +def _owlery_ids_cached(owl_key: str, owl_query_string: str, query_instances: bool = False, query_by_label: bool = True, force_refresh: bool = False): + """Cached Owlery id-list (the natural index for owlery-based queries). + + The Owlery reasoner call is the expensive part; caching just this small + ordered id list lets each offset page hydrate from SOLR without re-querying + Owlery, and lets the true total (len of this list) be reported even when a + page is limited — so no 50k ceiling truncation for these queries. + """ + if query_instances: + ids = vc.vfb.oc.get_instances(query=owl_query_string, query_by_label=query_by_label, verbose=False) + else: + ids = vc.vfb.oc.get_subclasses(query=owl_query_string, query_by_label=query_by_label, verbose=False) + return list(ids or []) + + def _owlery_query_to_results(owl_query_string: str, short_form: str, return_dataframe: bool = True, limit: int = -1, solr_field: str = 'anat_query', include_source: bool = False, query_by_label: bool = True, - query_instances: bool = False): + query_instances: bool = False, offset: int = 0): """ Unified helper function for Owlery-based queries. @@ -4760,21 +4777,10 @@ def _owlery_query_to_results(owl_query_string: str, short_form: str, return_data :return: Query results """ try: - # Step 1: Query Owlery for classes or instances matching the OWL pattern - if query_instances: - result_ids = vc.vfb.oc.get_instances( - query=owl_query_string, - query_by_label=query_by_label, - verbose=False - ) - else: - result_ids = vc.vfb.oc.get_subclasses( - query=owl_query_string, - query_by_label=query_by_label, - verbose=False - ) - - class_ids = result_ids # Keep variable name for compatibility + # Step 1: Owlery id list (cached) — the natural index for paging. Cache + # key is short_form + a hash of the OWL query (clean, unique doc id). + owl_key = f"{short_form}_{'i' if query_instances else 'c'}_{hashlib.md5(owl_query_string.encode()).hexdigest()[:10]}" + class_ids = _owlery_ids_cached(owl_key, owl_query_string, query_instances=query_instances, query_by_label=query_by_label) if not class_ids: # No results found - return empty @@ -4788,9 +4794,12 @@ def _owlery_query_to_results(owl_query_string: str, short_form: str, return_data total_count = len(class_ids) - # Apply limit if specified (before SOLR query to save processing) - if limit != -1 and limit > 0: - class_ids = class_ids[:limit] + # Page the id list: hydrate only [offset, offset+limit). total_count above + # is the TRUE total, so the client pages through everything (no ceiling). + if limit is not None and limit > 0: + class_ids = class_ids[offset:offset + limit] + elif offset and offset > 0: + class_ids = class_ids[offset:] # Step 2: Query SOLR for ALL classes in a single batch query # Use the {!terms f=id} syntax from XMI to fetch all results efficiently @@ -6082,7 +6091,7 @@ def get_all_aligned_image_ids(template_short_form: str): reproducible within a cache generation.""" q = (f"MATCH (:Template:Individual {{short_form:'{template_short_form}'}})" "<-[:depicts]-(:Template:Individual)<-[:in_register_with]-(:Individual)" - "-[:depicts]->(di:Individual) RETURN DISTINCT di.short_form AS id ORDER BY id ASC") + "-[:depicts]->(di:Individual) WHERE NOT di:Deprecated RETURN DISTINCT di.short_form AS id ORDER BY id ASC") results = vc.nc.commit_list([q]) rows = get_dict_cursor()(results) if results else [] return [r['id'] for r in rows]