From 961c5c8717ac3ebeafd26d9e9b966a9f4af78e81 Mon Sep 17 00:00:00 2001 From: dylan Date: Sat, 18 Jul 2026 06:20:48 +0000 Subject: [PATCH] Skip flash-attn in CI and harden set_weights error handling CI builds flash-attn from source since it became a hard dependency, but runners have no CUDA toolkit. Skip it by name like the rest of the GPU stack in the frozen sync. set_weights returns success=False with an empty message when the weights rate limit hasn't elapsed. Detect that case and stop retrying instead of logging a misleading "failed None". --- .github/workflows/ci.yml | 1 + gas/utils/metagraph.py | 35 +++++++++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0bdd38a..243f7f5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,7 @@ jobs: # needs it, drop the corresponding flag. run: > uv sync --frozen + --no-install-package flash-attn --no-install-package torch --no-install-package torchvision --no-install-package torchaudio diff --git a/gas/utils/metagraph.py b/gas/utils/metagraph.py index 1a1c2c7b..2df81550 100644 --- a/gas/utils/metagraph.py +++ b/gas/utils/metagraph.py @@ -55,7 +55,7 @@ def set_weights( #bt.logging.info("Setting Weights: " + str(processed_weights)) #bt.logging.info("Weight Uids: " + str(processed_weight_uids)) for _ in range(3): - result, message = subtensor.set_weights( + response = subtensor.set_weights( wallet=wallet, netuid=netuid, uids=processed_weight_uids, # type: ignore @@ -65,16 +65,43 @@ def set_weights( version_key=version, max_attempts=1, ) - if result is True: + success, message = response + if success is True: bt.logging.success("set_weights on chain successfully!") break - else: - bt.logging.error(f"set_weights failed {message}") + + # set_weights returns success=False with no message when the weights + # rate limit hasn't elapsed; retrying can't clear it, so stop. + reason = message or getattr(response, "error", None) + if reason is None and _weights_rate_limited(subtensor, wallet, netuid): + bt.logging.info("Skipping set_weights: rate limit not elapsed (too early)") + break + bt.logging.error(f"set_weights failed: {reason or 'unknown error'}") time.sleep(15) return set_weights +def _weights_rate_limited( + subtensor: "bt.Subtensor", wallet: "bt.Wallet", netuid: int +) -> bool: + """Return True if the weights rate limit has not yet elapsed for this hotkey.""" + try: + uid = subtensor.get_uid_for_hotkey_on_subnet( + wallet.hotkey.ss58_address, netuid + ) + if uid is None: + return False + blocks_since = subtensor.blocks_since_last_update(netuid, uid) + rate_limit = subtensor.weights_rate_limit(netuid) + if blocks_since is None or rate_limit is None: + return False + return blocks_since <= rate_limit + except Exception as e: + bt.logging.debug(f"Could not check weights rate limit: {e}") + return False + + def create_async_subscription_handler(callback: Callable): """Create async subscription handler - simple version.""" async def handler(obj):