diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..6bc9389 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,45 @@ +## Description + + + + +## Type of change + + + +- [ ] Bug fix +- [ ] New feature / tool addition +- [ ] Version update (tool or image) +- [ ] Configuration change (Ansible, Packer, OpenStack) +- [ ] Documentation only +- [ ] Other: + +--- + +## Version updates + + + +| Tool | Old version | New version | +|------|-------------|-------------| +| | | | + +**Image version** (`build/openstack-bioshell.pkr.hcl`) +- [ ] Bumped `image_version` to reflect changes in this PR +- [ ] No image version bump needed (docs/config only) + +--- + +## Changelog + +- [ ] Updated `CHANGELOG.md` with a summary of changes under the correct version heading +- [ ] No changelog entry needed (e.g. internal refactor, typo fix) + +--- + +## Checklist + +- [ ] My changes follow the existing conventions in this repo +- [ ] I have reviewed the diff and removed any unintended changes +- [ ] Any new tools or version pins are documented in the relevant Ansible vars file \ No newline at end of file diff --git a/.github/scripts/bump-image-version.py b/.github/scripts/bump-image-version.py new file mode 100644 index 0000000..c8a706f --- /dev/null +++ b/.github/scripts/bump-image-version.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import re +import sys + +PKR_FILE = "build/openstack-bioshell.pkr.hcl" + + +def bump_minor(version): + major, minor, patch = (int(p) for p in version.split(".")) + return f"{major}.{minor + 1}.0" + + +def main(): + with open(PKR_FILE) as f: + text = f.read() + + pattern = re.compile( + r'(variable\s+"image_version"\s*{\s*type\s*=\s*string\s*default\s*=\s*")' + r'(\d+\.\d+\.\d+)' + r'(")', + re.DOTALL, + ) + + match = pattern.search(text) + if not match: + print(f"Could not find image_version variable block in {PKR_FILE}", file=sys.stderr) + sys.exit(1) + + old_version = match.group(2) + new_version = bump_minor(old_version) + + new_text = pattern.sub(lambda m: f"{m.group(1)}{new_version}{m.group(3)}", text, count=1) + + with open(PKR_FILE, "w") as f: + f.write(new_text) + + print(f"image_version: {old_version} -> {new_version}") + + gha_output = sys.argv[1] if len(sys.argv) > 1 else None + if gha_output: + with open(gha_output, "a") as f: + f.write(f"old_image_version={old_version}\n") + f.write(f"new_image_version={new_version}\n") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/.github/scripts/check-versions.py b/.github/scripts/check-versions.py new file mode 100644 index 0000000..2e0a7df --- /dev/null +++ b/.github/scripts/check-versions.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 + +import json +import os +import re +import subprocess +import sys +import urllib.request +import urllib.error +from datetime import datetime, timezone + +import yaml + +VARS_FILE = "build/ansible/vars/tool-versions.yml" +USER_AGENT = "tool-version-checker (+https://github.com)" + +def http_get_json(url, headers=None): + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, **(headers or {})}) + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode()) + +def http_get_text(url, headers=None): + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, **(headers or {})}) + with urllib.request.urlopen(req, timeout=30) as resp: + return resp.read().decode() + +def github_latest_release_tag(repo): + headers = {} + token = os.environ.get("GITHUB_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + data = http_get_json(f"https://api.github.com/repos/{repo}/releases/latest", headers=headers) + return data["tag_name"] + +def strip_v_prefix(tag): + return tag[1:] if tag.lower().startswith("v") else tag + +def semver_tuple(v): + parts = re.findall(r"\d+", v) + return tuple(int(p) for p in parts) if parts else (0,) + + +# --------------------------------------------------------------------------- +# Per-tool checkers +# --------------------------------------------------------------------------- + +def check_singularity(current): + tag = github_latest_release_tag("sylabs/singularity") + latest = strip_v_prefix(tag) + if semver_tuple(latest) > semver_tuple(current): + return latest, f"https://github.com/sylabs/singularity/releases/tag/{tag}" + return None, None + +def check_shpc(current): + tag = github_latest_release_tag("singularityhub/singularity-hpc") + latest = strip_v_prefix(tag) + if semver_tuple(latest) > semver_tuple(current): + return latest, f"https://github.com/singularityhub/singularity-hpc/releases/tag/{tag}" + return None, None + +def check_go(current): + data = http_get_json("https://go.dev/dl/?mode=json") + stable = [d for d in data if d.get("stable")] + if not stable: + return None, None + latest_tag = stable[0]["version"] + latest = latest_tag[2:] if latest_tag.startswith("go") else latest_tag + if semver_tuple(latest) > semver_tuple(current): + return latest, "https://go.dev/dl/" + return None, None + +def check_nextflow(current): + tag = github_latest_release_tag("nextflow-io/nextflow") + latest = strip_v_prefix(tag) + if semver_tuple(latest) > semver_tuple(current): + return latest, f"https://github.com/nextflow-io/nextflow/releases/tag/{tag}" + return None, None + +def check_nfcore(current): + data = http_get_json("https://pypi.org/pypi/nf-core/json") + latest = data["info"]["version"] + if semver_tuple(latest) > semver_tuple(current): + return latest, "https://pypi.org/project/nf-core/" + return None, None + +RSTUDIO_VERSION_PATTERN = re.compile(r"^\d{4}\.\d{1,2}\.\d+[+-]\d+$") + +def find_rstudio_version_in_json(data): + if isinstance(data, str): + return data if RSTUDIO_VERSION_PATTERN.match(data) else None + if isinstance(data, dict): + for value in data.values(): + found = find_rstudio_version_in_json(value) + if found: + return found + elif isinstance(data, list): + for item in data: + found = find_rstudio_version_in_json(item) + if found: + return found + return None + + +def check_rstudio(current_full): + data = http_get_json("https://www.rstudio.com/wp-content/downloads.json") + version_full_raw = find_rstudio_version_in_json(data) + + if version_full_raw is None: + shape = list(data.keys()) if isinstance(data, dict) else type(data).__name__ + raise RuntimeError( + f"could not find an RStudio version string in downloads.json response " + f"(top-level shape: {shape})" + ) + + version_full = version_full_raw.replace("+", "-") + version_short = version_full.split("-")[0] + + if semver_tuple(version_full) > semver_tuple(current_full): + return {"rstudio_version_full": version_full, "rstudio_version": version_short}, \ + "https://www.rstudio.com/wp-content/downloads.json" + return None, None + + +def check_apt_package(package_name, current): + result = subprocess.run( + ["apt-cache", "madison", package_name], + capture_output=True, text=True, timeout=30, + ) + if result.returncode != 0 or not result.stdout.strip(): + raise RuntimeError(f"apt-cache madison {package_name} failed: {result.stderr.strip()}") + + first_line = result.stdout.strip().splitlines()[0] + pkg_version = [p.strip() for p in first_line.split("|")][1] + + latest = pkg_version.split("-")[0] + + if semver_tuple(latest) > semver_tuple(current): + return latest, f"apt-cache madison {package_name} (ubuntu noble/universe)" + return None, None + +def check_r(current): + return check_apt_package("r-base", current) + +def check_snakemake_apt(current): + return check_apt_package("snakemake", current) + +def check_jupyter(current): + now_stamp = datetime.now(timezone.utc).strftime("%Y.%m") + if now_stamp != current: + return now_stamp, "(timestamp - not an upstream version)" + return None, None + + +CHECKS = [ + ("Singularity", "singularity_version", check_singularity), + ("shpc", "shpc_version", check_shpc), + ("Go", "go_version", check_go), + ("Nextflow", "nextflow_version", check_nextflow), + ("nf-core", "nfcore_version", check_nfcore), + ("RStudio", "rstudio_version_full", check_rstudio), + ("Snakemake (apt)", "snakemake_version", check_snakemake_apt), + ("R (apt)", "r_version", check_r), + ("Jupyter (timestamp)", "jupyter_version", check_jupyter), +] + + +def main(): + with open(VARS_FILE) as f: + raw_text = f.read() + tool_vars = yaml.safe_load(raw_text) + + changes = [] # list of dicts: {label, key, old, new, source} + errors = [] # list of (label, error message) + + for label, key, checker in CHECKS: + current = tool_vars.get(key) + if current is None: + errors.append((label, f"key '{key}' not found in {VARS_FILE}")) + continue + try: + new_value, source = checker(current) + except (urllib.error.URLError, urllib.error.HTTPError, KeyError, + json.JSONDecodeError, RuntimeError, subprocess.SubprocessError) as e: + errors.append((label, str(e))) + continue + + if new_value is None: + continue + + if isinstance(new_value, dict): + # multi-key update (RStudio: full + short) + for sub_key, sub_new in new_value.items(): + old = tool_vars.get(sub_key) + if old != sub_new: + changes.append({"label": label, "key": sub_key, "old": old, "new": sub_new, "source": source}) + else: + old = tool_vars.get(key) + if old != new_value: + changes.append({"label": label, "key": key, "old": old, "new": new_value, "source": source}) + + updated_text = raw_text + for change in changes: + pattern = re.compile( + rf'^({re.escape(change["key"])}[ \t]*:[ \t]*)"?{re.escape(str(change["old"]))}"?[ \t]*$', + re.MULTILINE, + ) + replacement = f'{change["key"]}: "{change["new"]}"' + new_text, count = pattern.subn(replacement, updated_text) + if count == 0: + key_name = change["key"] + old_val = change["old"] + errors.append((change["label"], f"could not locate '{key_name}: {old_val}' in file to replace")) + continue + updated_text = new_text + + if changes: + with open(VARS_FILE, "w") as f: + f.write(updated_text) + + # Write outputs for the GitHub Actions workflow to consume + with open("version_check_summary.md", "w") as f: + if changes: + f.write("## Tool version updates found\n\n") + f.write("| Tool | Variable | Old | New | Source |\n") + f.write("|---|---|---|---|---|\n") + for c in changes: + f.write(f"| {c['label']} | `{c['key']}` | `{c['old']}` | `{c['new']}` | {c['source']} |\n") + else: + f.write("No tool version updates found.\n") + + if errors: + f.write("\n### Checks that could not complete\n\n") + for label, msg in errors: + f.write(f"- **{label}**: {msg}\n") + + f.write("\n### Tools intentionally not auto-checked\n\n") + f.write("- `java_version` - major LTS selector, not an auto-bump target.\n") + + # Set a GitHub Actions output so the workflow knows whether to open a PR + gha_output = sys.argv[1] if len(sys.argv) > 1 else None + if gha_output: + with open(gha_output, "a") as f: + f.write(f"changes_found={'true' if changes else 'false'}\n") + + print(f"Checked {len(CHECKS)} tools: {len(changes)} updates found, {len(errors)} errors.") + for c in changes: + print(f" {c['label']}: {c['key']} {c['old']} -> {c['new']}") + for label, msg in errors: + print(f" [error] {label}: {msg}", file=sys.stderr) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/.github/workflows/update-tool-versions.yml b/.github/workflows/update-tool-versions.yml new file mode 100644 index 0000000..0c1c9db --- /dev/null +++ b/.github/workflows/update-tool-versions.yml @@ -0,0 +1,84 @@ +name: update-tool-versions + +"on": + push: + branches: + - actions-update-versions + schedule: + - cron: "0 3 1 1,7 *" + workflow_dispatch: {} + +permissions: + contents: write + pull-requests: write + +jobs: + check-and-update: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install dependencies + run: pip install pyyaml + + - name: Update apt package index + run: sudo apt-get update -qq + + - name: Get current year-month + id: date + run: echo "yyyymm=$(date -u +'%Y.%m')" >> "$GITHUB_OUTPUT" + + - name: Check tool versions and update tool-versions.yml + id: check + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python3 .github/scripts/check-versions.py "$GITHUB_OUTPUT" + + - name: Bump image_version (only if tool versions changed) + if: steps.check.outputs.changes_found == 'true' + id: bump + run: python3 .github/scripts/bump-image-version.py "$GITHUB_OUTPUT" + + - name: Read summary for PR body + if: steps.check.outputs.changes_found == 'true' + id: summary + run: | + delimiter=$(openssl rand -hex 8) + { + echo "body<<${delimiter}" + cat version_check_summary.md + echo "" + echo "---" + echo "Image version bumped: \`${{ steps.bump.outputs.old_image_version }}\` -> \`${{ steps.bump.outputs.new_image_version }}\`" + echo "${delimiter}" + } >> "$GITHUB_OUTPUT" + rm -f version_check_summary.md + + - name: Create Pull Request + if: steps.check.outputs.changes_found == 'true' + uses: peter-evans/create-pull-request@v8 + with: + token: ${{ secrets.GITHUB_TOKEN }} + branch: automated/tool-version-updates-${{ steps.date.outputs.yyyymm }} + delete-branch: true + commit-message: "Update tool versions and bump image version" + title: "Tool version updates" + body: ${{ steps.summary.outputs.body }} + add-paths: | + build/ansible/vars/tool-versions.yml + build/openstack-bioshell.pkr.hcl + labels: | + automated + dependencies + + - name: No updates found + if: steps.check.outputs.changes_found != 'true' + run: | + echo "All tool versions are already current - no PR needed." + rm -f version_check_summary.md \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..faa2668 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,13 @@ +# Changelog + +All notable changes to BioShell will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/). + +See [docs/MAINTENANCE.md](docs/MAINTENANCE.md) for the versioning policy and release process. + +## [Unreleased] + +### Added +- GitHub Action (`update-tool-versions`) that checks pinned tool versions against upstream on a schedule (1 January and 1 July) and opens a pull request with any updates. See [docs/MAINTENANCE.md](docs/MAINTENANCE.md#5-github-actions-automation). diff --git a/README.md b/README.md index adb21f6..d590870 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ This repository contains configuration and automation for building a custom Ubun * [Activation](#activation) * [Build Image](#build-image) * [Using BioShell](#Using-BioShell) +* [Maintenance & Versioning](docs/MAINTENANCE.md) ## Spinning up a VM in OpenStack diff --git a/build/openstack-bioshell.pkr.hcl b/build/openstack-bioshell.pkr.hcl index ee6ecbe..5872b4b 100644 --- a/build/openstack-bioshell.pkr.hcl +++ b/build/openstack-bioshell.pkr.hcl @@ -39,8 +39,13 @@ variable "platform" { type = string } +variable "image_version" { + type = string + default = "1.0.0" +} + source "openstack" "ubuntu" { - image_name = "bioshell" + image_name = "bioshell-v${var.image_version}" flavor = var.flavor ssh_username = "ubuntu" volume_size = var.volume_size diff --git a/docs/MAINTENANCE.md b/docs/MAINTENANCE.md new file mode 100644 index 0000000..9026c4e --- /dev/null +++ b/docs/MAINTENANCE.md @@ -0,0 +1,61 @@ +# BioShell Operational Update Schedule & Maintenance Plan + +## 1. Background & Context + +This document defines the update cadence, automated tooling, roles, and review workflow that keeps BioShell images current, reproducible, and compatible across all supported infrastructures. + +## 2. Versioning + +BioShell follows [Semantic Versioning](https://semver.org/) (SemVer): `MAJOR.MINOR.PATCH` + +| Version increment | Trigger | Action | Example | +|---|---|---|---| +| MAJOR (`X.0.0`) | Breaking changes — users must change how they work | Manual | New base OS, dropping an infrastructure platform, restructured access model | +| MINOR (`1.X.0`) | New features, backwards compatible | GitHub Action (~6 monthly), or manual for larger additions | New software bundle, new interface (RStudio), new platform supported, significant new dataset | +| PATCH (`1.1.X`) | Bug fixes and small updates | Manual | Patched config, corrected dataset, minor documentation fix | + +The version number is defined by the `image_version` variable in [`build/openstack-bioshell.pkr.hcl`](../build/openstack-bioshell.pkr.hcl) and is tied to a specific VM image snapshot — not to documentation or policy changes alone. Notable changes are recorded in [CHANGELOG.md](../CHANGELOG.md). + +Pre-1.0.0, release-candidate labels (e.g. `1.0.0-rc1`) may be used to signal a build intended for validation ahead of a production release. The jump to `1.0.0` signals production readiness. + +## 3. Objectives of the Update Process + +- Maintain a predictable update cadence aligned with infrastructure maintenance windows. +- Automate routine base-tool version bumps via GitHub Actions, reducing manual effort. +- Ensure build reproducibility through modularised Ansible roles. +- Guarantee image compatibility across Nectar, Nirin, and future cloud platforms. +- Provide a consistent versioning and release workflow through GitHub. +- Maintain automated checks to validate tool versions before release. +- Facilitate timely cross-infrastructure review and approval. + +## 4. Update Frequency & Cadence + +Scheduled updates run twice a year (January and July), aligned to infrastructure maintenance windows, with patch releases as required outside that cycle. + +## 5. GitHub Actions Automation + +Routine base-tool version bumps are automated by the [`update-tool-versions`](../.github/workflows/update-tool-versions.yml) workflow. This removes the need for manual monitoring of upstream tool releases and ensures updates are reviewed through a consistent pull request workflow. + +| Element | Detail | +|---|---| +| Trigger | Scheduled cron job, `0 3 1 1,7 *` (03:00 UTC on 1 January and 1 July); also runnable on demand via `workflow_dispatch` | +| Check | [`check-versions.py`](../.github/scripts/check-versions.py) compares the pinned versions in [`build/ansible/vars/tool-versions.yml`](../build/ansible/vars/tool-versions.yml) against upstream (GitHub releases, PyPI, apt, vendor endpoints) for Singularity, shpc, Go, Nextflow, nf-core, RStudio, Snakemake, and R | +| Bump | If any tool version changed, [`bump-image-version.py`](../.github/scripts/bump-image-version.py) increments the MINOR version of `image_version` in `build/openstack-bioshell.pkr.hcl` | +| Output | Opens a pull request with a summary table of the tool versions changed | +| Review | PR reviewed | +| Merge & tag | Product owner (or delegate) merges the approved PR and tags the new version in GitHub | + +Notes on what the workflow does **not** cover: +- `java_version` is a major-LTS selector and is intentionally excluded from auto-bumping. +- Manual or ad-hoc updates outside the cron cycle remain possible for urgent security patches — these follow the same PR-review-merge workflow, just triggered manually (`workflow_dispatch` or a direct PR) instead of waiting for the next cron run. +- The workflow only bumps *pinned tool versions*. New software bundles, new platforms, or other MINOR/MAJOR changes are made by hand and versioned manually per the table in [Section 2](#2-versioning). + +## 6. Release Checklist + +When merging a version bump (automated or manual): + +1. Confirm the automated version-check PR (or manual change) has been reviewed by both platform collaborators. +2. Build and smoke-test the image on at least one of Nectar or Nirin before merge, where practical. +3. Merge the PR. +4. Tag the release in GitHub matching `image_version` (e.g. `v1.1.0`). +5. Add an entry to [CHANGELOG.md](../CHANGELOG.md) summarising the change.