diff --git a/.github/workflows/remove-canceled-netlify-builds.yml b/.github/workflows/remove-canceled-netlify-builds.yml new file mode 100644 index 0000000..4be7611 --- /dev/null +++ b/.github/workflows/remove-canceled-netlify-builds.yml @@ -0,0 +1,19 @@ +name: Remove Canceled Netlify Comments + +on: + issue_comment: + types: [created, edited] + +jobs: + cleanup: + if: github.event.comment.user.login == 'netlify[bot]' && contains(github.event.comment.body, 'Deploy Preview for') && contains(github.event.comment.body, 'canceled.') + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@v7 + with: + script: | + github.rest.issues.deleteComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id + }) diff --git a/_data/sidebars/main.yml b/_data/sidebars/main.yml index 62aef82..3b9e7de 100644 --- a/_data/sidebars/main.yml +++ b/_data/sidebars/main.yml @@ -14,6 +14,8 @@ subitems: url: /connecting - title: Tools and reference data url: /tools + - title: How to use Shelley + url: /shelley-howto - title: Interactive environments url: /interactive - title: Using Nextflow with CVMFS diff --git a/assets/img/module_avail.png b/assets/img/module_avail.png new file mode 100644 index 0000000..240ecb5 Binary files /dev/null and b/assets/img/module_avail.png differ diff --git a/assets/img/shelley_build_bowtie2_unregistered.png b/assets/img/shelley_build_bowtie2_unregistered.png new file mode 100644 index 0000000..a1b5aaa Binary files /dev/null and b/assets/img/shelley_build_bowtie2_unregistered.png differ diff --git a/assets/img/shelley_build_bwa-mem2.png b/assets/img/shelley_build_bwa-mem2.png new file mode 100644 index 0000000..fe3f7d2 Binary files /dev/null and b/assets/img/shelley_build_bwa-mem2.png differ diff --git a/assets/img/shelley_build_fastqc.png b/assets/img/shelley_build_fastqc.png new file mode 100644 index 0000000..8688145 Binary files /dev/null and b/assets/img/shelley_build_fastqc.png differ diff --git a/assets/img/shelley_build_multiple-builds-prompt.png b/assets/img/shelley_build_multiple-builds-prompt.png new file mode 100644 index 0000000..039931f Binary files /dev/null and b/assets/img/shelley_build_multiple-builds-prompt.png differ diff --git a/assets/img/shelley_build_tools-txt.png b/assets/img/shelley_build_tools-txt.png new file mode 100644 index 0000000..bb2cd74 Binary files /dev/null and b/assets/img/shelley_build_tools-txt.png differ diff --git a/assets/img/shelley_build_tools-txt_summary.png b/assets/img/shelley_build_tools-txt_summary.png new file mode 100644 index 0000000..4e5ce7c Binary files /dev/null and b/assets/img/shelley_build_tools-txt_summary.png differ diff --git a/assets/img/shelley_build_vcftools_alias-filter-vcf.png b/assets/img/shelley_build_vcftools_alias-filter-vcf.png new file mode 100644 index 0000000..267c9c9 Binary files /dev/null and b/assets/img/shelley_build_vcftools_alias-filter-vcf.png differ diff --git a/assets/img/shelley_build_vcftools_alias-select-all.png b/assets/img/shelley_build_vcftools_alias-select-all.png new file mode 100644 index 0000000..c0486c1 Binary files /dev/null and b/assets/img/shelley_build_vcftools_alias-select-all.png differ diff --git a/assets/img/shelley_build_vcftools_alias-selected.png b/assets/img/shelley_build_vcftools_alias-selected.png new file mode 100644 index 0000000..b73387c Binary files /dev/null and b/assets/img/shelley_build_vcftools_alias-selected.png differ diff --git a/assets/img/shelley_build_vcftools_complete.png b/assets/img/shelley_build_vcftools_complete.png new file mode 100644 index 0000000..e56d058 Binary files /dev/null and b/assets/img/shelley_build_vcftools_complete.png differ diff --git a/assets/img/shelley_find_bwa-mem2.png b/assets/img/shelley_find_bwa-mem2.png new file mode 100644 index 0000000..114791b Binary files /dev/null and b/assets/img/shelley_find_bwa-mem2.png differ diff --git a/assets/img/shelley_find_fastqc.png b/assets/img/shelley_find_fastqc.png new file mode 100644 index 0000000..51dfedd Binary files /dev/null and b/assets/img/shelley_find_fastqc.png differ diff --git a/assets/img/shelley_find_fastqc_v.png b/assets/img/shelley_find_fastqc_v.png new file mode 100644 index 0000000..62635fe Binary files /dev/null and b/assets/img/shelley_find_fastqc_v.png differ diff --git a/assets/img/shelley_help.png b/assets/img/shelley_help.png new file mode 100644 index 0000000..3656dc1 Binary files /dev/null and b/assets/img/shelley_help.png differ diff --git a/assets/img/shelley_search_de-novo-assembly.png b/assets/img/shelley_search_de-novo-assembly.png new file mode 100644 index 0000000..3ec3d3d Binary files /dev/null and b/assets/img/shelley_search_de-novo-assembly.png differ diff --git a/netlify.toml b/netlify.toml new file mode 100644 index 0000000..2191977 --- /dev/null +++ b/netlify.toml @@ -0,0 +1,6 @@ +[build] + command = "bundle exec jekyll build --baseurl ''" + publish = "_site" + +[build.environment] + RUBY_VERSION = "3.1" diff --git a/pages/shelley-howto.md b/pages/shelley-howto.md new file mode 100644 index 0000000..db3ccad --- /dev/null +++ b/pages/shelley-howto.md @@ -0,0 +1,301 @@ +--- +title: How to use Shelley +type: Using BioShell +description: A collection of guides for finding, building, and installing bioinformatics tools with Shelley +--- + +This page collects several use cases for Shelley to find and build tools. If you are new to Shelley and want a guided tour of how +it fits together, see the [**Getting started with Shelley**](tools#getting-started-with-shelley) +tutorial first. + +## Basic usage {#basic-usage} + +**From the command line:** + +```bash +shelley find # Look up a specific tool by name +shelley search "" # Search by keyword or function +shelley build # Install the tool as a loadable module +``` + +**In interactive mode:** + +```bash +shelley interactive # Launch Shelley in interactive mode +``` + +Interactive mode works the same way as the command line. The `find`, `search`, and `build` +behave identically, except you type just the command name and its arguments, without +prefixing every call with `shelley`. + +## How-to install `bwa-mem2` {#how-to-install-bwa-mem2} + +This worked example uses `bwa-mem2` to demonstrate the steps to use and apply to any tool in the CVMFS catalogue. + +**1.** Use `shelley` to look up `bwa-mem2` and the available versions + +```bash +shelley find bwa-mem2 +``` + +
+Show output +![](assets/img/shelley_find_bwa-mem2.png) +
+
+
+ +**2.** Build a loadable module + +```bash +shelley build bwa-mem2/2.3 +``` + +
+Show output +![](assets/img/shelley_build_bwa-mem2.png) +
+
+
+ +**3.** Confirm the module was built succesfully + +```bash +module avail +``` + +`bwa-mem2` is listed alongside the other pre-built BioShell modules. + +
+Show output +![](assets/img/module_avail.png) +
+
+
+ +**4.** Load the module + +```bash +module load bwa-mem2 +``` + +**5.** Finally, run `bwa-mem2` + +```bash +bwa-mem2 version +# 2.2.1 +``` + +You now have a working `bwa-mem2` module built from the BioContainers catalogue. +The same five steps: find, build, confirm, load, run - apply to any tool available +through Shelley. + +{% include callout.html type="note" content="Curious why Shelley automates this instead of running `shpc install` by hand? See [why Shelley over manual sHPC](https://github.com/Sydney-Informatics-Hub/shelley/blob/main/docs/explanation/why-shelley.md) in the Shelley docs." %} + +## How-to bulk install a list of containers {#bulk-install} + +If you need several modules built at once, for example, all the tools a pipeline +depends on, you can pass `shelley build` a plain-text file listing one tool spec per line +instead of building each tool individually. + +**1.** Create `tools.txt` with the following contents + +```md +# qc tools +fastqc + +# Core alignment tools +# some tools have versions specified for reproducibility +samtools/1.21 +bowtie2/2.5.1 +bwa-mem2 +``` + +Each line accepts the same format as a single-tool `build` call: +- ``, +- `/`, or +- `:--` + +Blank lines and `#` comments are ignored, so you can group and annotate the list. + +**2.** Run the `build` command: + +```bash +shelley build tools.txt +``` + +Shelley builds each tool in the file in sequence, showing a progress table as it goes. + +
+Show output +![](assets/img/shelley_build_tools-txt.png) +
+
+
+ +**3.** If a tool has more than one build for the same version (for example, several +`--hash` builds of `samtools/1.21`), Shelley pauses and asks you to pick one. + +
+Show output +![](assets/img/shelley_build_multiple-builds-prompt.png) +
+
+
+ +Use the arrow keys to select a build, then press Enter to continue building the tools. + +**4.** Once every tool has been built, Shelley prints a summary, and the modules are +available to load. + +
+Show output +![](assets/img/shelley_build_tools-txt_summary.png) +
+
+
+ +```bash +module avail +``` + +``` +---------------------------------------------------------- /apps/Modules/modulefiles ---------------------------------------------------------- + R/4.3.3 bowtie2/2.5.1--py39h6fed5c7_2 fastqc/0.12.1--hdfd78af_0 nextflow/25.10.4 rstudio/2023.12.1 + ansible/2.16.3 bwa-mem2/2.3--he70b90d_0 jupyter/2026.04 nf-core/3.5.2 samtools/1.21--h96c455f_1 + +---------------------------------------------------------- /opt/Modules/modulefiles ----------------------------------------------------------- + shpc (L) singularity (L) + + Where: + L: Module is loaded +``` + +## How-to find the full CVMFS path of containers {#find-cvmfs-path} + +Some workflows need the exact CVMFS container path rather than a loaded module. For +example, a Nextflow process configured to pull a container directly instead of using +`module load`. Use `find -v` to to list every individual build of a tool together +with its full CVMFS image path: + +```bash +shelley find fastqc -v +``` + +
+Show output +![](assets/img/shelley_find_fastqc_v.png) +
+
+
+ +Each row corresponds to one `--hash` build of a version, with its buildable/installed +status and the path you can copy into a config file, such as: + +``` +/cvmfs/singularity.galaxyproject.org/all/fastqc:0.12.1--hdfd78af_0 +``` + +See [Using Nextflow with CVMFS](nextflow) for a worked example of pointing a Nextflow +process at a container path like this. + +## How-to build a tool that isn't in the sHPC registry {#unregistered-tool} + +Building an Lmod module requires a registry entry describing the container. +sHPC's [shpc-registry](https://github.com/singularityhub/shpc-registry) supplies these +for most tools, but it doesn't cover every version. Very new containers, and containers +built before a tool was added to the registry, often have no entry at all. + +**You don't need to know whether a version is registered, as `shelley build` checks for you +and will build this on-the-fly.** + +```bash +shelley build bowtie2/2.5.1 +``` + +Shelley diffs the container against its base-image manifests to work out which files +are the tool's own, so the module's aliases don't get cluttered with every binary +bundled inside the container: + +
+Show output +``` +Generating diff for /cvmfs/singularity.galaxyproject.org/all/bowtie2:2.5.1--py39h6fed5c7_2 + +docker://docker.io/anaconda/miniconda:latest: removed 98 shared paths. +docker.io/library/busybox:1.34: removed 90 shared paths. + +``` + +![](assets/img/shelley_build_bowtie2_unregistered.png) +
+
+
+ +If the version is missing from the upstream registry, Shelley creates a local registry +entry from the CVMFS container itself and retries the install. This is transparent and +adds only a few extra minutes to the build. See [Why Shelley over manual sHPC](https://github.com/Sydney-Informatics-Hub/shelley/blob/main/docs/explanation/why-shelley.md) +for what this replaces manually, and [Build design](https://github.com/Sydney-Informatics-Hub/shelley/blob/main/docs/explanation/build-design.md) +for how the fallback works internally. + +{% include callout.html type="note" content="Because the alias list for a locally-registered entry is generated straight from the container, it can include extra utility binaries bundled alongside the tool (for example, conda-infrastructure commands) rather than only the tool's own executables." %} + +## [EXPERIMENTAL] How-to amend the aliases {#amend-aliases} + +{% include callout.html type="important" content="Experimental feature - use with caution. Curating aliases changes which commands a module exposes; an alias you remove or rename may break other users' or pipelines' expectations of that module." %} + +As noted [above](#unregistered-tool), a container can bundle utility binaries alongside +the tool itself, which can clutter the module's alias list with commands you don't +need. Add `-i` (or `--interactive`) to `build` to open a session where you can curate +the aliases a module exposes such as deselecting, renaming, or adding them; before the install +completes: + +```bash +shelley build vcftools/0.1.12b--pl5262h2e03b76_2 -i +``` + +**1.** Shelley first lists every binary in the container as a candidate alias. Without +curation, all of these would be exposed by the module. + +
+Example output +![](assets/img/shelley_build_vcftools_alias-select-all.png) +
+
+
+ +**2.** Type to filter the list down to the ones relevant to `vcftools`, for example `vcf`: + +
+Example output +![](assets/img/shelley_build_vcftools_alias-filter-vcf.png) +
+
+
+ +**3.** Toggle each one you want by pressing Space, then press Enter once they're all selected: + +
+Show output +![](assets/img/shelley_build_vcftools_alias-selected.png) +
+
+
+ +**4.** Shelley then asks whether to add or rename any aliases. Answer `n` to both to keep +the selection as-is: + +``` +? Add new aliases? (y/n)No +? Rename any aliases? (y/n)No +``` + +**5.** The module builds with only the curated aliases exposed: + +
+Show output +![](assets/img/shelley_build_vcftools_complete.png) +
+
+
+ diff --git a/pages/tools.md b/pages/tools.md index 8adf82f..b1b9388 100644 --- a/pages/tools.md +++ b/pages/tools.md @@ -8,7 +8,6 @@ BioShell gives you access to thousands of bioinformatics tools and reference dat two underlying systems, **CVMFS** and **sHPC**, and a built-in assistant called **Shelley** that automates working with both. - ## How the tooling stack works {#tooling-stack} ### CVMFS {#cvmfs} @@ -62,7 +61,7 @@ samtools --version sHPC turns containers into clean, versioned modules without requiring you to know how containers work. On BioShell, sHPC should be configured so that installations point at containers already present in CVMFS, so nothing is re-downloaded. -### Shelley {#Shelley} +## Introducing Shelley :turtle: {#Shelley} Working with CVMFS paths and sHPC registry recipes by hand is tedious and error-prone, particularly for older tool versions not listed in the standard registry. **Shelley** is @@ -70,204 +69,124 @@ BioShell's command-line assistant that automates the entire workflow: it searche BioContainers index, identifies the correct container version, creates any missing registry entries, and runs the sHPC install, all from a single command. ---- - -## Installing tools {#installing-tools} - -{% include callout.html type="important" content="Use the manual CVMFS/sHPC methods only if you need finer control or are troubleshooting." %} - -### Recommended: Shelley {#shelley} - -Shelley indexes over 700 tools and 118,000 container versions from the BioContainers -catalogue. You can run it directly from the command line or in interactive mode. - -**From the command line:** - -```bash -shelley find # Look up a specific tool by name -shelley search "" # Search by keyword or function -shelley versions # List all available versions -shelley build # Install the tool as a loadable module -``` - -**In interactive mode:** - -```bash -shelley interactive # Launch Shelley in interactive mode -``` - -**Example: installing `samtools`** - -```bash -shelley find samtools -# samtools: Tools for manipulating next-generation sequencing data - -shelley versions samtools -# samtools/1.21 -# samtools/1.20 -# samtools/1.19.2 -# ... - -shelley build samtools/1.21 -# Installing samtools/1.21 from CVMFS... -# Module samtools/1.21 was created. -``` - -![](images/bioshell/SCREENSHOT_NEEDED_bioshelley_build.png) - -_Fig 3. Using `Shelley build samtools/1.21` to install a tool module._ - -{% include callout.html type="tip" content="Use `Shelley search` when you know what you want to do but not the tool name. For example, `Shelley search 'variant calling'` returns tools relevant to that task." %} - -### Why Shelley over manual sHPC? {#why-shelley} - -The manual sHPC method requires you to load the module, search the registry, identify the -correct entry, check available versions, construct the install command with the right CVMFS -path and flags, then load the result. If the version you need is not in the registry (common -for anything older than the latest release), you also need to check CVMFS directly for the -image, compute its checksum, fetch the registry YAML, edit it to add the missing version, -create and register a local registry, ensure it takes precedence, then run the install. - -With Shelley, regardless of whether the version is in the sHPC registry or not it handles the registry check, CVMFS path resolution, local entry creation, and the sHPC installation. - -```bash -shelley build samtools/1.20 -``` - -The result is identical: a working `module load` command, -without requiring you to know how the underlying machinery works. - -{% include callout.html type="tip" content="If Shelley cannot find a tool, fall back to the manual sHPC method below." %} - -
-Advanced: manual installation - -#### Using sHPC directly {#shpc-direct} - -If Shelley cannot find what you need, or you prefer direct sHPC control, use the steps -below. This example uses `plink` throughout. - -**Load sHPC:** +{% include callout.html type="tip" content="**Recommended:** use Shelley rather than sHPC directly. The rest of this guide walks through how Shelley can be used to find, install, and run tools without interacting with the CVMFS or sHPC directly!" %} -```bash -module load shpc -``` - -**Search for a tool and view versions:** +### Getting started with Shelley {#getting-started-with-shelley} -```bash -shpc show -f plink -shpc show quay.io/biocontainers/plink -``` - -**Verify container path on CVMFS:** - -```bash -ls /cvmfs/singularity.galaxyproject.org/all/plink* -``` +Shelley indexes **over 700 tools and 118,000 container versions** from the BioContainers +catalogue, and you can run it directly from the command line or in an interactive mode. +This tutorial walks through finding and installing a bioinformatics tool on a BioShell VM +for the first time. -**Install a module directly from CVMFS:** +Before you start, confirm Shelley is available: ```bash -shpc install quay.io/biocontainers/plink:1.90b7.7--h18e278d_1 \ - /cvmfs/singularity.galaxyproject.org/all/plink:1.90b7.7--h18e278d_1 \ - --keep-path +shelley help ``` -The `--keep-path` flag tells sHPC to use the container already present in CVMFS rather than -downloading a fresh copy. The tag numbers must match. +You will see a list of available commands. -**Load and use the module:** +
+Example output +![](assets/img/shelley_help.png) +
+
+
-```bash -module use ~/shpc/modules -module load quay.io/biocontainers/plink/1.90b7.7--h18e278d_1 -plink --version -``` +#### Finding a tool you already know by name -**Or run the container directly without a module:** +Say you already know you need `fastqc`. Look it up with `find`: ```bash -singularity exec \ - /cvmfs/singularity.galaxyproject.org/all/plink:1.90b7.7--h18e278d_1 \ - plink --version +shelley find fastqc ``` -For full documentation see the -[sHPC user guide](https://singularity-hpc.readthedocs.io/en/latest/getting_started/user-guide.html). +
+Example output +![](assets/img/shelley_find_fastqc.png) +
+
+
-#### Installing older or unlisted versions {#older-versions} +Shelley returns the tool's description together with its most recent container versions, +plus whether it is installed as a module yet. `find` is forgiving about +naming case, hyphens, and underscores are all handled for you, so `shelley find STAR`, +`shelley find bwa-mem2`, and `shelley find samtools` all work as expected. -Some older tool versions exist in CVMFS but are not listed in the default sHPC registry. -Shelley handles this automatically, but if you need to do it manually, follow these steps. +#### Searching when you only know the task -**1. Check CVMFS for the version:** +Sometimes you know what you want to do but not which tool does it. That's what `search` +is for: ```bash -ls /cvmfs/singularity.galaxyproject.org/all/plink:1.90b4* +shelley search "quality control" +shelley search "variant calling" +shelley search "de novo assembly" ``` -**2. Get its checksum:** - -```bash -sha256sum /cvmfs/singularity.galaxyproject.org/all/plink:1.90b4--h0a6d026_2 -``` +Each result will show you the tool name and a brief description of what it does. **Shorter, more specific phrases tend to work better than full sentences.** -**3. Create a local registry entry:** - -```bash -sudo mkdir -p /apps/local/quay.io/biocontainers/plink +
+Example output +![](assets/img/shelley_search_de-novo-assembly.png) +
+
+
-# Fetch the existing remote recipe as a base -curl -fsSL \ - https://raw.githubusercontent.com/singularityhub/shpc-registry/main/quay.io/biocontainers/plink/container.yaml \ - -o /apps/local/quay.io/biocontainers/plink/container.yaml +{% include callout.html type="note" content="Search is under active development. All results are broad, and currently presented alphabetically. We recommend using shorter and more specific phrases as each extra word broadens the match rather than narrowing it, so a broad query like "dna sequence quality control" can return a large number of tools. Use the fewest, most specific terms you know, and remove words rather than adding them if you get too many results." %} -# Edit the YAML to add the missing version tag and its checksum -``` +### Checking every available version -**4. Register your local registry:** +By default `find` only shows the most recent versions of a tool. If you need to pin +an exact version for reproducibility, or to match a pipeline's requirements, you can add the +`-v` (verbose) flag to see every available container, sorted newest-first: ```bash -sudo shpc config add registry /apps/local +shelley find fastqc -v ``` -Verify your local registry appears first (it must take precedence over the remote registry): +
+Example output +![](assets/img/shelley_find_fastqc_v.png) +
+
+
-```bash -shpc config get registry -# ['/apps/local', 'https://github.com/singularityhub/shpc-registry'] -``` +### Building a module -**5. Install from CVMFS:** +Once you know the tool and version you want, build its Lmod module with `shelley build`: ```bash -sudo shpc install quay.io/biocontainers/plink:1.90b4--h0a6d026_2 \ - /cvmfs/singularity.galaxyproject.org/all/plink:1.90b4--h0a6d026_2 \ - --keep-path +shelley build fastqc ``` -{% include callout.html type="tip" content="If Shelley recognises the container path but the version is not in the registry, `Shelley build` handles the local registry creation automatically." %} +This installs the most recent available version by default. +
+Example output +![](assets/img/shelley_build_fastqc.png) +
+
---- +{% include callout.html type="tip" content="To install a specific version instead of the most recent one, give `build` the same `/` spec that `find -v` showed you, for example `shelley build fastqc/0.12.1`." %} -## Loading modules {#load-tool} +### Loading and running the tool -Once Shelley or sHPC has installed a module, load it with: +After a successful build, load the module the same way you would on any HPC system and +run the tool: ```bash -module load / +module load fastqc +fastqc --version +# FastQC v0.12.1 ``` -Verify the tool is ready: +That's the whole loop: search, find, build, load, and run. This is the same loop you'll use +for any tool in the BioContainers catalogue. -```bash - --version -``` - ---- +Once you've got the hang of this, the [**How to use Shelley**](shelley-howto) guide covers +the other use cases that will come in handy! ## Reference datasets {#reference-data} @@ -326,14 +245,17 @@ sHPC requires Singularity to execute containers. Try `search` with different keywords, for example `Shelley search "alignment"` instead of a specific tool name. If the container exists in CVMFS but Shelley does not index it, -fall back to the manual sHPC method above. - ---- +fall back to installing the module manually with `shpc install` - see the +[sHPC user guide](https://singularity-hpc.readthedocs.io/en/latest/getting_started/user-guide.html). ## Further reading {#further-reading} +**CVMFS, sHPC, and reference data** + - [sHPC user guide](https://singularity-hpc.readthedocs.io/en/latest/getting_started/user-guide.html) - [BioContainers registry](https://biocontainers.pro/registry) -- [Shelley on GitHub](https://github.com/Sydney-Informatics-Hub/Shelley) - [CVMFS documentation](https://cvmfs.readthedocs.io/en/stable/) - [Galaxy Project CVMFS repositories](https://galaxyproject.org/admin/cvmfs/) +- [**How to use Shelley**](shelley-howto) - snippets for various use cases +- [Full CLI reference](https://github.com/Sydney-Informatics-Hub/shelley/blob/main/docs/reference/cli.md) +- [Design rationale](https://github.com/Sydney-Informatics-Hub/shelley/tree/main/docs/explanation)