feat(tokens): exclude selected paths from token accounting

README.md

@@ -36,6 +36,7 @@ Spec compliance is table stakes. `skill-validator` goes further: it checks that
     - [Flat skill layouts](#flat-skill-layouts)
     - [Allowing non-standard directories](#allowing-non-standard-directories)
     - [Allowing nesting at specific paths](#allowing-nesting-at-specific-paths)
+    - [Excluding paths from non-standard token accounting](#excluding-paths-from-non-standard-token-accounting)
   - [Link validation](#link-validation-validate-links)
   - [Content analysis](#content-analysis-analyze-content)
   - [Contamination analysis](#contamination-analysis-analyze-contamination)
@@ -190,6 +191,7 @@ skill-validator validate structure --allow-extra-frontmatter <path>
 skill-validator validate structure --allow-flat-layouts <path>
 skill-validator validate structure --allow-dirs=evals,testing <path>
 skill-validator validate structure --allow-nested-paths=assets/components <path>
+skill-validator validate structure --exclude-token-paths=site <path>
 ```
 
 Checks spec compliance: directory structure, frontmatter fields, token limits, skill ratio, code fence integrity, internal link validity, and orphan file detection.
@@ -202,6 +204,7 @@ Checks spec compliance: directory structure, frontmatter fields, token limits, s
 | `--allow-flat-layouts` | Allow files at the skill root without warnings (see [Flat skill layouts](#flat-skill-layouts)) |
 | `--allow-dirs=evals,testing` | Accept specific non-standard directories without warnings (see [Allowing non-standard directories](#allowing-non-standard-directories)) |
 | `--allow-nested-paths=assets/components` | Allow deep nesting only within specific skill-relative paths (see [Allowing nesting at specific paths](#allowing-nesting-at-specific-paths)) |
+| `--exclude-token-paths=site` | Exclude specific skill-relative subtrees from non-standard token accounting (see [Excluding paths from non-standard token accounting](#excluding-paths-from-non-standard-token-accounting)) |
 
 ```
 Validating skill: my-skill/
@@ -300,6 +303,7 @@ skill-validator check --allow-extra-frontmatter <path>
 skill-validator check --allow-flat-layouts <path>
 skill-validator check --allow-dirs=evals,testing <path>
 skill-validator check --allow-nested-paths=assets/components <path>
+skill-validator check --exclude-token-paths=site <path>
 ```
 
 Runs all checks (structure + links + content + contamination).
@@ -315,6 +319,7 @@ Runs all checks (structure + links + content + contamination).
 | `--allow-flat-layouts` | Allow files at the skill root without warnings (see [Flat skill layouts](#flat-skill-layouts)) |
 | `--allow-dirs=evals,testing` | Accept specific non-standard directories without warnings (see [Allowing non-standard directories](#allowing-non-standard-directories)) |
 | `--allow-nested-paths=assets/components` | Allow deep nesting only within specific skill-relative paths (see [Allowing nesting at specific paths](#allowing-nesting-at-specific-paths)) |
+| `--exclude-token-paths=site` | Exclude specific skill-relative subtrees from non-standard token accounting (see [Excluding paths from non-standard token accounting](#excluding-paths-from-non-standard-token-accounting)) |
 
 Valid check groups: `structure`, `links`, `content`, `contamination`.
 
@@ -785,6 +790,21 @@ Paths are relative to the skill root. The flag accepts a comma-separated list or
 
 This option affects only deep-nesting warnings. Structure checks outside the selected subtree and all frontmatter, orphan, token, Markdown, and link checks continue unchanged. When a nesting warning is suppressed, the report includes an informational result identifying the allowed path.
 
+**Excluding paths from non-standard token accounting**
+
+Some skill packages commit generated output for portability or as a reference contract even though agents do not load it as instruction or reference content. Use `--exclude-token-paths` to remove an explicitly selected subtree from non-standard token accounting:
+
+```
+skill-validator validate structure --allow-dirs=site --exclude-token-paths=site my-skill/
+skill-validator check --allow-dirs=site --exclude-token-paths=site,dist my-skill/
+```
+
+Excluded files do not appear in the "Other files" per-file table, do not contribute to its aggregate token limit, and do not contribute to the holistic standard-to-non-standard content ratio. Other non-standard files remain counted.
+
+Paths are relative to the skill root. The flag accepts a comma-separated list or can be repeated, normalizes `/` and `\` separators, rejects absolute paths or paths that escape the skill root, and observes path boundaries. For example, excluding `site` does not exclude `site-extra`.
+
+This option affects only non-standard token accounting. It does not allow an unknown directory, suppress orphan detection, or bypass frontmatter, structure, Markdown, or link validation. When files are excluded, the report includes an informational result identifying the excluded path.
+
 ### Link validation (`validate links`)
 
 - Checks external (HTTP/HTTPS) links only -- internal (relative) links are validated by `validate structure`

cmd/check.go

@@ -22,6 +22,7 @@ var (
 	checkAllowFlatLayouts      bool
 	checkAllowDirs             []string
 	checkAllowNestedPaths      []string
+	checkExcludeTokenPaths     []string
 )
 
 var checkCmd = &cobra.Command{
@@ -47,6 +48,8 @@ func init() {
 		"comma-separated list of directory names to accept without warnings (e.g. --allow-dirs=evals,testing)")
 	checkCmd.Flags().StringSliceVar(&checkAllowNestedPaths, "allow-nested-paths", nil,
 		"comma-separated skill-relative paths where deep nesting is allowed (e.g. --allow-nested-paths=assets/components)")
+	checkCmd.Flags().StringSliceVar(&checkExcludeTokenPaths, "exclude-token-paths", nil,
+		"comma-separated skill-relative subtrees to exclude from non-standard token accounting (e.g. --exclude-token-paths=site)")
 	rootCmd.AddCommand(checkCmd)
 }
 
@@ -67,6 +70,11 @@ func runCheck(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("invalid --allow-nested-paths: %w", err)
 	}
 
+	excludeTokenPaths, err := structure.NormalizeRelativePaths(checkExcludeTokenPaths)
+	if err != nil {
+		return fmt.Errorf("invalid --exclude-token-paths: %w", err)
+	}
+
 	enabled, err := resolveCheckGroups(checkOnly, checkSkip)
 	if err != nil {
 		return err
@@ -85,6 +93,7 @@ func runCheck(cmd *cobra.Command, args []string) error {
 			AllowFlatLayouts:      checkAllowFlatLayouts,
 			AllowDirs:             checkAllowDirs,
 			AllowNestedPaths:      allowNestedPaths,
+			ExcludeTokenPaths:     excludeTokenPaths,
 		},
 	}
 	eopts := exitOpts{strict: strictCheck}

cmd/exitcode_integration_test.go

@@ -186,6 +186,13 @@ func TestSliceFlags(t *testing.T) {
 			wantStdout: "deep nesting allowed: assets/components/",
 			noStdout:   "deep nesting detected:",
 		},
+		{
+			name:       "check exclude-token-paths removes selected subtree only",
+			args:       []string{"check", "--only=structure", "--skip-orphans", "--allow-dirs=site,site-extra", "--exclude-token-paths=site", fixture(t, "token-exclusion-skill")},
+			wantCode:   0,
+			wantStdout: "site-extra/keep.md",
+			noStdout:   "site/generated.md",
+		},
 	}
 
 	for _, tt := range tests {
@@ -228,6 +235,24 @@ func TestAllowNestedPathsInvalidPath(t *testing.T) {
 	}
 }
 
+func TestExcludeTokenPathsInvalidPath(t *testing.T) {
+	bin := buildBinary(t)
+
+	for _, args := range [][]string{
+		{"validate", "structure", "--exclude-token-paths=/site", fixture(t, "valid-skill")},
+		{"check", "--exclude-token-paths=../site", fixture(t, "valid-skill")},
+	} {
+		cmd := exec.Command(bin, args...)
+		out, _ := cmd.CombinedOutput()
+		if got := cmd.ProcessState.ExitCode(); got != 3 {
+			t.Errorf("exit code = %d, want 3 (args: %v)\noutput: %s", got, args, out)
+		}
+		if !strings.Contains(string(out), "invalid --exclude-token-paths") {
+			t.Errorf("expected invalid path error, got:\n%s", out)
+		}
+	}
+}
+
 func TestAllowNestedPathsOutputFormats(t *testing.T) {
 	bin := buildBinary(t)
 
@@ -255,3 +280,35 @@ func TestAllowNestedPathsOutputFormats(t *testing.T) {
 		})
 	}
 }
+
+func TestExcludeTokenPathsOutputFormats(t *testing.T) {
+	bin := buildBinary(t)
+
+	for _, format := range []string{"text", "json", "markdown"} {
+		t.Run(format, func(t *testing.T) {
+			args := []string{
+				"validate", "structure",
+				"--skip-orphans",
+				"--allow-dirs=site,site-extra",
+				"--exclude-token-paths=site",
+				"--output=" + format,
+				fixture(t, "token-exclusion-skill"),
+			}
+			cmd := exec.Command(bin, args...)
+			out, _ := cmd.CombinedOutput()
+
+			if got := cmd.ProcessState.ExitCode(); got != 0 {
+				t.Errorf("exit code = %d, want 0\noutput: %s", got, out)
+			}
+			if !strings.Contains(string(out), "excluded from token accounting: site") {
+				t.Errorf("expected exclusion info in %s output, got:\n%s", format, out)
+			}
+			if strings.Contains(string(out), "site/generated.md") {
+				t.Errorf("excluded file appeared in %s output:\n%s", format, out)
+			}
+			if !strings.Contains(string(out), "site-extra/keep.md") {
+				t.Errorf("non-excluded sibling missing from %s output:\n%s", format, out)
+			}
+		})
+	}
+}

cmd/validate_structure.go

@@ -16,6 +16,7 @@ var (
 	structAllowFlatLayouts      bool
 	structAllowDirs             []string
 	structAllowNestedPaths      []string
+	structExcludeTokenPaths     []string
 )
 
 var validateStructureCmd = &cobra.Command{
@@ -38,6 +39,8 @@ func init() {
 		"comma-separated list of directory names to accept without warnings (e.g. --allow-dirs=evals,testing)")
 	validateStructureCmd.Flags().StringSliceVar(&structAllowNestedPaths, "allow-nested-paths", nil,
 		"comma-separated skill-relative paths where deep nesting is allowed (e.g. --allow-nested-paths=assets/components)")
+	validateStructureCmd.Flags().StringSliceVar(&structExcludeTokenPaths, "exclude-token-paths", nil,
+		"comma-separated skill-relative subtrees to exclude from non-standard token accounting (e.g. --exclude-token-paths=site)")
 	validateCmd.AddCommand(validateStructureCmd)
 }
 
@@ -47,6 +50,11 @@ func runValidateStructure(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("invalid --allow-nested-paths: %w", err)
 	}
 
+	excludeTokenPaths, err := structure.NormalizeRelativePaths(structExcludeTokenPaths)
+	if err != nil {
+		return fmt.Errorf("invalid --exclude-token-paths: %w", err)
+	}
+
 	_, mode, dirs, err := detectAndResolve(args)
 	if err != nil {
 		return err
@@ -58,6 +66,7 @@ func runValidateStructure(cmd *cobra.Command, args []string) error {
 		AllowFlatLayouts:      structAllowFlatLayouts,
 		AllowDirs:             structAllowDirs,
 		AllowNestedPaths:      allowNestedPaths,
+		ExcludeTokenPaths:     excludeTokenPaths,
 	}
 	eopts := exitOpts{strict: strictStructure}
 

structure/paths.go

@@ -27,6 +27,19 @@ func NormalizeRelativePaths(paths []string) ([]string, error) {
 	return normalized, nil
 }
 
+func normalizedRelativePaths(paths []string) []string {
+	normalized := make([]string, 0, len(paths))
+	seen := make(map[string]bool, len(paths))
+	for _, raw := range paths {
+		value, err := normalizeRelativePath(raw)
+		if err == nil && !seen[value] {
+			normalized = append(normalized, value)
+			seen[value] = true
+		}
+	}
+	return normalized
+}
+
 func normalizeRelativePath(raw string) (string, error) {
 	value := strings.TrimSpace(strings.ReplaceAll(raw, `\`, "/"))
 	if value == "" {

structure/paths_test.go

@@ -7,11 +7,11 @@ import (
 
 func TestNormalizeRelativePaths(t *testing.T) {
 	t.Run("normalizes separators and redundant components", func(t *testing.T) {
-		got, err := NormalizeRelativePaths([]string{`assets\components`, "references/./generated"})
+		got, err := NormalizeRelativePaths([]string{`generated\site`, "docs/./generated"})
 		if err != nil {
 			t.Fatal(err)
 		}
-		want := []string{"assets/components", "references/generated"}
+		want := []string{"generated/site", "docs/generated"}
 		if !reflect.DeepEqual(got, want) {
 			t.Fatalf("NormalizeRelativePaths() = %#v, want %#v", got, want)
 		}
@@ -21,11 +21,11 @@ func TestNormalizeRelativePaths(t *testing.T) {
 		name string
 		path string
 	}{
-		{name: "Unix absolute", path: "/assets/components"},
-		{name: "Windows drive absolute", path: `C:\assets\components`},
-		{name: "Windows UNC absolute", path: `\\server\share\components`},
-		{name: "escapes root", path: "../components"},
-		{name: "escapes root after clean", path: "assets/../../components"},
+		{name: "Unix absolute", path: "/site"},
+		{name: "Windows drive absolute", path: `C:\site`},
+		{name: "Windows UNC absolute", path: `\\server\share\site`},
+		{name: "escapes root", path: "../site"},
+		{name: "escapes root after clean", path: "generated/../../site"},
 	} {
 		t.Run("rejects "+tt.name, func(t *testing.T) {
 			if _, err := NormalizeRelativePaths([]string{tt.path}); err == nil {

structure/tokens.go

@@ -171,7 +171,12 @@ func CheckTokens(dir, body string, opts Options) ([]types.Result, []types.TokenC
 	}
 
 	// Count tokens in non-standard files
-	otherCounts := countOtherFiles(dir, enc, opts)
+	exclusions := newTokenPathExclusions(opts.ExcludeTokenPaths)
+	otherCounts := countOtherFiles(dir, enc, opts, exclusions)
+
+	for _, excludedPath := range exclusions.appliedPaths() {
+		results = append(results, ctx.Infof("excluded from token accounting: %s", excludedPath))
+	}
 
 	// Check other-files aggregate limits
 	otherTotal := 0
@@ -278,7 +283,41 @@ func countAssetFiles(dir string, enc tokenizer.Codec) []types.TokenCount {
 	return counts
 }
 
-func countOtherFiles(dir string, enc tokenizer.Codec, opts Options) []types.TokenCount {
+type tokenPathExclusions struct {
+	paths   []string
+	applied map[string]bool
+}
+
+func newTokenPathExclusions(paths []string) *tokenPathExclusions {
+	normalized := normalizedRelativePaths(paths)
+	return &tokenPathExclusions{
+		paths:   normalized,
+		applied: make(map[string]bool, len(normalized)),
+	}
+}
+
+func (e *tokenPathExclusions) excludes(candidate string) bool {
+	candidate = filepath.ToSlash(candidate)
+	for _, excludedPath := range e.paths {
+		if pathInSubtree(candidate, excludedPath) {
+			e.applied[excludedPath] = true
+			return true
+		}
+	}
+	return false
+}
+
+func (e *tokenPathExclusions) appliedPaths() []string {
+	var paths []string
+	for _, excludedPath := range e.paths {
+		if e.applied[excludedPath] {
+			paths = append(paths, excludedPath)
+		}
+	}
+	return paths
+}
+
+func countOtherFiles(dir string, enc tokenizer.Codec, opts Options, exclusions *tokenPathExclusions) []types.TokenCount {
 	var counts []types.TokenCount
 
 	entries, err := os.ReadDir(dir)
@@ -296,15 +335,21 @@ func countOtherFiles(dir string, enc tokenizer.Codec, opts Options) []types.Toke
 			if standardDirs[strings.ToLower(name)] {
 				continue
 			}
+			if exclusions.excludes(name) {
+				continue
+			}
 			// Walk files in unknown directory
-			counts = append(counts, countFilesInDir(dir, name, enc)...)
+			counts = append(counts, countFilesInDir(dir, name, enc, exclusions)...)
 		} else {
 			if !entry.Type().IsRegular() {
 				continue
 			}
 			if standardRootFiles[strings.ToLower(name)] || opts.AllowFlatLayouts {
 				continue
 			}
+			if exclusions.excludes(name) {
+				continue
+			}
 			if binaryExtensions[strings.ToLower(filepath.Ext(name))] {
 				continue
 			}
@@ -320,7 +365,7 @@ func countOtherFiles(dir string, enc tokenizer.Codec, opts Options) []types.Toke
 	return counts
 }
 
-func countFilesInDir(rootDir, dirName string, enc tokenizer.Codec) []types.TokenCount {
+func countFilesInDir(rootDir, dirName string, enc tokenizer.Codec, exclusions *tokenPathExclusions) []types.TokenCount {
 	var counts []types.TokenCount
 	fullDir := filepath.Join(rootDir, dirName)
 
@@ -332,19 +377,31 @@ func countFilesInDir(rootDir, dirName string, enc tokenizer.Codec) []types.Token
 			if strings.HasPrefix(info.Name(), ".") && path != fullDir {
 				return filepath.SkipDir
 			}
+			if path != fullDir {
+				rel, relErr := filepath.Rel(rootDir, path)
+				if relErr == nil && exclusions.excludes(rel) {
+					return filepath.SkipDir
+				}
+			}
 			return nil
 		}
 		if strings.HasPrefix(info.Name(), ".") {
 			return nil
 		}
+		rel, relErr := filepath.Rel(rootDir, path)
+		if relErr != nil {
+			return nil
+		}
+		if exclusions.excludes(rel) {
+			return nil
+		}
 		if binaryExtensions[strings.ToLower(filepath.Ext(info.Name()))] {
 			return nil
 		}
 		data, err := readFileWithCap(path)
 		if err != nil {
 			return nil
 		}
-		rel, _ := filepath.Rel(rootDir, path)
 		tokens, _, _ := enc.Encode(string(data))
 		counts = append(counts, types.TokenCount{File: filepath.ToSlash(rel), Tokens: len(tokens)})
 		return nil