Addon A11y: Introduce parameters.a11y.test

code/addons/a11y/src/constants.ts

@@ -13,5 +13,3 @@ export const DOCUMENTATION_DISCREPANCY_LINK = `${DOCUMENTATION_LINK}#why-are-my-
 export const TEST_PROVIDER_ID = 'storybook/addon-a11y/test-provider';
 
 export const EVENTS = { RESULT, REQUEST, RUNNING, ERROR, MANUAL };
-
-export const A11Y_TEST_TAG = 'a11y-test';

code/addons/a11y/src/params.ts

@@ -1,5 +1,7 @@
 import type { ElementContext, RunOptions, Spec } from 'axe-core';
 
+type A11yTest = 'off' | 'todo' | 'error';
+
 export interface Setup {
   element?: ElementContext;
   config: Spec;
@@ -13,4 +15,5 @@ export interface A11yParameters {
   /** @deprecated Use globals.a11y.manual instead */
   manual?: boolean;
   disable?: boolean;
+  test?: A11yTest;
 }

code/addons/a11y/src/preview.test.tsx

@@ -4,7 +4,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest';
 import type { StoryContext } from 'storybook/internal/csf';
 
 import { run } from './a11yRunner';
-import { A11Y_TEST_TAG } from './constants';
 import { experimental_afterEach } from './preview';
 import { getIsVitestRunning, getIsVitestStandaloneRun } from './utils';
 
@@ -88,12 +87,13 @@ describe('afterEach', () => {
         addReport: vi.fn(),
       },
       parameters: {
-        a11y: {},
+        a11y: {
+          test: 'error',
+        },
       },
       globals: {
         a11y: {},
       },
-      tags: [A11Y_TEST_TAG],
       ...overrides,
     }) as any;
 
@@ -138,33 +138,39 @@ describe('afterEach', () => {
     });
   });
 
-  it('should report passed status when there are no violations', async () => {
-    const context = createContext();
+  it('should run accessibility checks and should report them as warnings', async () => {
+    const context = createContext({
+      parameters: {
+        a11y: {
+          test: 'todo',
+        },
+      },
+    });
     const result = {
-      violations: [],
+      violations,
     };
+
     mockedRun.mockResolvedValue(result as any);
+    mocks.getIsVitestStandaloneRun.mockReturnValue(false);
 
     await experimental_afterEach(context);
 
     expect(mockedRun).toHaveBeenCalledWith(context.parameters.a11y);
+
     expect(context.reporting.addReport).toHaveBeenCalledWith({
       type: 'a11y',
       version: 1,
       result,
-      status: 'passed',
+      status: 'warning',
     });
   });
 
-  it('should run accessibility checks if "a11y-test" flag is not available and is not running in Vitest', async () => {
-    const context = createContext({
-      tags: [],
-    });
+  it('should report passed status when there are no violations', async () => {
+    const context = createContext();
     const result = {
       violations: [],
     };
     mockedRun.mockResolvedValue(result as any);
-    vi.mocked(getIsVitestRunning).mockReturnValue(false);
 
     await experimental_afterEach(context);
 
@@ -222,11 +228,14 @@ describe('afterEach', () => {
     expect(context.reporting.addReport).not.toHaveBeenCalled();
   });
 
-  it('should not run accessibility checks if vitest is running and story is not tagged with a11ytest', async () => {
+  it('should not run accessibility checks when parameters.a11y.test is "off"', async () => {
     const context = createContext({
-      tags: [],
+      parameters: {
+        a11y: {
+          test: 'off',
+        },
+      },
     });
-    vi.mocked(getIsVitestRunning).mockReturnValue(true);
 
     await experimental_afterEach(context);
 

code/addons/a11y/src/preview.tsx

@@ -3,9 +3,8 @@ import type { AfterEach } from 'storybook/internal/types';
 import { expect } from '@storybook/test';
 
 import { run } from './a11yRunner';
-import { A11Y_TEST_TAG } from './constants';
 import type { A11yParameters } from './params';
-import { getIsVitestRunning, getIsVitestStandaloneRun } from './utils';
+import { getIsVitestStandaloneRun } from './utils';
 
 let vitestMatchersExtended = false;
 
@@ -14,20 +13,27 @@ export const experimental_afterEach: AfterEach<any> = async ({
   reporting,
   parameters,
   globals,
-  tags,
 }) => {
   const a11yParameter: A11yParameters | undefined = parameters.a11y;
   const a11yGlobals = globals.a11y;
 
   const shouldRunEnvironmentIndependent =
     a11yParameter?.manual !== true &&
     a11yParameter?.disable !== true &&
+    a11yParameter?.test !== 'off' &&
     a11yGlobals?.manual !== true;
 
-  if (shouldRunEnvironmentIndependent) {
-    if (getIsVitestRunning() && !tags.includes(A11Y_TEST_TAG)) {
-      return;
+  const getMode = (): (typeof reporting)['reports'][0]['status'] => {
+    switch (a11yParameter?.test) {
+      case 'todo':
+        return 'warning';
+      case 'error':
+      default:
+        return 'failed';
     }
+  };
+
+  if (shouldRunEnvironmentIndependent) {
     try {
       const result = await run(a11yParameter);
 
@@ -38,7 +44,7 @@ export const experimental_afterEach: AfterEach<any> = async ({
           type: 'a11y',
           version: 1,
           result: result,
-          status: hasViolations ? 'failed' : 'passed',
+          status: hasViolations ? getMode() : 'passed',
         });
 
         /**
@@ -50,7 +56,7 @@ export const experimental_afterEach: AfterEach<any> = async ({
          *   implement proper try catch handling.
          */
         if (getIsVitestStandaloneRun()) {
-          if (hasViolations) {
+          if (hasViolations && getMode() === 'failed') {
             if (!vitestMatchersExtended) {
               const { toHaveNoViolations } = await import('vitest-axe/matchers');
               expect.extend({ toHaveNoViolations });
@@ -88,3 +94,9 @@ export const initialGlobals = {
     manual: false,
   },
 };
+
+export const parameters = {
+  a11y: {
+    test: 'todo',
+  } as A11yParameters,
+};

code/addons/test/src/vitest-plugin/index.ts

@@ -224,15 +224,6 @@ export const storybookTest = async (options?: UserOptions): Promise<Plugin[]> =>
                   },
                 };
               },
-              getTags: () => {
-                const envConfig = JSON.parse(process.env.VITEST_STORYBOOK_CONFIG ?? '{}');
-
-                const shouldSetTag = process.env.VITEST_STORYBOOK
-                  ? (envConfig.a11y ?? false)
-                  : false;
-
-                return shouldSetTag ? ['a11y-test'] : [];
-              },
             },
             // if there is a test.browser config AND test.browser.screenshotFailures is not explicitly set, we set it to false
             ...(inputConfig_ONLY_MUTATE_WHEN_STRICTLY_NEEDED_OR_YOU_WILL_BE_FIRED.test?.browser &&

code/addons/test/src/vitest-plugin/test-utils.ts

@@ -17,11 +17,10 @@ import { setViewport } from './viewports';
 declare module '@vitest/browser/context' {
   interface BrowserCommands {
     getInitialGlobals: () => Promise<Record<string, any>>;
-    getTags: () => Promise<string[] | undefined>;
   }
 }
 
-const { getInitialGlobals, getTags } = server.commands;
+const { getInitialGlobals } = server.commands;
 
 export const testStory = (
   exportName: string,
@@ -34,7 +33,7 @@ export const testStory = (
     const composedStory = composeStory(
       annotations.story,
       annotations.meta!,
-      { initialGlobals: (await getInitialGlobals?.()) ?? {}, tags: await getTags?.() },
+      { initialGlobals: (await getInitialGlobals?.()) ?? {} },
       annotations.preview,
       exportName
     );

code/core/template/stories/tags-add.stories.ts

@@ -17,17 +17,11 @@ export default {
 
 export const Inheritance = {
   tags: ['story-one', '!vitest'],
-  play: async ({ canvasElement, tags }: PlayFunctionContext<any>) => {
+  play: async ({ canvasElement }: PlayFunctionContext<any>) => {
     const canvas = within(canvasElement);
-    if (tags.includes('a11y-test')) {
-      await expect(JSON.parse(canvas.getByTestId('pre').innerText)).toEqual({
-        tags: ['a11y-test', 'story-one'],
-      });
-    } else {
-      await expect(JSON.parse(canvas.getByTestId('pre').innerText)).toEqual({
-        tags: ['story-one'],
-      });
-    }
+    await expect(JSON.parse(canvas.getByTestId('pre').innerText)).toEqual({
+      tags: ['story-one'],
+    });
   },
   parameters: { chromatic: { disable: false } },
 };

code/core/template/stories/tags-config.stories.ts

@@ -17,25 +17,11 @@ export default {
 
 export const Inheritance = {
   tags: ['story-one', '!vitest'],
-  play: async ({ canvasElement, tags }: PlayFunctionContext<any>) => {
+  play: async ({ canvasElement }: PlayFunctionContext<any>) => {
     const canvas = within(canvasElement);
-    if (tags.includes('a11y-test')) {
-      await expect(JSON.parse(canvas.getByTestId('pre').innerText)).toEqual({
-        tags: [
-          'dev',
-          'test',
-          'a11y-test',
-          'component-one',
-          'component-two',
-          'autodocs',
-          'story-one',
-        ],
-      });
-    } else {
-      await expect(JSON.parse(canvas.getByTestId('pre').innerText)).toEqual({
-        tags: ['dev', 'test', 'component-one', 'component-two', 'autodocs', 'story-one'],
-      });
-    }
+    await expect(JSON.parse(canvas.getByTestId('pre').innerText)).toEqual({
+      tags: ['dev', 'test', 'component-one', 'component-two', 'autodocs', 'story-one'],
+    });
   },
   parameters: { chromatic: { disable: false } },
 };

code/core/template/stories/tags-remove.stories.ts

@@ -17,17 +17,11 @@ export default {
 
 export const Inheritance = {
   tags: ['story-one', '!vitest'],
-  play: async ({ canvasElement, tags }: PlayFunctionContext<any>) => {
+  play: async ({ canvasElement }: PlayFunctionContext<any>) => {
     const canvas = within(canvasElement);
-    if (tags.includes('a11y-test')) {
-      await expect(JSON.parse(canvas.getByTestId('pre').innerText)).toEqual({
-        tags: ['dev', 'test', 'a11y-test', 'component-one', 'autodocs', 'story-one'],
-      });
-    } else {
-      await expect(JSON.parse(canvas.getByTestId('pre').innerText)).toEqual({
-        tags: ['dev', 'test', 'component-one', 'autodocs', 'story-one'],
-      });
-    }
+    await expect(JSON.parse(canvas.getByTestId('pre').innerText)).toEqual({
+      tags: ['dev', 'test', 'component-one', 'autodocs', 'story-one'],
+    });
   },
   parameters: { chromatic: { disable: false } },
 };