From e3a9952d605d3a859a8df0e579487d47a957e93d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Jastrz=C4=99bski?= Date: Thu, 29 Feb 2024 14:38:22 +0100 Subject: [PATCH 1/9] crate extract work in progress --- tests/tester/src/exec/mod.rs | 3 +- tests/tester/src/git.rs | 142 ++++++ tests/tester/src/main.rs | 444 +----------------- .../src/{ => test252_parser}/edition.rs | 4 +- tests/tester/src/test252_parser/mod.rs | 8 + tests/tester/src/{ => test252_parser}/read.rs | 80 +--- tests/tester/src/test252_parser/structs.rs | 366 +++++++++++++++ 7 files changed, 543 insertions(+), 504 deletions(-) create mode 100644 tests/tester/src/git.rs rename tests/tester/src/{ => test252_parser}/edition.rs (99%) create mode 100644 tests/tester/src/test252_parser/mod.rs rename tests/tester/src/{ => test252_parser}/read.rs (70%) create mode 100644 tests/tester/src/test252_parser/structs.rs diff --git a/tests/tester/src/exec/mod.rs b/tests/tester/src/exec/mod.rs index f57d55aff89..dd54bdd7c22 100644 --- a/tests/tester/src/exec/mod.rs +++ b/tests/tester/src/exec/mod.rs @@ -3,8 +3,7 @@ mod js262; use crate::{ - read::ErrorType, Harness, Outcome, Phase, SpecEdition, Statistics, SuiteResult, Test, - TestFlags, TestOutcomeResult, TestResult, TestSuite, VersionedStats, + test252_parser::{ErrorType, Harness, Outcome, Phase, SpecEdition, Test, TestSuite}, Statistics, SuiteResult, TestFlags, TestOutcomeResult, TestResult, VersionedStats }; use boa_engine::{ builtins::promise::PromiseState, diff --git a/tests/tester/src/git.rs b/tests/tester/src/git.rs new file mode 100644 index 00000000000..9288838d9fc --- /dev/null +++ b/tests/tester/src/git.rs @@ -0,0 +1,142 @@ +use color_eyre::{eyre::bail, Result}; +use std::{path::Path, process::Command}; + +/// Returns the commit hash and commit message of the provided branch name. +fn get_last_branch_commit(directory: &str, branch: &str, verbose: u8) -> Result<(String, String)> { + if verbose > 1 { + println!("Getting last commit on '{branch}' branch"); + } + let result = Command::new("git") + .arg("log") + .args(["-n", "1"]) + .arg("--pretty=format:%H %s") + .arg(branch) + .current_dir(directory) + .output()?; + + if !result.status.success() { + bail!( + "{directory} getting commit hash and message failed with return code {:?}", + result.status.code() + ); + } + + let output = std::str::from_utf8(&result.stdout)?.trim(); + + let (hash, message) = output + .split_once(' ') + .expect("git log output to contain hash and message"); + + Ok((hash.into(), message.into())) +} + +fn reset_commit(directory: &str, commit: &str, verbose: u8) -> Result<()> { + if verbose != 0 { + println!("Reset {directory} to commit: {commit}..."); + } + + let result = Command::new("git") + .arg("reset") + .arg("--hard") + .arg(commit) + .current_dir(directory) + .status()?; + + if !result.success() { + bail!( + "{directory} commit {commit} checkout failed with return code: {:?}", + result.code() + ); + } + + Ok(()) +} + +pub(super) fn clone( + directory: &str, + repor_url: &str, // "https://github.com/tc39/test262" + baranch: &str, // "origin/main" + commit: Option<&str>, + verbose: u8, +) -> Result<()> { + let update = commit.is_none(); + + if Path::new(directory).is_dir() { + let (current_commit_hash, current_commit_message) = + get_last_branch_commit(directory, "HEAD", verbose)?; + + if let Some(commit) = commit { + if current_commit_hash == commit { + return Ok(()); + } + } + + if verbose != 0 { + println!("Fetching latest {directory} commits..."); + } + let result = Command::new("git") + .arg("fetch") + .current_dir(directory) + .status()?; + + if !result.success() { + bail!( + "{directory} fetching latest failed with return code {:?}", + result.code() + ); + } + + if let Some(commit) = commit { + println!("{directory} switching to commit {commit}..."); + reset_commit(directory, commit, verbose)?; + return Ok(()); + } + + if verbose != 0 { + println!("Checking latest {directory} with current HEAD..."); + } + let (latest_commit_hash, latest_commit_message) = + get_last_branch_commit(directory, baranch, verbose)?; + + if current_commit_hash != latest_commit_hash { + if update { + println!("Updating {directory} repository:"); + } else { + println!("Warning {directory} repository is not in sync, use '--test262-commit latest' to automatically update it:"); + } + + println!(" Current commit: {current_commit_hash} {current_commit_message}"); + println!(" Latest commit: {latest_commit_hash} {latest_commit_message}"); + + if update { + reset_commit(&directory, &latest_commit_hash, verbose)?; + } + } + + return Ok(()); + } + + println!("Cloning {repor_url} into {directory} ..."); + let result = Command::new("git") + .arg("clone") + .arg(repor_url) + .arg(directory) + .status()?; + + if !result.success() { + bail!( + "Cloning {repor_url} repository failed with return code {:?}", + result.code() + ); + } + + if let Some(commit) = commit { + if verbose != 0 { + println!("Reset {repor_url} to commit: {commit}..."); + } + + reset_commit(directory, commit, verbose)?; + } + + Ok(()) +} diff --git a/tests/tester/src/main.rs b/tests/tester/src/main.rs index 649f47837f5..7536f5c13ed 100644 --- a/tests/tester/src/main.rs +++ b/tests/tester/src/main.rs @@ -9,35 +9,29 @@ clippy::cast_precision_loss )] -mod edition; mod exec; -mod read; +mod git; +mod test252_parser; mod results; -use self::{ - read::{read_harness, read_suite, read_test, MetaData, Negative, TestFlag}, - results::{compare_results, write_json}, -}; -use bitflags::bitflags; +use test252_parser::{TestFlags, SpecEdition, Ignored, read_harness, read_suite, read_test}; +use self::results::{compare_results, write_json}; + use boa_engine::optimizer::OptimizerOptions; use clap::{ArgAction, Parser, ValueHint}; use color_eyre::{ - eyre::{bail, eyre, WrapErr}, + eyre::{bail, WrapErr}, Result, }; use colored::Colorize; -use edition::SpecEdition; use once_cell::sync::Lazy; -use read::ErrorType; -use rustc_hash::{FxHashMap, FxHashSet}; +use rustc_hash::FxHashSet; use serde::{ - de::{Unexpected, Visitor}, Deserialize, Deserializer, Serialize, }; use std::{ ops::{Add, AddAssign}, path::{Path, PathBuf}, - process::Command, time::Instant, }; @@ -64,55 +58,6 @@ impl Config { } } -/// Structure to allow defining ignored tests, features and files that should -/// be ignored even when reading. -#[derive(Debug, Deserialize)] -struct Ignored { - #[serde(default)] - tests: FxHashSet>, - #[serde(default)] - features: FxHashSet>, - #[serde(default = "TestFlags::empty")] - flags: TestFlags, -} - -impl Ignored { - /// Checks if the ignore list contains the given test name in the list of - /// tests to ignore. - pub(crate) fn contains_test(&self, test: &str) -> bool { - self.tests.contains(test) - } - - /// Checks if the ignore list contains the given feature name in the list - /// of features to ignore. - pub(crate) fn contains_feature(&self, feature: &str) -> bool { - if self.features.contains(feature) { - return true; - } - // Some features are an accessor instead of a simple feature name e.g. `Intl.DurationFormat`. - // This ensures those are also ignored. - feature - .split('.') - .next() - .map(|feat| self.features.contains(feat)) - .unwrap_or_default() - } - - pub(crate) const fn contains_any_flag(&self, flags: TestFlags) -> bool { - flags.intersects(self.flags) - } -} - -impl Default for Ignored { - fn default() -> Self { - Self { - tests: FxHashSet::default(), - features: FxHashSet::default(), - flags: TestFlags::empty(), - } - } -} - /// Boa test262 tester #[derive(Debug, Parser)] #[command(author, version, about, name = "Boa test262 tester")] @@ -253,140 +198,15 @@ fn main() -> Result<()> { } } -/// Returns the commit hash and commit message of the provided branch name. -fn get_last_branch_commit(branch: &str, verbose: u8) -> Result<(String, String)> { - if verbose > 1 { - println!("Getting last commit on '{branch}' branch"); - } - let result = Command::new("git") - .arg("log") - .args(["-n", "1"]) - .arg("--pretty=format:%H %s") - .arg(branch) - .current_dir(DEFAULT_TEST262_DIRECTORY) - .output()?; - - if !result.status.success() { - bail!( - "test262 getting commit hash and message failed with return code {:?}", - result.status.code() - ); - } - - let output = std::str::from_utf8(&result.stdout)?.trim(); - - let (hash, message) = output - .split_once(' ') - .expect("git log output to contain hash and message"); - - Ok((hash.into(), message.into())) -} - -fn reset_test262_commit(commit: &str, verbose: u8) -> Result<()> { - if verbose != 0 { - println!("Reset test262 to commit: {commit}..."); - } - - let result = Command::new("git") - .arg("reset") - .arg("--hard") - .arg(commit) - .current_dir(DEFAULT_TEST262_DIRECTORY) - .status()?; - - if !result.success() { - bail!( - "test262 commit {commit} checkout failed with return code: {:?}", - result.code() - ); - } - - Ok(()) -} - fn clone_test262(commit: Option<&str>, verbose: u8) -> Result<()> { const TEST262_REPOSITORY: &str = "https://github.com/tc39/test262"; - - let update = commit.is_none(); - - if Path::new(DEFAULT_TEST262_DIRECTORY).is_dir() { - let (current_commit_hash, current_commit_message) = - get_last_branch_commit("HEAD", verbose)?; - - if let Some(commit) = commit { - if current_commit_hash == commit { - return Ok(()); - } - } - - if verbose != 0 { - println!("Fetching latest test262 commits..."); - } - let result = Command::new("git") - .arg("fetch") - .current_dir(DEFAULT_TEST262_DIRECTORY) - .status()?; - - if !result.success() { - bail!( - "Test262 fetching latest failed with return code {:?}", - result.code() - ); - } - - if let Some(commit) = commit { - println!("Test262 switching to commit {commit}..."); - reset_test262_commit(commit, verbose)?; - return Ok(()); - } - - if verbose != 0 { - println!("Checking latest Test262 with current HEAD..."); - } - let (latest_commit_hash, latest_commit_message) = - get_last_branch_commit("origin/main", verbose)?; - - if current_commit_hash != latest_commit_hash { - if update { - println!("Updating Test262 repository:"); - } else { - println!("Warning Test262 repository is not in sync, use '--test262-commit latest' to automatically update it:"); - } - - println!(" Current commit: {current_commit_hash} {current_commit_message}"); - println!(" Latest commit: {latest_commit_hash} {latest_commit_message}"); - - if update { - reset_test262_commit(&latest_commit_hash, verbose)?; - } - } - - return Ok(()); - } - - println!("Cloning test262..."); - let result = Command::new("git") - .arg("clone") - .arg(TEST262_REPOSITORY) - .arg(DEFAULT_TEST262_DIRECTORY) - .status()?; - - if !result.success() { - bail!( - "Cloning Test262 repository failed with return code {:?}", - result.code() - ); - } - - if let Some(commit) = commit { - if verbose != 0 { - println!("Reset Test262 to commit: {commit}..."); - } - - reset_test262_commit(commit, verbose)?; - } - - Ok(()) + git::clone( + DEFAULT_TEST262_DIRECTORY, + TEST262_REPOSITORY, + &"origin/main", + commit, + verbose, + ) } /// Runs the full test suite. @@ -523,30 +343,6 @@ fn run_test_suite( Ok(()) } -/// All the harness include files. -#[derive(Debug, Clone)] -struct Harness { - assert: HarnessFile, - sta: HarnessFile, - doneprint_handle: HarnessFile, - includes: FxHashMap, HarnessFile>, -} - -#[derive(Debug, Clone)] -struct HarnessFile { - content: Box, - path: Box, -} - -/// Represents a test suite. -#[derive(Debug, Clone)] -struct TestSuite { - name: Box, - path: Box, - suites: Box<[TestSuite]>, - tests: Box<[Test]>, -} - /// Represents a tests statistic #[derive(Default, Debug, Copy, Clone, Serialize, Deserialize)] struct Statistics { @@ -778,212 +574,12 @@ enum TestOutcomeResult { Panic, } -/// Represents a test. -#[derive(Debug, Clone)] -#[allow(dead_code)] -struct Test { - name: Box, - path: Box, - description: Box, - esid: Option>, - edition: SpecEdition, - flags: TestFlags, - information: Box, - expected_outcome: Outcome, - features: FxHashSet>, - includes: FxHashSet>, - locale: Locale, - ignored: bool, -} - -impl Test { - /// Creates a new test. - fn new(name: N, path: C, metadata: MetaData) -> Result - where - N: Into>, - C: Into>, - { - let edition = SpecEdition::from_test_metadata(&metadata) - .map_err(|feats| eyre!("test metadata contained unknown features: {feats:?}"))?; - - Ok(Self { - edition, - name: name.into(), - description: metadata.description, - esid: metadata.esid, - flags: metadata.flags.into(), - information: metadata.info, - features: metadata.features.into_vec().into_iter().collect(), - expected_outcome: Outcome::from(metadata.negative), - includes: metadata.includes.into_vec().into_iter().collect(), - locale: metadata.locale, - path: path.into(), - ignored: false, - }) - } - - /// Sets the test as ignored. - #[inline] - fn set_ignored(&mut self) { - self.ignored = true; - } - - /// Checks if this is a module test. - #[inline] - const fn is_module(&self) -> bool { - self.flags.contains(TestFlags::MODULE) - } -} - -/// An outcome for a test. -#[derive(Debug, Clone)] -enum Outcome { - Positive, - Negative { phase: Phase, error_type: ErrorType }, -} - -impl Default for Outcome { - fn default() -> Self { - Self::Positive - } -} - -impl From> for Outcome { - fn from(neg: Option) -> Self { - neg.map(|neg| Self::Negative { - phase: neg.phase, - error_type: neg.error_type, - }) - .unwrap_or_default() - } -} +#[cfg(test)] +mod tests { -bitflags! { - #[derive(Debug, Clone, Copy)] - struct TestFlags: u16 { - const STRICT = 0b0_0000_0001; - const NO_STRICT = 0b0_0000_0010; - const MODULE = 0b0_0000_0100; - const RAW = 0b0_0000_1000; - const ASYNC = 0b0_0001_0000; - const GENERATED = 0b0_0010_0000; - const CAN_BLOCK_IS_FALSE = 0b0_0100_0000; - const CAN_BLOCK_IS_TRUE = 0b0_1000_0000; - const NON_DETERMINISTIC = 0b1_0000_0000; + #[test] + #[ignore = "manual testing"] + fn should_clone_test262() { + super::clone_test262(None, 0).unwrap(); } } - -impl Default for TestFlags { - fn default() -> Self { - Self::STRICT | Self::NO_STRICT - } -} - -impl From for TestFlags { - fn from(flag: TestFlag) -> Self { - match flag { - TestFlag::OnlyStrict => Self::STRICT, - TestFlag::NoStrict => Self::NO_STRICT, - TestFlag::Module => Self::MODULE, - TestFlag::Raw => Self::RAW, - TestFlag::Async => Self::ASYNC, - TestFlag::Generated => Self::GENERATED, - TestFlag::CanBlockIsFalse => Self::CAN_BLOCK_IS_FALSE, - TestFlag::CanBlockIsTrue => Self::CAN_BLOCK_IS_TRUE, - TestFlag::NonDeterministic => Self::NON_DETERMINISTIC, - } - } -} - -impl From for TestFlags -where - T: AsRef<[TestFlag]>, -{ - fn from(flags: T) -> Self { - let flags = flags.as_ref(); - if flags.is_empty() { - Self::default() - } else { - let mut result = Self::empty(); - for flag in flags { - result |= Self::from(*flag); - } - - if !result.intersects(Self::default()) { - result |= Self::default(); - } - - result - } - } -} - -impl<'de> Deserialize<'de> for TestFlags { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - struct FlagsVisitor; - - impl<'de> Visitor<'de> for FlagsVisitor { - type Value = TestFlags; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(formatter, "a sequence of flags") - } - - fn visit_seq(self, mut seq: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - let mut flags = TestFlags::empty(); - while let Some(elem) = seq.next_element::()? { - flags |= elem.into(); - } - Ok(flags) - } - } - - struct RawFlagsVisitor; - - impl Visitor<'_> for RawFlagsVisitor { - type Value = TestFlags; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(formatter, "a flags number") - } - - fn visit_u16(self, v: u16) -> Result - where - E: serde::de::Error, - { - TestFlags::from_bits(v).ok_or_else(|| { - E::invalid_value(Unexpected::Unsigned(v.into()), &"a valid flag number") - }) - } - } - - if deserializer.is_human_readable() { - deserializer.deserialize_seq(FlagsVisitor) - } else { - deserializer.deserialize_u16(RawFlagsVisitor) - } - } -} - -/// Phase for an error. -#[derive(Debug, Clone, Copy, Deserialize)] -#[serde(rename_all = "lowercase")] -enum Phase { - Parse, - Resolution, - Runtime, -} - -/// Locale information structure. -#[derive(Debug, Default, Clone, Deserialize)] -#[serde(transparent)] -#[allow(dead_code)] -struct Locale { - locale: Box<[Box]>, -} diff --git a/tests/tester/src/edition.rs b/tests/tester/src/test252_parser/edition.rs similarity index 99% rename from tests/tester/src/edition.rs rename to tests/tester/src/test252_parser/edition.rs index 1df3f8ba68f..4f8c1dedacb 100644 --- a/tests/tester/src/edition.rs +++ b/tests/tester/src/test252_parser/edition.rs @@ -7,7 +7,7 @@ use std::fmt::Display; use serde_repr::{Deserialize_repr, Serialize_repr}; -use crate::read::{MetaData, TestFlag}; +use super::structs::{MetaData, TestFlag}; /// Minimum edition required by a specific feature in the `test262` repository. static FEATURE_EDITION: phf::Map<&'static str, SpecEdition> = phf::phf_map! { @@ -259,7 +259,7 @@ static FEATURE_EDITION: phf::Map<&'static str, SpecEdition> = phf::phf_map! { clap::ValueEnum, )] #[repr(u8)] -pub(crate) enum SpecEdition { +pub enum SpecEdition { /// ECMAScript 5.1 Edition /// /// diff --git a/tests/tester/src/test252_parser/mod.rs b/tests/tester/src/test252_parser/mod.rs new file mode 100644 index 00000000000..d62e9c67219 --- /dev/null +++ b/tests/tester/src/test252_parser/mod.rs @@ -0,0 +1,8 @@ +mod structs; +mod edition; +mod read; + +pub use structs::{TestFlags, Ignored, ErrorType, Harness, Outcome, Phase, Test, TestSuite}; +pub use read::{read_suite, read_harness, read_test}; + +pub use edition::SpecEdition; diff --git a/tests/tester/src/read.rs b/tests/tester/src/test252_parser/read.rs similarity index 70% rename from tests/tester/src/read.rs rename to tests/tester/src/test252_parser/read.rs index b486b381b2c..5c8eecb8ae3 100644 --- a/tests/tester/src/read.rs +++ b/tests/tester/src/test252_parser/read.rs @@ -1,93 +1,21 @@ //! Module to read the list of test suites from disk. -use crate::{HarnessFile, Ignored}; +use super::structs::{Harness, HarnessFile, MetaData, Ignored, Test, TestSuite}; -use super::{Harness, Locale, Phase, Test, TestSuite}; use color_eyre::{ eyre::{eyre, WrapErr}, Result, }; use rustc_hash::FxHashMap; -use serde::Deserialize; use std::{ ffi::OsStr, fs, io, path::{Path, PathBuf}, }; -/// Representation of the YAML metadata in Test262 tests. -#[derive(Debug, Clone, Deserialize)] -pub(super) struct MetaData { - pub(super) description: Box, - pub(super) esid: Option>, - #[allow(dead_code)] - pub(super) es5id: Option>, - pub(super) es6id: Option>, - #[serde(default)] - pub(super) info: Box, - #[serde(default)] - pub(super) features: Box<[Box]>, - #[serde(default)] - pub(super) includes: Box<[Box]>, - #[serde(default)] - pub(super) flags: Box<[TestFlag]>, - #[serde(default)] - pub(super) negative: Option, - #[serde(default)] - pub(super) locale: Locale, -} - -/// Negative test information structure. -#[derive(Debug, Clone, Deserialize)] -pub(super) struct Negative { - pub(super) phase: Phase, - #[serde(rename = "type")] - pub(super) error_type: ErrorType, -} - -/// All possible error types -#[derive(Debug, Copy, Clone, Deserialize, PartialEq, Eq)] -#[allow(clippy::enum_variant_names)] // Better than appending `rename` to all variants -pub(super) enum ErrorType { - Test262Error, - SyntaxError, - ReferenceError, - RangeError, - TypeError, -} - -impl ErrorType { - pub(super) const fn as_str(self) -> &'static str { - match self { - Self::Test262Error => "Test262Error", - Self::SyntaxError => "SyntaxError", - Self::ReferenceError => "ReferenceError", - Self::RangeError => "RangeError", - Self::TypeError => "TypeError", - } - } -} - -/// Individual test flag. -#[derive(Debug, Clone, Copy, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(super) enum TestFlag { - OnlyStrict, - NoStrict, - Module, - Raw, - Async, - Generated, - #[serde(rename = "CanBlockIsFalse")] - CanBlockIsFalse, - #[serde(rename = "CanBlockIsTrue")] - CanBlockIsTrue, - #[serde(rename = "non-deterministic")] - NonDeterministic, -} /// Reads the Test262 defined bindings. -pub(super) fn read_harness(test262_path: &Path) -> Result { +pub fn read_harness(test262_path: &Path) -> Result { fn read_harness_file(path: PathBuf) -> Result { let content = fs::read_to_string(path.as_path()) .wrap_err_with(|| format!("error reading the harness file `{}`", path.display()))?; @@ -128,7 +56,7 @@ pub(super) fn read_harness(test262_path: &Path) -> Result { } /// Reads a test suite in the given path. -pub(super) fn read_suite( +pub fn read_suite( path: &Path, ignored: &Ignored, mut ignore_suite: bool, @@ -203,7 +131,7 @@ pub(super) fn read_suite( } /// Reads information about a given test case. -pub(super) fn read_test(path: &Path) -> Result { +pub fn read_test(path: &Path) -> Result { let name = path .file_stem() .ok_or_else(|| eyre!("path for test `{}` has no file name", path.display()))? diff --git a/tests/tester/src/test252_parser/structs.rs b/tests/tester/src/test252_parser/structs.rs new file mode 100644 index 00000000000..2ffb4a20b59 --- /dev/null +++ b/tests/tester/src/test252_parser/structs.rs @@ -0,0 +1,366 @@ +use bitflags::bitflags; +use color_eyre::eyre::eyre; +use rustc_hash::{FxHashMap, FxHashSet}; +use serde::{ + de::{Unexpected, Visitor}, + Deserialize, Deserializer, +}; +use std::path::Path; + +use super::edition::SpecEdition; + +/// All the harness include files. +#[derive(Debug, Clone)] +pub struct Harness { + pub assert: HarnessFile, + pub sta: HarnessFile, + pub doneprint_handle: HarnessFile, + pub includes: FxHashMap, HarnessFile>, +} + +#[derive(Debug, Clone)] +pub struct HarnessFile { + pub content: Box, + pub path: Box, +} + +/// Represents a test. +#[derive(Debug, Clone)] +#[allow(dead_code)] +pub struct Test { + pub name: Box, + pub path: Box, + pub description: Box, + pub esid: Option>, + pub edition: SpecEdition, + pub flags: TestFlags, + pub information: Box, + pub expected_outcome: Outcome, + pub features: FxHashSet>, + pub includes: FxHashSet>, + pub locale: Locale, + pub ignored: bool, +} + +impl Test { + /// Creates a new test. + pub fn new(name: N, path: C, metadata: MetaData) -> color_eyre::Result + where + N: Into>, + C: Into>, + { + let edition = SpecEdition::from_test_metadata(&metadata) + .map_err(|feats| eyre!("test metadata contained unknown features: {feats:?}"))?; + + Ok(Self { + edition, + name: name.into(), + description: metadata.description, + esid: metadata.esid, + flags: metadata.flags.into(), + information: metadata.info, + features: metadata.features.into_vec().into_iter().collect(), + expected_outcome: Outcome::from(metadata.negative), + includes: metadata.includes.into_vec().into_iter().collect(), + locale: metadata.locale, + path: path.into(), + ignored: false, + }) + } + + /// Sets the test as ignored. + #[inline] + pub fn set_ignored(&mut self) { + self.ignored = true; + } + + /// Checks if this is a module test. + #[inline] + pub const fn is_module(&self) -> bool { + self.flags.contains(TestFlags::MODULE) + } +} + +/// All possible error types +#[derive(Debug, Copy, Clone, Deserialize, PartialEq, Eq)] +#[allow(clippy::enum_variant_names)] // Better than appending `rename` to all variants +pub enum ErrorType { + Test262Error, + SyntaxError, + ReferenceError, + RangeError, + TypeError, +} + +impl ErrorType { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Test262Error => "Test262Error", + Self::SyntaxError => "SyntaxError", + Self::ReferenceError => "ReferenceError", + Self::RangeError => "RangeError", + Self::TypeError => "TypeError", + } + } +} + +/// Phase for an error. +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Phase { + Parse, + Resolution, + Runtime, +} + +/// Negative test information structure. +#[derive(Debug, Clone, Deserialize)] +pub(crate) struct Negative { + pub(super) phase: Phase, + #[serde(rename = "type")] + pub(super) error_type: ErrorType, +} + +/// An outcome for a test. +#[derive(Debug, Clone)] +pub enum Outcome { + Positive, + Negative { phase: Phase, error_type: ErrorType }, +} + +impl Default for Outcome { + fn default() -> Self { + Self::Positive + } +} + +impl From> for Outcome { + fn from(neg: Option) -> Self { + neg.map(|neg| Self::Negative { + phase: neg.phase, + error_type: neg.error_type, + }) + .unwrap_or_default() + } +} + +/// Locale information structure. +#[derive(Debug, Default, Clone, Deserialize)] +#[serde(transparent)] +#[allow(dead_code)] +pub struct Locale { + locale: Box<[Box]>, +} + +/// Represents a test suite. +#[derive(Debug, Clone)] +pub struct TestSuite { + pub name: Box, + pub path: Box, + pub suites: Box<[TestSuite]>, + pub tests: Box<[Test]>, +} + +/// Representation of the YAML metadata in Test262 tests. +#[derive(Debug, Clone, Deserialize)] +pub struct MetaData { + pub description: Box, + pub esid: Option>, + #[allow(dead_code)] + pub es5id: Option>, + pub es6id: Option>, + #[serde(default)] + pub info: Box, + #[serde(default)] + pub features: Box<[Box]>, + #[serde(default)] + pub includes: Box<[Box]>, + #[serde(default)] + pub flags: Box<[TestFlag]>, + #[serde(default)] + pub negative: Option, + #[serde(default)] + pub locale: Locale, +} + +// tests flags + +/// Individual test flag. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum TestFlag { + OnlyStrict, + NoStrict, + Module, + Raw, + Async, + Generated, + #[serde(rename = "CanBlockIsFalse")] + CanBlockIsFalse, + #[serde(rename = "CanBlockIsTrue")] + CanBlockIsTrue, + #[serde(rename = "non-deterministic")] + NonDeterministic, +} + +bitflags! { + #[derive(Debug, Clone, Copy)] + pub struct TestFlags: u16 { + const STRICT = 0b0_0000_0001; + const NO_STRICT = 0b0_0000_0010; + const MODULE = 0b0_0000_0100; + const RAW = 0b0_0000_1000; + const ASYNC = 0b0_0001_0000; + const GENERATED = 0b0_0010_0000; + const CAN_BLOCK_IS_FALSE = 0b0_0100_0000; + const CAN_BLOCK_IS_TRUE = 0b0_1000_0000; + const NON_DETERMINISTIC = 0b1_0000_0000; + } +} + +impl Default for TestFlags { + fn default() -> Self { + Self::STRICT | Self::NO_STRICT + } +} + +impl From for TestFlags { + fn from(flag: TestFlag) -> Self { + match flag { + TestFlag::OnlyStrict => Self::STRICT, + TestFlag::NoStrict => Self::NO_STRICT, + TestFlag::Module => Self::MODULE, + TestFlag::Raw => Self::RAW, + TestFlag::Async => Self::ASYNC, + TestFlag::Generated => Self::GENERATED, + TestFlag::CanBlockIsFalse => Self::CAN_BLOCK_IS_FALSE, + TestFlag::CanBlockIsTrue => Self::CAN_BLOCK_IS_TRUE, + TestFlag::NonDeterministic => Self::NON_DETERMINISTIC, + } + } +} + +impl From for TestFlags +where + T: AsRef<[TestFlag]>, +{ + fn from(flags: T) -> Self { + let flags = flags.as_ref(); + if flags.is_empty() { + Self::default() + } else { + let mut result = Self::empty(); + for flag in flags { + result |= Self::from(*flag); + } + + if !result.intersects(Self::default()) { + result |= Self::default(); + } + + result + } + } +} + +impl<'de> Deserialize<'de> for TestFlags { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct FlagsVisitor; + + impl<'de> Visitor<'de> for FlagsVisitor { + type Value = TestFlags; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "a sequence of flags") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let mut flags = TestFlags::empty(); + while let Some(elem) = seq.next_element::()? { + flags |= elem.into(); + } + Ok(flags) + } + } + + struct RawFlagsVisitor; + + impl Visitor<'_> for RawFlagsVisitor { + type Value = TestFlags; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "a flags number") + } + + fn visit_u16(self, v: u16) -> Result + where + E: serde::de::Error, + { + TestFlags::from_bits(v).ok_or_else(|| { + E::invalid_value(Unexpected::Unsigned(v.into()), &"a valid flag number") + }) + } + } + + if deserializer.is_human_readable() { + deserializer.deserialize_seq(FlagsVisitor) + } else { + deserializer.deserialize_u16(RawFlagsVisitor) + } + } +} + +/// Structure to allow defining ignored tests, features and files that should +/// be ignored even when reading. +#[derive(Debug, Deserialize)] +pub struct Ignored { + #[serde(default)] + tests: FxHashSet>, + #[serde(default)] + features: FxHashSet>, + #[serde(default = "TestFlags::empty")] + flags: TestFlags, +} + +impl Ignored { + /// Checks if the ignore list contains the given test name in the list of + /// tests to ignore. + pub(crate) fn contains_test(&self, test: &str) -> bool { + self.tests.contains(test) + } + + /// Checks if the ignore list contains the given feature name in the list + /// of features to ignore. + pub(crate) fn contains_feature(&self, feature: &str) -> bool { + if self.features.contains(feature) { + return true; + } + // Some features are an accessor instead of a simple feature name e.g. `Intl.DurationFormat`. + // This ensures those are also ignored. + feature + .split('.') + .next() + .map(|feat| self.features.contains(feat)) + .unwrap_or_default() + } + + pub(crate) const fn contains_any_flag(&self, flags: TestFlags) -> bool { + flags.intersects(self.flags) + } +} + +impl Default for Ignored { + fn default() -> Self { + Self { + tests: FxHashSet::default(), + features: FxHashSet::default(), + flags: TestFlags::empty(), + } + } +} From 154f7a1228da886827975a53ccf80792a9782fa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Jastrz=C4=99bski?= Date: Wed, 6 Mar 2024 15:19:06 +0100 Subject: [PATCH 2/9] extracted tc39-test262 crate --- Cargo.lock | 22 +- tests/tester/Cargo.toml | 6 +- tests/tester/src/exec/mod.rs | 167 ++++---- tests/tester/src/main.rs | 46 +-- tests/tester/src/test252_parser/mod.rs | 8 - tests/tester/src/test252_parser/structs.rs | 366 ------------------ tools/tc39-test262/Cargo.toml | 26 ++ .../tc39-test262/src}/edition.rs | 8 +- .../tester => tools/tc39-test262}/src/git.rs | 5 +- tools/tc39-test262/src/lib.rs | 39 ++ .../tc39-test262/src}/read.rs | 8 +- tools/tc39-test262/src/structs.rs | 120 ++++++ tools/tc39-test262/src/test_files.rs | 125 ++++++ tools/tc39-test262/src/test_flags.rs | 138 +++++++ 14 files changed, 584 insertions(+), 500 deletions(-) delete mode 100644 tests/tester/src/test252_parser/mod.rs delete mode 100644 tests/tester/src/test252_parser/structs.rs create mode 100644 tools/tc39-test262/Cargo.toml rename {tests/tester/src/test252_parser => tools/tc39-test262/src}/edition.rs (99%) rename {tests/tester => tools/tc39-test262}/src/git.rs (98%) create mode 100644 tools/tc39-test262/src/lib.rs rename {tests/tester/src/test252_parser => tools/tc39-test262/src}/read.rs (96%) create mode 100644 tools/tc39-test262/src/structs.rs create mode 100644 tools/tc39-test262/src/test_files.rs create mode 100644 tools/tc39-test262/src/test_flags.rs diff --git a/Cargo.lock b/Cargo.lock index fda01722a46..0e7a1d943ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -571,7 +571,6 @@ dependencies = [ name = "boa_tester" version = "0.17.0" dependencies = [ - "bitflags 2.4.2", "boa_engine", "boa_gc", "boa_runtime", @@ -581,14 +580,11 @@ dependencies = [ "colored", "comfy-table", "once_cell", - "phf", "rayon", - "regex", "rustc-hash", "serde", "serde_json", - "serde_repr", - "serde_yaml", + "tc39-test262", "time 0.3.34", "toml 0.8.10", ] @@ -3683,6 +3679,22 @@ version = "0.12.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae" +[[package]] +name = "tc39-test262" +version = "0.17.0" +dependencies = [ + "bitflags 2.4.2", + "clap", + "color-eyre", + "once_cell", + "phf", + "regex", + "rustc-hash", + "serde", + "serde_repr", + "serde_yaml", +] + [[package]] name = "tempfile" version = "3.10.0" diff --git a/tests/tester/Cargo.toml b/tests/tester/Cargo.toml index 8c344c0d7e8..56c94e2b2de 100644 --- a/tests/tester/Cargo.toml +++ b/tests/tester/Cargo.toml @@ -17,19 +17,15 @@ boa_runtime.workspace = true boa_gc.workspace = true clap = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] } -serde_yaml = "0.9.32" +tc39-test262 = { path = "../../tools/tc39-test262"} serde_json.workspace = true -bitflags.workspace = true -regex.workspace = true once_cell.workspace = true colored.workspace = true rustc-hash = { workspace = true, features = ["std"] } rayon = "1.8.1" toml = "0.8.10" color-eyre = "0.6.2" -phf = { workspace = true, features = ["macros"] } comfy-table = "7.1.0" -serde_repr = "0.1.18" bus = "2.4.1" time.workspace = true diff --git a/tests/tester/src/exec/mod.rs b/tests/tester/src/exec/mod.rs index dd54bdd7c22..a8bb535938a 100644 --- a/tests/tester/src/exec/mod.rs +++ b/tests/tester/src/exec/mod.rs @@ -2,9 +2,7 @@ mod js262; -use crate::{ - test252_parser::{ErrorType, Harness, Outcome, Phase, SpecEdition, Test, TestSuite}, Statistics, SuiteResult, TestFlags, TestOutcomeResult, TestResult, VersionedStats -}; +use crate::{Statistics, SuiteResult, TestFlags, TestOutcomeResult, TestResult, VersionedStats}; use boa_engine::{ builtins::promise::PromiseState, js_string, @@ -20,12 +18,41 @@ use colored::Colorize; use rayon::prelude::*; use rustc_hash::FxHashSet; use std::{cell::RefCell, eprintln, rc::Rc}; +use tc39_test262::{ErrorType, Harness, Outcome, Phase, SpecEdition, Test, TestSuite}; use self::js262::WorkerHandles; -impl TestSuite { +pub(crate) trait RunTest { + /// Runs the test. + fn run(&self, harness: &Harness, verbose: u8, optimizer_options: TO, console: bool) -> TR; + + /// Runs the test once, in strict or non-strict mode + fn run_once( + &self, + harness: &Harness, + strict: bool, + verbose: u8, + optimizer_options: OptimizerOptions, + console: bool, + ) -> TestResult; +} + +pub(crate) trait RunTestSuite { /// Runs the test suite. - pub(crate) fn run( + fn run( + &self, + harness: &Harness, + verbose: u8, + parallel: bool, + max_edition: SpecEdition, + optimizer_options: OptimizerOptions, + console: bool, + ) -> SuiteResult; +} + +impl RunTestSuite for TestSuite { + /// Runs the test suite. + fn run( &self, harness: &Harness, verbose: u8, @@ -159,9 +186,9 @@ impl TestSuite { } } -impl Test { +impl RunTest for Test { /// Runs the test. - pub(crate) fn run( + fn run( &self, harness: &Harness, verbose: u8, @@ -262,7 +289,8 @@ impl Test { let mut handles = WorkerHandles::new(); - if let Err(e) = self.set_up_env( + if let Err(e) = set_up_env( + &self, harness, context, async_result.clone(), @@ -452,7 +480,8 @@ impl Test { let mut handles = WorkerHandles::new(); - if let Err(e) = self.set_up_env( + if let Err(e) = set_up_env( + &self, harness, context, AsyncResult::default(), @@ -588,73 +617,73 @@ impl Test { result_text: result_text.into_boxed_str(), } } +} - /// Sets the environment up to run the test. - fn set_up_env( - &self, - harness: &Harness, - context: &mut Context, - async_result: AsyncResult, - handles: WorkerHandles, - console: bool, - ) -> Result<(), String> { - // Register the print() function. - register_print_fn(context, async_result); - - // add the $262 object. - let _js262 = js262::register_js262(handles, context); - - if console { - let console = boa_runtime::Console::init(context); - context - .register_global_property( - js_string!(boa_runtime::Console::NAME), - console, - Attribute::all(), - ) - .expect("the console builtin shouldn't exist"); - } - - if self.flags.contains(TestFlags::RAW) { - return Ok(()); - } +/// Sets the environment up to run the test. +fn set_up_env( + test: &Test, + harness: &Harness, + context: &mut Context, + async_result: AsyncResult, + handles: WorkerHandles, + console: bool, +) -> Result<(), String> { + // Register the print() function. + register_print_fn(context, async_result); + + // add the $262 object. + let _js262 = js262::register_js262(handles, context); + + if console { + let console = boa_runtime::Console::init(context); + context + .register_global_property( + js_string!(boa_runtime::Console::NAME), + console, + Attribute::all(), + ) + .expect("the console builtin shouldn't exist"); + } - let assert = Source::from_reader( - harness.assert.content.as_bytes(), - Some(&harness.assert.path), - ); - let sta = Source::from_reader(harness.sta.content.as_bytes(), Some(&harness.sta.path)); + if test.flags.contains(TestFlags::RAW) { + return Ok(()); + } - context - .eval(assert) - .map_err(|e| format!("could not run assert.js:\n{e}"))?; - context - .eval(sta) - .map_err(|e| format!("could not run sta.js:\n{e}"))?; + let assert = Source::from_reader( + harness.assert.content.as_bytes(), + Some(&harness.assert.path), + ); + let sta = Source::from_reader(harness.sta.content.as_bytes(), Some(&harness.sta.path)); - if self.flags.contains(TestFlags::ASYNC) { - let dph = Source::from_reader( - harness.doneprint_handle.content.as_bytes(), - Some(&harness.doneprint_handle.path), - ); - context - .eval(dph) - .map_err(|e| format!("could not run doneprintHandle.js:\n{e}"))?; - } + context + .eval(assert) + .map_err(|e| format!("could not run assert.js:\n{e}"))?; + context + .eval(sta) + .map_err(|e| format!("could not run sta.js:\n{e}"))?; - for include_name in &self.includes { - let include = harness - .includes - .get(include_name) - .ok_or_else(|| format!("could not find the {include_name} include file."))?; - let source = Source::from_reader(include.content.as_bytes(), Some(&include.path)); - context.eval(source).map_err(|e| { - format!("could not run the harness `{include_name}`:\nUncaught {e}",) - })?; - } + if test.flags.contains(TestFlags::ASYNC) { + let dph = Source::from_reader( + harness.doneprint_handle.content.as_bytes(), + Some(&harness.doneprint_handle.path), + ); + context + .eval(dph) + .map_err(|e| format!("could not run doneprintHandle.js:\n{e}"))?; + } - Ok(()) + for include_name in &test.includes { + let include = harness + .includes + .get(include_name) + .ok_or_else(|| format!("could not find the {include_name} include file."))?; + let source = Source::from_reader(include.content.as_bytes(), Some(&include.path)); + context + .eval(source) + .map_err(|e| format!("could not run the harness `{include_name}`:\nUncaught {e}",))?; } + + Ok(()) } /// Returns `true` if `error` is a `target_type` error. diff --git a/tests/tester/src/main.rs b/tests/tester/src/main.rs index 7536f5c13ed..2c1324612ea 100644 --- a/tests/tester/src/main.rs +++ b/tests/tester/src/main.rs @@ -10,11 +10,11 @@ )] mod exec; -mod git; -mod test252_parser; mod results; -use test252_parser::{TestFlags, SpecEdition, Ignored, read_harness, read_suite, read_test}; +use exec::{RunTestSuite, RunTest}; +use tc39_test262::{read, Ignored, SpecEdition, TestFlags}; + use self::results::{compare_results, write_json}; use boa_engine::optimizer::OptimizerOptions; @@ -26,9 +26,7 @@ use color_eyre::{ use colored::Colorize; use once_cell::sync::Lazy; use rustc_hash::FxHashSet; -use serde::{ - Deserialize, Deserializer, Serialize, -}; +use serde::{Deserialize, Deserializer, Serialize}; use std::{ ops::{Add, AddAssign}, path::{Path, PathBuf}, @@ -128,8 +126,6 @@ enum Cli { }, } -const DEFAULT_TEST262_DIRECTORY: &str = "test262"; - /// Program entry point. fn main() -> Result<()> { // Safety: This is needed because we run tests in multiple threads. @@ -168,9 +164,8 @@ fn main() -> Result<()> { let test262_path = if let Some(path) = test262_path.as_deref() { path } else { - clone_test262(test262_commit, verbose)?; - - Path::new(DEFAULT_TEST262_DIRECTORY) + tc39_test262::clone_test262(test262_commit, verbose)?; + Path::new(tc39_test262::TEST262_DIRECTORY) }; run_test_suite( @@ -198,17 +193,6 @@ fn main() -> Result<()> { } } -fn clone_test262(commit: Option<&str>, verbose: u8) -> Result<()> { - const TEST262_REPOSITORY: &str = "https://github.com/tc39/test262"; - git::clone( - DEFAULT_TEST262_DIRECTORY, - TEST262_REPOSITORY, - &"origin/main", - commit, - verbose, - ) -} - /// Runs the full test suite. #[allow(clippy::too_many_arguments)] fn run_test_suite( @@ -236,10 +220,10 @@ fn run_test_suite( if verbose != 0 { println!("Loading the test suite..."); } - let harness = read_harness(test262_path).wrap_err("could not read harness")?; + let harness = read::read_harness(test262_path).wrap_err("could not read harness")?; if suite.to_string_lossy().ends_with(".js") { - let test = read_test(&test262_path.join(suite)).wrap_err_with(|| { + let test = read::read_test(&test262_path.join(suite)).wrap_err_with(|| { let suite = suite.display(); format!("could not read the test {suite}") })?; @@ -257,8 +241,8 @@ fn run_test_suite( println!(); } else { - let suite = - read_suite(&test262_path.join(suite), config.ignored(), false).wrap_err_with(|| { + let suite = read::read_suite(&test262_path.join(suite), config.ignored(), false) + .wrap_err_with(|| { let suite = suite.display(); format!("could not read the suite {suite}") })?; @@ -573,13 +557,3 @@ enum TestOutcomeResult { #[serde(rename = "P")] Panic, } - -#[cfg(test)] -mod tests { - - #[test] - #[ignore = "manual testing"] - fn should_clone_test262() { - super::clone_test262(None, 0).unwrap(); - } -} diff --git a/tests/tester/src/test252_parser/mod.rs b/tests/tester/src/test252_parser/mod.rs deleted file mode 100644 index d62e9c67219..00000000000 --- a/tests/tester/src/test252_parser/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -mod structs; -mod edition; -mod read; - -pub use structs::{TestFlags, Ignored, ErrorType, Harness, Outcome, Phase, Test, TestSuite}; -pub use read::{read_suite, read_harness, read_test}; - -pub use edition::SpecEdition; diff --git a/tests/tester/src/test252_parser/structs.rs b/tests/tester/src/test252_parser/structs.rs deleted file mode 100644 index 2ffb4a20b59..00000000000 --- a/tests/tester/src/test252_parser/structs.rs +++ /dev/null @@ -1,366 +0,0 @@ -use bitflags::bitflags; -use color_eyre::eyre::eyre; -use rustc_hash::{FxHashMap, FxHashSet}; -use serde::{ - de::{Unexpected, Visitor}, - Deserialize, Deserializer, -}; -use std::path::Path; - -use super::edition::SpecEdition; - -/// All the harness include files. -#[derive(Debug, Clone)] -pub struct Harness { - pub assert: HarnessFile, - pub sta: HarnessFile, - pub doneprint_handle: HarnessFile, - pub includes: FxHashMap, HarnessFile>, -} - -#[derive(Debug, Clone)] -pub struct HarnessFile { - pub content: Box, - pub path: Box, -} - -/// Represents a test. -#[derive(Debug, Clone)] -#[allow(dead_code)] -pub struct Test { - pub name: Box, - pub path: Box, - pub description: Box, - pub esid: Option>, - pub edition: SpecEdition, - pub flags: TestFlags, - pub information: Box, - pub expected_outcome: Outcome, - pub features: FxHashSet>, - pub includes: FxHashSet>, - pub locale: Locale, - pub ignored: bool, -} - -impl Test { - /// Creates a new test. - pub fn new(name: N, path: C, metadata: MetaData) -> color_eyre::Result - where - N: Into>, - C: Into>, - { - let edition = SpecEdition::from_test_metadata(&metadata) - .map_err(|feats| eyre!("test metadata contained unknown features: {feats:?}"))?; - - Ok(Self { - edition, - name: name.into(), - description: metadata.description, - esid: metadata.esid, - flags: metadata.flags.into(), - information: metadata.info, - features: metadata.features.into_vec().into_iter().collect(), - expected_outcome: Outcome::from(metadata.negative), - includes: metadata.includes.into_vec().into_iter().collect(), - locale: metadata.locale, - path: path.into(), - ignored: false, - }) - } - - /// Sets the test as ignored. - #[inline] - pub fn set_ignored(&mut self) { - self.ignored = true; - } - - /// Checks if this is a module test. - #[inline] - pub const fn is_module(&self) -> bool { - self.flags.contains(TestFlags::MODULE) - } -} - -/// All possible error types -#[derive(Debug, Copy, Clone, Deserialize, PartialEq, Eq)] -#[allow(clippy::enum_variant_names)] // Better than appending `rename` to all variants -pub enum ErrorType { - Test262Error, - SyntaxError, - ReferenceError, - RangeError, - TypeError, -} - -impl ErrorType { - pub(crate) const fn as_str(self) -> &'static str { - match self { - Self::Test262Error => "Test262Error", - Self::SyntaxError => "SyntaxError", - Self::ReferenceError => "ReferenceError", - Self::RangeError => "RangeError", - Self::TypeError => "TypeError", - } - } -} - -/// Phase for an error. -#[derive(Debug, Clone, Copy, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum Phase { - Parse, - Resolution, - Runtime, -} - -/// Negative test information structure. -#[derive(Debug, Clone, Deserialize)] -pub(crate) struct Negative { - pub(super) phase: Phase, - #[serde(rename = "type")] - pub(super) error_type: ErrorType, -} - -/// An outcome for a test. -#[derive(Debug, Clone)] -pub enum Outcome { - Positive, - Negative { phase: Phase, error_type: ErrorType }, -} - -impl Default for Outcome { - fn default() -> Self { - Self::Positive - } -} - -impl From> for Outcome { - fn from(neg: Option) -> Self { - neg.map(|neg| Self::Negative { - phase: neg.phase, - error_type: neg.error_type, - }) - .unwrap_or_default() - } -} - -/// Locale information structure. -#[derive(Debug, Default, Clone, Deserialize)] -#[serde(transparent)] -#[allow(dead_code)] -pub struct Locale { - locale: Box<[Box]>, -} - -/// Represents a test suite. -#[derive(Debug, Clone)] -pub struct TestSuite { - pub name: Box, - pub path: Box, - pub suites: Box<[TestSuite]>, - pub tests: Box<[Test]>, -} - -/// Representation of the YAML metadata in Test262 tests. -#[derive(Debug, Clone, Deserialize)] -pub struct MetaData { - pub description: Box, - pub esid: Option>, - #[allow(dead_code)] - pub es5id: Option>, - pub es6id: Option>, - #[serde(default)] - pub info: Box, - #[serde(default)] - pub features: Box<[Box]>, - #[serde(default)] - pub includes: Box<[Box]>, - #[serde(default)] - pub flags: Box<[TestFlag]>, - #[serde(default)] - pub negative: Option, - #[serde(default)] - pub locale: Locale, -} - -// tests flags - -/// Individual test flag. -#[derive(Debug, Clone, Copy, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum TestFlag { - OnlyStrict, - NoStrict, - Module, - Raw, - Async, - Generated, - #[serde(rename = "CanBlockIsFalse")] - CanBlockIsFalse, - #[serde(rename = "CanBlockIsTrue")] - CanBlockIsTrue, - #[serde(rename = "non-deterministic")] - NonDeterministic, -} - -bitflags! { - #[derive(Debug, Clone, Copy)] - pub struct TestFlags: u16 { - const STRICT = 0b0_0000_0001; - const NO_STRICT = 0b0_0000_0010; - const MODULE = 0b0_0000_0100; - const RAW = 0b0_0000_1000; - const ASYNC = 0b0_0001_0000; - const GENERATED = 0b0_0010_0000; - const CAN_BLOCK_IS_FALSE = 0b0_0100_0000; - const CAN_BLOCK_IS_TRUE = 0b0_1000_0000; - const NON_DETERMINISTIC = 0b1_0000_0000; - } -} - -impl Default for TestFlags { - fn default() -> Self { - Self::STRICT | Self::NO_STRICT - } -} - -impl From for TestFlags { - fn from(flag: TestFlag) -> Self { - match flag { - TestFlag::OnlyStrict => Self::STRICT, - TestFlag::NoStrict => Self::NO_STRICT, - TestFlag::Module => Self::MODULE, - TestFlag::Raw => Self::RAW, - TestFlag::Async => Self::ASYNC, - TestFlag::Generated => Self::GENERATED, - TestFlag::CanBlockIsFalse => Self::CAN_BLOCK_IS_FALSE, - TestFlag::CanBlockIsTrue => Self::CAN_BLOCK_IS_TRUE, - TestFlag::NonDeterministic => Self::NON_DETERMINISTIC, - } - } -} - -impl From for TestFlags -where - T: AsRef<[TestFlag]>, -{ - fn from(flags: T) -> Self { - let flags = flags.as_ref(); - if flags.is_empty() { - Self::default() - } else { - let mut result = Self::empty(); - for flag in flags { - result |= Self::from(*flag); - } - - if !result.intersects(Self::default()) { - result |= Self::default(); - } - - result - } - } -} - -impl<'de> Deserialize<'de> for TestFlags { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - struct FlagsVisitor; - - impl<'de> Visitor<'de> for FlagsVisitor { - type Value = TestFlags; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(formatter, "a sequence of flags") - } - - fn visit_seq(self, mut seq: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - let mut flags = TestFlags::empty(); - while let Some(elem) = seq.next_element::()? { - flags |= elem.into(); - } - Ok(flags) - } - } - - struct RawFlagsVisitor; - - impl Visitor<'_> for RawFlagsVisitor { - type Value = TestFlags; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(formatter, "a flags number") - } - - fn visit_u16(self, v: u16) -> Result - where - E: serde::de::Error, - { - TestFlags::from_bits(v).ok_or_else(|| { - E::invalid_value(Unexpected::Unsigned(v.into()), &"a valid flag number") - }) - } - } - - if deserializer.is_human_readable() { - deserializer.deserialize_seq(FlagsVisitor) - } else { - deserializer.deserialize_u16(RawFlagsVisitor) - } - } -} - -/// Structure to allow defining ignored tests, features and files that should -/// be ignored even when reading. -#[derive(Debug, Deserialize)] -pub struct Ignored { - #[serde(default)] - tests: FxHashSet>, - #[serde(default)] - features: FxHashSet>, - #[serde(default = "TestFlags::empty")] - flags: TestFlags, -} - -impl Ignored { - /// Checks if the ignore list contains the given test name in the list of - /// tests to ignore. - pub(crate) fn contains_test(&self, test: &str) -> bool { - self.tests.contains(test) - } - - /// Checks if the ignore list contains the given feature name in the list - /// of features to ignore. - pub(crate) fn contains_feature(&self, feature: &str) -> bool { - if self.features.contains(feature) { - return true; - } - // Some features are an accessor instead of a simple feature name e.g. `Intl.DurationFormat`. - // This ensures those are also ignored. - feature - .split('.') - .next() - .map(|feat| self.features.contains(feat)) - .unwrap_or_default() - } - - pub(crate) const fn contains_any_flag(&self, flags: TestFlags) -> bool { - flags.intersects(self.flags) - } -} - -impl Default for Ignored { - fn default() -> Self { - Self { - tests: FxHashSet::default(), - features: FxHashSet::default(), - flags: TestFlags::empty(), - } - } -} diff --git a/tools/tc39-test262/Cargo.toml b/tools/tc39-test262/Cargo.toml new file mode 100644 index 00000000000..b8431958143 --- /dev/null +++ b/tools/tc39-test262/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "tc39-test262" +edition.workspace = true +version.workspace = true +rust-version.workspace = true +authors.workspace = true +repository.workspace = true +license.workspace = true +description.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde = { workspace = true, features = ["derive"] } +phf = { workspace = true, features = ["macros"] } +clap = { workspace = true, features = ["derive"] } +bitflags.workspace = true +once_cell.workspace = true +regex.workspace = true +serde_repr = "^0.1.18" +color-eyre = "0.6.2" +rustc-hash = "1.1.0" +serde_yaml = "0.9.32" + +[lints] +workspace = true diff --git a/tests/tester/src/test252_parser/edition.rs b/tools/tc39-test262/src/edition.rs similarity index 99% rename from tests/tester/src/test252_parser/edition.rs rename to tools/tc39-test262/src/edition.rs index 4f8c1dedacb..2bde3fcb590 100644 --- a/tests/tester/src/test252_parser/edition.rs +++ b/tools/tc39-test262/src/edition.rs @@ -3,11 +3,9 @@ //! This module contains the [`SpecEdition`] struct, which is used in the tester to //! classify all tests per minimum required ECMAScript edition. -use std::fmt::Display; - +use crate::{test_flags::TestFlag, MetaData}; use serde_repr::{Deserialize_repr, Serialize_repr}; - -use super::structs::{MetaData, TestFlag}; +use std::fmt::Display; /// Minimum edition required by a specific feature in the `test262` repository. static FEATURE_EDITION: phf::Map<&'static str, SpecEdition> = phf::phf_map! { @@ -351,7 +349,7 @@ impl SpecEdition { } /// Gets an iterator of all currently available editions. - pub(crate) fn all_editions() -> impl Iterator { + pub fn all_editions() -> impl Iterator { [ Self::ES5, Self::ES6, diff --git a/tests/tester/src/git.rs b/tools/tc39-test262/src/git.rs similarity index 98% rename from tests/tester/src/git.rs rename to tools/tc39-test262/src/git.rs index 9288838d9fc..b674e6d94c9 100644 --- a/tests/tester/src/git.rs +++ b/tools/tc39-test262/src/git.rs @@ -1,3 +1,5 @@ +//! Git common + use color_eyre::{eyre::bail, Result}; use std::{path::Path, process::Command}; @@ -52,7 +54,8 @@ fn reset_commit(directory: &str, commit: &str, verbose: u8) -> Result<()> { Ok(()) } -pub(super) fn clone( +/// Clone repository +pub fn clone( directory: &str, repor_url: &str, // "https://github.com/tc39/test262" baranch: &str, // "origin/main" diff --git a/tools/tc39-test262/src/lib.rs b/tools/tc39-test262/src/lib.rs new file mode 100644 index 00000000000..afbbdb769c5 --- /dev/null +++ b/tools/tc39-test262/src/lib.rs @@ -0,0 +1,39 @@ +//! TC39 test262 +mod edition; +mod structs; +mod test_files; +mod test_flags; + +pub mod git; +pub mod read; +pub use structs::{ErrorType, Outcome, Phase, Ignored}; +pub use test_files::{Harness, HarnessFile, MetaData, Test, TestSuite}; +pub use test_flags::TestFlags; +pub use edition::SpecEdition; + +/// Repository Url +pub const TEST262_REPOSITORY: &str = "https://github.com/tc39/test262"; +/// Git clone directory +pub const TEST262_DIRECTORY: &str = "test262"; + +/// Clone TC39 test262 repostiory +pub fn clone_test262(commit: Option<&str>, verbose: u8) -> color_eyre::Result<()> { + const TEST262_REPOSITORY: &str = "https://github.com/tc39/test262"; + git::clone( + TEST262_DIRECTORY, + TEST262_REPOSITORY, + &"origin/main", + commit, + verbose, + ) +} + +#[cfg(test)] +mod tests { + + #[test] + #[ignore = "manual testing"] + fn should_clone_test262() { + super::clone_test262(None, 0).unwrap(); + } +} diff --git a/tests/tester/src/test252_parser/read.rs b/tools/tc39-test262/src/read.rs similarity index 96% rename from tests/tester/src/test252_parser/read.rs rename to tools/tc39-test262/src/read.rs index 5c8eecb8ae3..d013a28f92f 100644 --- a/tests/tester/src/test252_parser/read.rs +++ b/tools/tc39-test262/src/read.rs @@ -1,6 +1,5 @@ //! Module to read the list of test suites from disk. - -use super::structs::{Harness, HarnessFile, MetaData, Ignored, Test, TestSuite}; +use super::test_files::{Harness, HarnessFile, MetaData, Test, TestSuite}; use color_eyre::{ eyre::{eyre, WrapErr}, @@ -13,7 +12,6 @@ use std::{ path::{Path, PathBuf}, }; - /// Reads the Test262 defined bindings. pub fn read_harness(test262_path: &Path) -> Result { fn read_harness_file(path: PathBuf) -> Result { @@ -58,7 +56,7 @@ pub fn read_harness(test262_path: &Path) -> Result { /// Reads a test suite in the given path. pub fn read_suite( path: &Path, - ignored: &Ignored, + ignored: &crate::structs::Ignored, mut ignore_suite: bool, ) -> Result { let name = path @@ -149,7 +147,7 @@ pub fn read_test(path: &Path) -> Result { } /// Reads the metadata from the input test code. -fn read_metadata(test: &Path) -> io::Result { +pub fn read_metadata(test: &Path) -> io::Result { use once_cell::sync::Lazy; use regex::bytes::Regex; diff --git a/tools/tc39-test262/src/structs.rs b/tools/tc39-test262/src/structs.rs new file mode 100644 index 00000000000..3c6e57e9ca3 --- /dev/null +++ b/tools/tc39-test262/src/structs.rs @@ -0,0 +1,120 @@ +#![allow(missing_docs)] +use rustc_hash::FxHashSet; +use serde::Deserialize; +use crate::test_flags::TestFlags; + +/// All possible error types +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone, Deserialize, PartialEq, Eq)] +#[allow(clippy::enum_variant_names)] // Better than appending `rename` to all variants +pub enum ErrorType { + Test262Error, + SyntaxError, + ReferenceError, + RangeError, + TypeError, +} + +impl ErrorType { + /// str representation + pub const fn as_str(self) -> &'static str { + match self { + Self::Test262Error => "Test262Error", + Self::SyntaxError => "SyntaxError", + Self::ReferenceError => "ReferenceError", + Self::RangeError => "RangeError", + Self::TypeError => "TypeError", + } + } +} + +/// Phase for an error. +#[allow(missing_docs)] +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Phase { + Parse, + Resolution, + Runtime, +} + +/// Negative test information structure. +#[derive(Debug, Clone, Copy, Deserialize)] +pub struct Negative { + pub(super) phase: Phase, + #[serde(rename = "type")] + pub(super) error_type: ErrorType, +} + +/// An outcome for a test. +#[allow(missing_docs)] +#[derive(Debug, Clone, Copy)] +pub enum Outcome { + Positive, + Negative { phase: Phase, error_type: ErrorType }, +} + +impl Default for Outcome { + fn default() -> Self { + Self::Positive + } +} + +impl From> for Outcome { + fn from(neg: Option) -> Self { + neg.map(|neg| Self::Negative { + phase: neg.phase, + error_type: neg.error_type, + }) + .unwrap_or_default() + } +} + +/// Structure to allow defining ignored tests, features and files that should +/// be ignored even when reading. +#[derive(Debug, Deserialize)] +pub struct Ignored { + #[serde(default)] + tests: FxHashSet>, + #[serde(default)] + features: FxHashSet>, + #[serde(default = "TestFlags::empty")] + flags: TestFlags, +} + +impl Ignored { + /// Checks if the ignore list contains the given test name in the list of + /// tests to ignore. + pub(crate) fn contains_test(&self, test: &str) -> bool { + self.tests.contains(test) + } + + /// Checks if the ignore list contains the given feature name in the list + /// of features to ignore. + pub(crate) fn contains_feature(&self, feature: &str) -> bool { + if self.features.contains(feature) { + return true; + } + // Some features are an accessor instead of a simple feature name e.g. `Intl.DurationFormat`. + // This ensures those are also ignored. + feature + .split('.') + .next() + .map(|feat| self.features.contains(feat)) + .unwrap_or_default() + } + + pub(crate) const fn contains_any_flag(&self, flags: TestFlags) -> bool { + flags.intersects(self.flags) + } +} + +impl Default for Ignored { + fn default() -> Self { + Self { + tests: FxHashSet::default(), + features: FxHashSet::default(), + flags: TestFlags::empty(), + } + } +} diff --git a/tools/tc39-test262/src/test_files.rs b/tools/tc39-test262/src/test_files.rs new file mode 100644 index 00000000000..53e94112b97 --- /dev/null +++ b/tools/tc39-test262/src/test_files.rs @@ -0,0 +1,125 @@ +use std::path::Path; +use rustc_hash::{FxHashMap, FxHashSet}; +use serde::Deserialize; + +use crate::{TestFlags, SpecEdition}; +use super::structs::*; + +/// All the harness include files. +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub struct Harness { + pub assert: HarnessFile, + pub sta: HarnessFile, + pub doneprint_handle: HarnessFile, + pub includes: FxHashMap, HarnessFile>, +} + +/// Represents harness file. +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub struct HarnessFile { + pub content: Box, + pub path: Box, +} + +/// Represents a test. +#[allow(dead_code)] +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub struct Test { + pub name: Box, + pub path: Box, + pub description: Box, + pub esid: Option>, + pub edition: SpecEdition, + pub flags: TestFlags, + pub information: Box, + pub expected_outcome: Outcome, + pub features: FxHashSet>, + pub includes: FxHashSet>, + pub locale: Locale, + pub ignored: bool, +} + +impl Test { + /// Creates a new test. + pub fn new(name: N, path: C, metadata: MetaData) -> color_eyre::Result + where + N: Into>, + C: Into>, + { + let edition = SpecEdition::from_test_metadata(&metadata) + .map_err(|feats| color_eyre::eyre::eyre!("test metadata contained unknown features: {feats:?}"))?; + + Ok(Self { + edition, + name: name.into(), + description: metadata.description, + esid: metadata.esid, + flags: metadata.flags.into(), + information: metadata.info, + features: metadata.features.into_vec().into_iter().collect(), + expected_outcome: Outcome::from(metadata.negative), + includes: metadata.includes.into_vec().into_iter().collect(), + locale: metadata.locale, + path: path.into(), + ignored: false, + }) + } + + /// Sets the test as ignored. + #[inline] + pub fn set_ignored(&mut self) { + self.ignored = true; + } + + /// Checks if this is a module test. + #[inline] + pub const fn is_module(&self) -> bool { + self.flags.contains(TestFlags::MODULE) + } +} + + +/// Represents a test suite. +#[allow(missing_docs)] +#[derive(Debug, Clone)] +pub struct TestSuite { + pub name: Box, + pub path: Box, + pub suites: Box<[TestSuite]>, + pub tests: Box<[Test]>, +} + +/// Representation of the YAML metadata in Test262 tests. +#[allow(missing_docs)] +#[derive(Debug, Clone, Deserialize)] +pub struct MetaData { + pub description: Box, + pub esid: Option>, + #[allow(dead_code)] + pub es5id: Option>, + pub es6id: Option>, + #[serde(default)] + pub info: Box, + #[serde(default)] + pub features: Box<[Box]>, + #[serde(default)] + pub includes: Box<[Box]>, + #[serde(default)] + pub flags: Box<[crate::test_flags::TestFlag]>, + #[serde(default)] + pub negative: Option, + #[serde(default)] + pub locale: Locale, +} + +/// Locale information structure. +#[derive(Debug, Default, Clone, Deserialize)] +#[serde(transparent)] +#[allow(dead_code)] +pub struct Locale { + locale: Box<[Box]>, +} + diff --git a/tools/tc39-test262/src/test_flags.rs b/tools/tc39-test262/src/test_flags.rs new file mode 100644 index 00000000000..53f38af4651 --- /dev/null +++ b/tools/tc39-test262/src/test_flags.rs @@ -0,0 +1,138 @@ +#![allow(missing_docs)] +use bitflags::bitflags; +use serde::{ + de::{Unexpected, Visitor}, + Deserialize, Deserializer, +}; + + +/// Individual test flag. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum TestFlag { + OnlyStrict, + NoStrict, + Module, + Raw, + Async, + Generated, + #[serde(rename = "CanBlockIsFalse")] + CanBlockIsFalse, + #[serde(rename = "CanBlockIsTrue")] + CanBlockIsTrue, + #[serde(rename = "non-deterministic")] + NonDeterministic, +} + +bitflags! { + #[derive(Debug, Clone, Copy)] + pub struct TestFlags: u16 { + const STRICT = 0b0_0000_0001; + const NO_STRICT = 0b0_0000_0010; + const MODULE = 0b0_0000_0100; + const RAW = 0b0_0000_1000; + const ASYNC = 0b0_0001_0000; + const GENERATED = 0b0_0010_0000; + const CAN_BLOCK_IS_FALSE = 0b0_0100_0000; + const CAN_BLOCK_IS_TRUE = 0b0_1000_0000; + const NON_DETERMINISTIC = 0b1_0000_0000; + } +} + +impl Default for TestFlags { + fn default() -> Self { + Self::STRICT | Self::NO_STRICT + } +} + +impl From for TestFlags { + fn from(flag: TestFlag) -> Self { + match flag { + TestFlag::OnlyStrict => Self::STRICT, + TestFlag::NoStrict => Self::NO_STRICT, + TestFlag::Module => Self::MODULE, + TestFlag::Raw => Self::RAW, + TestFlag::Async => Self::ASYNC, + TestFlag::Generated => Self::GENERATED, + TestFlag::CanBlockIsFalse => Self::CAN_BLOCK_IS_FALSE, + TestFlag::CanBlockIsTrue => Self::CAN_BLOCK_IS_TRUE, + TestFlag::NonDeterministic => Self::NON_DETERMINISTIC, + } + } +} + +impl From for TestFlags +where + T: AsRef<[TestFlag]>, +{ + fn from(flags: T) -> Self { + let flags = flags.as_ref(); + if flags.is_empty() { + Self::default() + } else { + let mut result = Self::empty(); + for flag in flags { + result |= Self::from(*flag); + } + + if !result.intersects(Self::default()) { + result |= Self::default(); + } + + result + } + } +} + +impl<'de> Deserialize<'de> for TestFlags { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct FlagsVisitor; + + impl<'de> Visitor<'de> for FlagsVisitor { + type Value = TestFlags; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "a sequence of flags") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let mut flags = TestFlags::empty(); + while let Some(elem) = seq.next_element::()? { + flags |= elem.into(); + } + Ok(flags) + } + } + + struct RawFlagsVisitor; + + impl Visitor<'_> for RawFlagsVisitor { + type Value = TestFlags; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "a flags number") + } + + fn visit_u16(self, v: u16) -> Result + where + E: serde::de::Error, + { + TestFlags::from_bits(v).ok_or_else(|| { + E::invalid_value(Unexpected::Unsigned(v.into()), &"a valid flag number") + }) + } + } + + if deserializer.is_human_readable() { + deserializer.deserialize_seq(FlagsVisitor) + } else { + deserializer.deserialize_u16(RawFlagsVisitor) + } + } +} From 80a129b10c1c001695d9b8aff57d6b334bf6945e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Jastrz=C4=99bski?= Date: Sat, 16 Mar 2024 11:00:15 +0100 Subject: [PATCH 3/9] manual tests --- Cargo.lock | 81 +++++++++++++++++++++++++++++++ tests/tester/Cargo.toml | 3 ++ tests/tester/src/main.rs | 30 +++++++++++- tools/tc39-test262/src/edition.rs | 4 ++ tools/tc39-test262/src/lib.rs | 21 +++++++- 5 files changed, 137 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0e7a1d943ce..17eac578bbd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,6 +124,21 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +[[package]] +name = "assert_cmd" +version = "2.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed72493ac66d5804837f480ab3766c72bdfab91a65e565fc54fa9e42db0073a8" +dependencies = [ + "anstyle", + "bstr", + "doc-comment", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + [[package]] name = "async-channel" version = "1.9.0" @@ -571,6 +586,7 @@ dependencies = [ name = "boa_tester" version = "0.17.0" dependencies = [ + "assert_cmd", "boa_engine", "boa_gc", "boa_runtime", @@ -599,6 +615,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "bstr" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" +dependencies = [ + "memchr", + "regex-automata 0.4.5", + "serde", +] + [[package]] name = "bumpalo" version = "3.15.0" @@ -1182,6 +1209,12 @@ dependencies = [ "thousands", ] +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + [[package]] name = "discard" version = "1.0.4" @@ -1199,6 +1232,12 @@ dependencies = [ "syn 2.0.51", ] +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + [[package]] name = "dynasm" version = "1.2.3" @@ -2853,6 +2892,33 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "predicates" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b87bfd4605926cdfefc1c3b5f8fe560e3feca9d5552cf68c466d3d8236c7e8" +dependencies = [ + "anstyle", + "difflib", + "predicates-core", +] + +[[package]] +name = "predicates-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" + +[[package]] +name = "predicates-tree" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "proc-macro-crate" version = "3.1.0" @@ -3730,6 +3796,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "termtree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" + [[package]] name = "textwrap" version = "0.16.1" @@ -4156,6 +4228,15 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "waker-fn" version = "1.1.1" diff --git a/tests/tester/Cargo.toml b/tests/tester/Cargo.toml index 56c94e2b2de..b4e8d25add4 100644 --- a/tests/tester/Cargo.toml +++ b/tests/tester/Cargo.toml @@ -29,6 +29,9 @@ comfy-table = "7.1.0" bus = "2.4.1" time.workspace = true +[dev-dependencies] +assert_cmd = "2.0.14" + [features] default = ["boa_engine/intl_bundled", "boa_engine/experimental", "boa_engine/annex-b"] diff --git a/tests/tester/src/main.rs b/tests/tester/src/main.rs index 2c1324612ea..937bde6e35f 100644 --- a/tests/tester/src/main.rs +++ b/tests/tester/src/main.rs @@ -12,7 +12,7 @@ mod exec; mod results; -use exec::{RunTestSuite, RunTest}; +use exec::{RunTest, RunTestSuite}; use tc39_test262::{read, Ignored, SpecEdition, TestFlags}; use self::results::{compare_results, write_json}; @@ -557,3 +557,31 @@ enum TestOutcomeResult { #[serde(rename = "P")] Panic, } + +#[cfg(test)] +mod tests { + use assert_cmd::Command; + + fn cmd() -> Command { + let mut cmd: Command = assert_cmd::Command::cargo_bin("boa_tester").unwrap(); + cmd.current_dir("../../"); + cmd + } + + #[test] + #[ignore = "manual"] + fn test_help() { + let output = cmd().output().unwrap(); + let err = String::from_utf8(output.stderr).unwrap(); + assert!(err.contains("boa_tester ")); + + cmd().arg("run").arg("--help").assert().success(); + cmd().arg("compare").arg("--help").assert().success(); + } + + #[test] + #[ignore = "manual"] + fn test() { + cmd().arg("run").arg("--edition es5").arg("-O").assert().success(); + } +} diff --git a/tools/tc39-test262/src/edition.rs b/tools/tc39-test262/src/edition.rs index 2bde3fcb590..0dd5eac8011 100644 --- a/tools/tc39-test262/src/edition.rs +++ b/tools/tc39-test262/src/edition.rs @@ -79,6 +79,10 @@ static FEATURE_EDITION: phf::Map<&'static str, SpecEdition> = phf::phf_map! { // https://github.com/tc39/proposal-set-methods "set-methods" => SpecEdition::ESNext, + // Regular Expression Pattern Modifiers + // https://github.com/tc39/proposal-regexp-modifiers + "regexp-modifiers" => SpecEdition::ESNext, + // Part of the next ES15 edition "Atomics.waitAsync" => SpecEdition::ESNext, "regexp-v-flag" => SpecEdition::ESNext, diff --git a/tools/tc39-test262/src/lib.rs b/tools/tc39-test262/src/lib.rs index afbbdb769c5..69ec7c4854a 100644 --- a/tools/tc39-test262/src/lib.rs +++ b/tools/tc39-test262/src/lib.rs @@ -30,10 +30,29 @@ pub fn clone_test262(commit: Option<&str>, verbose: u8) -> color_eyre::Result<() #[cfg(test)] mod tests { + use std::path::Path; + use crate::{Ignored, TEST262_DIRECTORY}; #[test] - #[ignore = "manual testing"] + #[ignore = "manual"] fn should_clone_test262() { super::clone_test262(None, 0).unwrap(); } + + #[test] + #[ignore = "manual"] + fn should_read_harness() { + let harness = super::read::read_harness(Path::new(TEST262_DIRECTORY)).unwrap(); + assert!(harness.assert.path.is_file()); + assert!(harness.sta.path.is_file()); + assert!(harness.doneprint_handle.path.is_file()); + } + + #[test] + #[ignore = "manual"] + fn should_read_test_suite() { + let path = Path::new(TEST262_DIRECTORY).join("test").join("language"); + let test_suite = super::read::read_suite(&path, &Ignored::default(), false).unwrap(); + assert!(!test_suite.name.is_empty()); + } } From 391ac73dc72cc18da0c24cd9452611377ff9ce07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Jastrzebski?= Date: Thu, 22 Aug 2024 04:53:20 +0200 Subject: [PATCH 4/9] SpecEdition - Temporally fallback to ESNext for unknown features --- Cargo.lock | 2 +- tools/tc39-test262/src/edition.rs | 11 +++---- tools/tc39-test262/src/lib.rs | 49 +++++++++++++++++++++++++++++-- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be08af9cae9..97909ef544a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3282,7 +3282,7 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tc39-test262" -version = "0.18.0" +version = "0.19.0" dependencies = [ "bitflags 2.6.0", "clap", diff --git a/tools/tc39-test262/src/edition.rs b/tools/tc39-test262/src/edition.rs index 140c089fe26..41d87b30b16 100644 --- a/tools/tc39-test262/src/edition.rs +++ b/tools/tc39-test262/src/edition.rs @@ -39,12 +39,6 @@ static FEATURE_EDITION: phf::Map<&'static str, SpecEdition> = phf::phf_map! { // https://github.com/tc39/proposal-json-modules "json-modules" => SpecEdition::ESNext, - // https://github.com/tc39/proposal-source-phase-imports - "source-phase-imports" => SpecEdition::ESNext, - - // https://github.com/tc39/proposal-source-phase-imports - "source-phase-imports-module-source" => SpecEdition::ESNext, - // ArrayBuffer transfer // https://github.com/tc39/proposal-arraybuffer-transfer "arraybuffer-transfer" => SpecEdition::ESNext, @@ -387,7 +381,10 @@ impl SpecEdition { if unknowns.is_empty() { Ok(min_edition) } else { - Err(unknowns) + // TODO - Temporally fallback to ESNext for unknown features. + // Pending on feature->edition map: https://github.com/tc39/test262/issues/4161 + println!("Unknown test262 features in test metadata: {unknowns:?}"); + Ok(SpecEdition::ESNext) } } diff --git a/tools/tc39-test262/src/lib.rs b/tools/tc39-test262/src/lib.rs index 69ec7c4854a..d3caab5d225 100644 --- a/tools/tc39-test262/src/lib.rs +++ b/tools/tc39-test262/src/lib.rs @@ -31,7 +31,8 @@ pub fn clone_test262(commit: Option<&str>, verbose: u8) -> color_eyre::Result<() #[cfg(test)] mod tests { use std::path::Path; - use crate::{Ignored, TEST262_DIRECTORY}; + use crate::{Ignored, MetaData, TEST262_DIRECTORY}; + use crate::edition::SpecEdition; #[test] #[ignore = "manual"] @@ -50,9 +51,51 @@ mod tests { #[test] #[ignore = "manual"] - fn should_read_test_suite() { - let path = Path::new(TEST262_DIRECTORY).join("test").join("language"); + fn should_read_test_suite_and_test() { + let path = Path::new(TEST262_DIRECTORY).join("test").join("language").join("import"); let test_suite = super::read::read_suite(&path, &Ignored::default(), false).unwrap(); assert!(!test_suite.name.is_empty()); + assert!(!test_suite.tests.is_empty()); + + let test_path = &test_suite.tests[0].path; + let test = super::read::read_test(test_path); + assert!(test.is_ok()); + } + + #[test] + fn should_ignore_unknown_features() { + let metadata = MetaData { + description: String::into_boxed_str("test_example description".to_string()), + esid: None, + es5id: None, + es6id: None, + info: String::into_boxed_str("test_example".to_string()), + features: Box::new([String::into_boxed_str("unknown_feature_abc".to_string())]), + includes: Box::new([]), + flags: Box::new([]), + negative: None, + locale: Default::default(), + }; + assert_eq!(Ok(SpecEdition::ESNext), SpecEdition::from_test_metadata(&metadata)); + } + + #[test] + fn should_get_minimal_required_edition_from_test_metadata() { + let metadata = MetaData { + description: String::into_boxed_str("test_example description".to_string()), + esid: None, + es5id: None, + es6id: None, + info: String::into_boxed_str("test_example".to_string()), + features: Box::new([ + String::into_boxed_str("TypedArray".to_string()), // ES6 + String::into_boxed_str("well-formed-json-stringify".to_string())] // ES10 + ), + includes: Box::new([]), + flags: Box::new([]), + negative: None, + locale: Default::default(), + }; + assert_eq!(Ok(SpecEdition::ES10), SpecEdition::from_test_metadata(&metadata)); } } From 746320b764cac322ca0f5742dac0e4f56558ac27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Jastrzebski?= Date: Fri, 13 Sep 2024 12:13:51 +0200 Subject: [PATCH 5/9] removed redundand generics --- tests/tester/src/exec/mod.rs | 10 +++++----- tests/tester/src/main.rs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/tester/src/exec/mod.rs b/tests/tester/src/exec/mod.rs index 9673c91a5ed..07e23591896 100644 --- a/tests/tester/src/exec/mod.rs +++ b/tests/tester/src/exec/mod.rs @@ -23,9 +23,9 @@ use tc39_test262::{ErrorType, Harness, Outcome, Phase, SpecEdition, Test, TestSu use self::js262::WorkerHandles; -pub(crate) trait RunTest { +pub(crate) trait RunTest { /// Runs the test. - fn run(&self, harness: &Harness, verbose: u8, optimizer_options: TO, console: bool) -> TR; + fn run(&self, harness: &Harness, verbose: u8, optimizer_options: OptimizerOptions, console: bool) -> TestResult; /// Runs the test once, in strict or non-strict mode fn run_once( @@ -38,7 +38,7 @@ pub(crate) trait RunTest { ) -> TestResult; } -pub(crate) trait RunTestSuite { +pub(crate) trait RunTestSuite{ /// Runs the test suite. fn run( &self, @@ -51,7 +51,7 @@ pub(crate) trait RunTestSuite { ) -> SuiteResult; } -impl RunTestSuite for TestSuite { +impl RunTestSuite for TestSuite { /// Runs the test suite. fn run( &self, @@ -187,7 +187,7 @@ impl RunTestSuite for TestSuite { } } -impl RunTest for Test { +impl RunTest for Test { /// Runs the test. fn run( &self, diff --git a/tests/tester/src/main.rs b/tests/tester/src/main.rs index 7e4813b95f0..cd26f5982c0 100644 --- a/tests/tester/src/main.rs +++ b/tests/tester/src/main.rs @@ -586,7 +586,7 @@ mod tests { use assert_cmd::Command; fn cmd() -> Command { - let mut cmd: Command = assert_cmd::Command::cargo_bin("boa_tester").unwrap(); + let mut cmd: Command = Command::cargo_bin("boa_tester").unwrap(); cmd.current_dir("../../"); cmd } From 29e18d2d31530243e67d65fca20f46cca2564381 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Jastrzebski?= Date: Fri, 13 Sep 2024 12:21:07 +0200 Subject: [PATCH 6/9] rename tc39_test262 to test262, refactor SpecEdition to open enum --- .gitignore | 2 +- Cargo.lock | 40 ++---- tests/tester/Cargo.toml | 2 +- tests/tester/src/exec/mod.rs | 2 +- tests/tester/src/main.rs | 8 +- tools/test262/.gitignore | 1 + tools/{tc39-test262 => test262}/Cargo.toml | 7 +- .../{tc39-test262 => test262}/src/edition.rs | 122 ++++++++++++++---- tools/{tc39-test262 => test262}/src/git.rs | 0 tools/{tc39-test262 => test262}/src/lib.rs | 0 tools/{tc39-test262 => test262}/src/read.rs | 0 .../{tc39-test262 => test262}/src/structs.rs | 0 .../src/test_files.rs | 0 .../src/test_flags.rs | 0 14 files changed, 121 insertions(+), 63 deletions(-) create mode 100644 tools/test262/.gitignore rename tools/{tc39-test262 => test262}/Cargo.toml (85%) rename tools/{tc39-test262 => test262}/src/edition.rs (82%) rename tools/{tc39-test262 => test262}/src/git.rs (100%) rename tools/{tc39-test262 => test262}/src/lib.rs (100%) rename tools/{tc39-test262 => test262}/src/read.rs (100%) rename tools/{tc39-test262 => test262}/src/structs.rs (100%) rename tools/{tc39-test262 => test262}/src/test_files.rs (100%) rename tools/{tc39-test262 => test262}/src/test_flags.rs (100%) diff --git a/.gitignore b/.gitignore index 040db13443d..e446fa4a7c7 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,7 @@ yarn-error.log .boa_history # test262 testing suite -test262 +/test262 # Profiling *.string_data diff --git a/Cargo.lock b/Cargo.lock index f723b4bce8c..443ecdb99a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -583,7 +583,7 @@ dependencies = [ "rustc-hash 2.0.0", "serde", "serde_json", - "tc39-test262", + "test262", "time", "toml 0.8.19", ] @@ -3057,17 +3057,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_repr" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "serde_spanned" version = "0.6.7" @@ -3280,20 +3269,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" -[[package]] -name = "tc39-test262" -version = "0.19.0" -dependencies = [ - "bitflags 2.6.0", - "clap", - "color-eyre", - "phf", - "rustc-hash 2.0.0", - "serde", - "serde_repr", - "serde_yaml", -] - [[package]] name = "temporal_rs" version = "0.0.3" @@ -3356,6 +3331,19 @@ dependencies = [ "test-case-core", ] +[[package]] +name = "test262" +version = "0.19.0" +dependencies = [ + "bitflags 2.6.0", + "color-eyre", + "phf", + "rustc-hash 2.0.0", + "serde", + "serde_json", + "serde_yaml", +] + [[package]] name = "textwrap" version = "0.16.1" diff --git a/tests/tester/Cargo.toml b/tests/tester/Cargo.toml index bb211fef659..3c2156f32dc 100644 --- a/tests/tester/Cargo.toml +++ b/tests/tester/Cargo.toml @@ -17,7 +17,7 @@ boa_runtime.workspace = true boa_gc.workspace = true clap = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] } -tc39-test262 = { path = "../../tools/tc39-test262"} +test262 = { path = "../../tools/test262"} serde_json.workspace = true colored.workspace = true rustc-hash = { workspace = true, features = ["std"] } diff --git a/tests/tester/src/exec/mod.rs b/tests/tester/src/exec/mod.rs index 07e23591896..bb9277a4194 100644 --- a/tests/tester/src/exec/mod.rs +++ b/tests/tester/src/exec/mod.rs @@ -19,7 +19,7 @@ use colored::Colorize; use rayon::prelude::*; use rustc_hash::FxHashSet; use std::{cell::RefCell, eprintln, path::Path, rc::Rc}; -use tc39_test262::{ErrorType, Harness, Outcome, Phase, SpecEdition, Test, TestSuite}; +use test262::{ErrorType, Harness, Outcome, Phase, SpecEdition, Test, TestSuite}; use self::js262::WorkerHandles; diff --git a/tests/tester/src/main.rs b/tests/tester/src/main.rs index cd26f5982c0..56234218fa2 100644 --- a/tests/tester/src/main.rs +++ b/tests/tester/src/main.rs @@ -15,7 +15,7 @@ mod exec; mod results; use exec::{RunTest, RunTestSuite}; -use tc39_test262::{read, Ignored, SpecEdition, TestFlags}; +use test262::{read, Ignored, SpecEdition, TestFlags}; use self::results::{compare_results, write_json}; @@ -175,8 +175,8 @@ fn main() -> Result<()> { let test262_path = if let Some(path) = test262_path.as_deref() { path } else { - tc39_test262::clone_test262(test262_commit, verbose)?; - Path::new(tc39_test262::TEST262_DIRECTORY) + test262::clone_test262(test262_commit, verbose)?; + Path::new(test262::TEST262_DIRECTORY) } .canonicalize(); let test262_path = &test262_path.wrap_err("could not get the Test262 path")?; @@ -474,6 +474,7 @@ impl VersionedStats { SpecEdition::ES14 => self.es14, SpecEdition::ES15 => self.es15, SpecEdition::ESNext => return None, + _ => return None, }; Some(stats) } @@ -494,6 +495,7 @@ impl VersionedStats { SpecEdition::ES14 => &mut self.es14, SpecEdition::ES15 => &mut self.es15, SpecEdition::ESNext => return None, + _ => return None, }; Some(stats) } diff --git a/tools/test262/.gitignore b/tools/test262/.gitignore new file mode 100644 index 00000000000..0f0c4b7486c --- /dev/null +++ b/tools/test262/.gitignore @@ -0,0 +1 @@ +/test262 diff --git a/tools/tc39-test262/Cargo.toml b/tools/test262/Cargo.toml similarity index 85% rename from tools/tc39-test262/Cargo.toml rename to tools/test262/Cargo.toml index e946334d440..22cbd29ac5c 100644 --- a/tools/tc39-test262/Cargo.toml +++ b/tools/test262/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "tc39-test262" +name = "test262" edition.workspace = true version.workspace = true rust-version.workspace = true @@ -13,12 +13,13 @@ description.workspace = true [dependencies] serde = { workspace = true, features = ["derive"] } phf = { workspace = true, features = ["macros"] } -clap = { workspace = true, features = ["derive"] } rustc-hash = { workspace = true, features = ["std"] } bitflags.workspace = true -serde_repr.workspace = true color-eyre.workspace = true serde_yaml = "0.9.34" # TODO: Track https://github.com/saphyr-rs/saphyr. +[dev-dependencies] +serde_json = "1" + [lints] workspace = true diff --git a/tools/tc39-test262/src/edition.rs b/tools/test262/src/edition.rs similarity index 82% rename from tools/tc39-test262/src/edition.rs rename to tools/test262/src/edition.rs index 41d87b30b16..242ec0366d8 100644 --- a/tools/tc39-test262/src/edition.rs +++ b/tools/test262/src/edition.rs @@ -4,7 +4,6 @@ //! classify all tests per minimum required ECMAScript edition. use crate::{test_flags::TestFlag, MetaData}; -use serde_repr::{Deserialize_repr, Serialize_repr}; use std::fmt::Display; /// Minimum edition required by a specific feature in the `test262` repository. @@ -277,69 +276,104 @@ static FEATURE_EDITION: phf::Map<&'static str, SpecEdition> = phf::phf_map! { /// List of ECMAScript editions that can be tested in the `test262` repository. #[derive( - Debug, - Clone, - Copy, - PartialEq, - Eq, - PartialOrd, - Ord, - Default, - Serialize_repr, - Deserialize_repr, - clap::ValueEnum, + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize, )] -#[repr(u8)] -pub enum SpecEdition { +#[serde(transparent)] +#[repr(transparent)] +pub struct SpecEdition(u8); + +impl Default for SpecEdition { + fn default() -> Self { + SpecEdition::ESNext + } +} + +impl SpecEdition { + /// Deserialize SpecEditon from string (label) for example `es5` + pub fn from_label(value: &str) -> Result { + match &*value.to_uppercase() { + "ES5" => Ok(SpecEdition::ES5), + "ES6" => Ok(SpecEdition::ES6), + "ES7" => Ok(SpecEdition::ES7), + "ES8" => Ok(SpecEdition::ES8), + "ES9" => Ok(SpecEdition::ES9), + "ES10" => Ok(SpecEdition::ES10), + "ES11" => Ok(SpecEdition::ES11), + "ES12" => Ok(SpecEdition::ES12), + "ES13" => Ok(SpecEdition::ES13), + "ES14" => Ok(SpecEdition::ES14), + "ES15" => Ok(SpecEdition::ES15), + "ESNEXT" => Ok(SpecEdition::ESNext), + _ => { + if let Ok(nr) = value.parse::() { + if nr >= 5 && nr <= 15 { + return Ok(SpecEdition(nr)); + } + } + + Err("Invalid SpecEdition label".to_string()) + } + } + } +} + +// clap arg parser +impl From<&str> for SpecEdition { + fn from(value: &str) -> Self { + SpecEdition::from_label(value).unwrap_or_default() + } +} + +impl SpecEdition { /// ECMAScript 5.1 Edition /// /// - ES5 = 5, + pub const ES5: SpecEdition = SpecEdition(5); /// ECMAScript 6th Edition /// /// - ES6, + pub const ES6: SpecEdition = SpecEdition(6); /// ECMAScript 7th Edition /// /// - ES7, + pub const ES7: SpecEdition = SpecEdition(7); /// ECMAScript 8th Edition /// /// - ES8, + pub const ES8: SpecEdition = SpecEdition(8); /// ECMAScript 9th Edition /// /// - ES9, + pub const ES9: SpecEdition = SpecEdition(9); /// ECMAScript 10th Edition /// /// - ES10, + pub const ES10: SpecEdition = SpecEdition(10); /// ECMAScript 11th Edition /// /// - ES11, + pub const ES11: SpecEdition = SpecEdition(11); /// ECMAScript 12th Edition /// /// - ES12, + pub const ES12: SpecEdition = SpecEdition(12); /// ECMAScript 13th Edition /// /// - ES13, + pub const ES13: SpecEdition = SpecEdition(13); /// ECMAScript 14th Edition /// /// - ES14, + pub const ES14: SpecEdition = SpecEdition(14); /// ECMAScript 15th Edition /// /// - ES15, + pub const ES15: SpecEdition = SpecEdition(15); /// The edition being worked on right now. /// /// A draft is currently available [here](https://tc39.es/ecma262). - #[default] - ESNext = 255, + #[allow(non_upper_case_globals)] + pub const ESNext: SpecEdition = SpecEdition(255); } impl Display for SpecEdition { @@ -347,7 +381,7 @@ impl Display for SpecEdition { match *self { Self::ESNext => write!(f, "ECMAScript Next"), Self::ES5 => write!(f, "ECMAScript 5.1"), - v => write!(f, "ECMAScript {}", v as u8), + v => write!(f, "ECMAScript {}", v.0), } } } @@ -407,3 +441,35 @@ impl SpecEdition { .into_iter() } } + +#[cfg(test)] +mod test { + use serde_json::json; + + use super::SpecEdition; + + #[test] + fn serialize_spec_edition() { + let spec: SpecEdition = serde_json::from_str("6").expect("SpecEdition from number"); + assert_eq!(spec, SpecEdition::ES6); + + let spec_nr = serde_json::to_value(SpecEdition::ES13).expect("SpecEdition to number"); + assert_eq!(spec_nr, json!(13)) + } + + #[test] + fn deserialize_from_label() { + assert_eq!( + SpecEdition::from_label("es6").unwrap_or_default(), + SpecEdition::ES6 + ); + assert_eq!( + SpecEdition::from_label("ES6").unwrap_or_default(), + SpecEdition::ES6 + ); + assert_eq!( + SpecEdition::from_label("6").unwrap_or_default(), + SpecEdition::ES6 + ); + } +} diff --git a/tools/tc39-test262/src/git.rs b/tools/test262/src/git.rs similarity index 100% rename from tools/tc39-test262/src/git.rs rename to tools/test262/src/git.rs diff --git a/tools/tc39-test262/src/lib.rs b/tools/test262/src/lib.rs similarity index 100% rename from tools/tc39-test262/src/lib.rs rename to tools/test262/src/lib.rs diff --git a/tools/tc39-test262/src/read.rs b/tools/test262/src/read.rs similarity index 100% rename from tools/tc39-test262/src/read.rs rename to tools/test262/src/read.rs diff --git a/tools/tc39-test262/src/structs.rs b/tools/test262/src/structs.rs similarity index 100% rename from tools/tc39-test262/src/structs.rs rename to tools/test262/src/structs.rs diff --git a/tools/tc39-test262/src/test_files.rs b/tools/test262/src/test_files.rs similarity index 100% rename from tools/tc39-test262/src/test_files.rs rename to tools/test262/src/test_files.rs diff --git a/tools/tc39-test262/src/test_flags.rs b/tools/test262/src/test_flags.rs similarity index 100% rename from tools/tc39-test262/src/test_flags.rs rename to tools/test262/src/test_flags.rs From d4f55fc2185ecc9e917efc19f67df5c1d5bf5147 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Jastrzebski?= Date: Fri, 13 Sep 2024 15:20:04 +0200 Subject: [PATCH 7/9] test262 Error --- tools/test262/src/error.rs | 61 ++++++++++++++++++++++ tools/test262/src/lib.rs | 33 ++++++++---- tools/test262/src/read.rs | 91 ++++++++++++++++++++------------- tools/test262/src/test_files.rs | 13 +++-- 4 files changed, 145 insertions(+), 53 deletions(-) create mode 100644 tools/test262/src/error.rs diff --git a/tools/test262/src/error.rs b/tools/test262/src/error.rs new file mode 100644 index 00000000000..2cbe2166bbc --- /dev/null +++ b/tools/test262/src/error.rs @@ -0,0 +1,61 @@ +use std::path::{Path, PathBuf}; + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Errors +#[allow(missing_docs)] +pub enum Error262 { + InvalidHarnessDirecory { + path: String, + }, + HarnessFileReadError { + path: String, + }, + FailedToGetFileType { + path: String, + }, + // test + InvalidPathToTest, + SubTestReadError { + path: String, + suite: String, + error: Box, + }, + // test suite + InvalidPathToTestSuite, + SubSuiteReadError { + path: String, + suite: String, + error: Box, + }, + // test metadata + MetadataUnknownFeatures(Vec), + MetadateReadError { + path: String, + }, + MetadateParseError { + path: String, + }, +} +impl std::error::Error for Error262 {} + +impl std::fmt::Display for Error262 { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } +} + +pub(super) trait PathToString { + fn string(&self) -> String; +} + +impl PathToString for PathBuf { + fn string(&self) -> String { + self.display().to_string() + } +} + +impl PathToString for Path { + fn string(&self) -> String { + self.display().to_string() + } +} diff --git a/tools/test262/src/lib.rs b/tools/test262/src/lib.rs index d3caab5d225..bd7e4a58df9 100644 --- a/tools/test262/src/lib.rs +++ b/tools/test262/src/lib.rs @@ -3,13 +3,15 @@ mod edition; mod structs; mod test_files; mod test_flags; +mod error; pub mod git; pub mod read; -pub use structs::{ErrorType, Outcome, Phase, Ignored}; +pub use error::Error262; +pub use edition::SpecEdition; +pub use structs::{ErrorType, Ignored, Outcome, Phase}; pub use test_files::{Harness, HarnessFile, MetaData, Test, TestSuite}; pub use test_flags::TestFlags; -pub use edition::SpecEdition; /// Repository Url pub const TEST262_REPOSITORY: &str = "https://github.com/tc39/test262"; @@ -30,9 +32,9 @@ pub fn clone_test262(commit: Option<&str>, verbose: u8) -> color_eyre::Result<() #[cfg(test)] mod tests { - use std::path::Path; - use crate::{Ignored, MetaData, TEST262_DIRECTORY}; use crate::edition::SpecEdition; + use crate::{Ignored, MetaData, TEST262_DIRECTORY}; + use std::path::Path; #[test] #[ignore = "manual"] @@ -52,7 +54,10 @@ mod tests { #[test] #[ignore = "manual"] fn should_read_test_suite_and_test() { - let path = Path::new(TEST262_DIRECTORY).join("test").join("language").join("import"); + let path = Path::new(TEST262_DIRECTORY) + .join("test") + .join("language") + .join("import"); let test_suite = super::read::read_suite(&path, &Ignored::default(), false).unwrap(); assert!(!test_suite.name.is_empty()); assert!(!test_suite.tests.is_empty()); @@ -76,7 +81,10 @@ mod tests { negative: None, locale: Default::default(), }; - assert_eq!(Ok(SpecEdition::ESNext), SpecEdition::from_test_metadata(&metadata)); + assert_eq!( + Ok(SpecEdition::ESNext), + SpecEdition::from_test_metadata(&metadata) + ); } #[test] @@ -87,15 +95,20 @@ mod tests { es5id: None, es6id: None, info: String::into_boxed_str("test_example".to_string()), - features: Box::new([ - String::into_boxed_str("TypedArray".to_string()), // ES6 - String::into_boxed_str("well-formed-json-stringify".to_string())] // ES10 + features: Box::new( + [ + String::into_boxed_str("TypedArray".to_string()), // ES6 + String::into_boxed_str("well-formed-json-stringify".to_string()), + ], // ES10 ), includes: Box::new([]), flags: Box::new([]), negative: None, locale: Default::default(), }; - assert_eq!(Ok(SpecEdition::ES10), SpecEdition::from_test_metadata(&metadata)); + assert_eq!( + Ok(SpecEdition::ES10), + SpecEdition::from_test_metadata(&metadata) + ); } } diff --git a/tools/test262/src/read.rs b/tools/test262/src/read.rs index caf6b6f2e77..aff63237b52 100644 --- a/tools/test262/src/read.rs +++ b/tools/test262/src/read.rs @@ -1,23 +1,21 @@ //! Module to read the list of test suites from disk. +use crate::{error::PathToString, Error262}; + use super::test_files::{Harness, HarnessFile, MetaData, Test, TestSuite}; +use rustc_hash::FxHashMap; use std::{ ffi::OsStr, fs, path::{Path, PathBuf}, }; -use color_eyre::{ - eyre::{OptionExt, WrapErr}, - Result, -}; -use rustc_hash::FxHashMap; - /// Reads the Test262 defined bindings. -pub fn read_harness(test262_path: &Path) -> Result { - fn read_harness_file(path: PathBuf) -> Result { - let content = fs::read_to_string(path.as_path()) - .wrap_err_with(|| format!("error reading the harness file `{}`", path.display()))?; +pub fn read_harness(test262_path: &Path) -> Result { + fn read_harness_file(path: PathBuf) -> Result { + let content = fs::read_to_string(&path).map_err(|_| Error262::HarnessFileReadError { + path: path.string(), + })?; Ok(HarnessFile { content: content.into_boxed_str(), @@ -26,10 +24,13 @@ pub fn read_harness(test262_path: &Path) -> Result { } let mut includes = FxHashMap::default(); - for entry in fs::read_dir(test262_path.join("harness")) - .wrap_err("error reading the harness directory")? - { - let entry = entry?; + let harness_dir = test262_path.join("harness"); + for entry in fs::read_dir(&harness_dir).map_err(|_| Error262::InvalidHarnessDirecory { + path: harness_dir.string(), + })? { + let entry = entry.map_err(|_| Error262::InvalidHarnessDirecory { + path: harness_dir.string(), + })?; let file_name = entry.file_name(); let file_name = file_name.to_string_lossy(); @@ -59,11 +60,11 @@ pub fn read_suite( path: &Path, ignored: &crate::structs::Ignored, mut ignore_suite: bool, -) -> Result { +) -> Result { let name = path .file_name() .and_then(OsStr::to_str) - .ok_or_eyre("invalid path for test suite")?; + .ok_or(Error262::InvalidPathToTestSuite)?; ignore_suite |= ignored.contains_test(name); @@ -71,16 +72,25 @@ pub fn read_suite( let mut tests = Vec::new(); // TODO: iterate in parallel - for entry in path.read_dir().wrap_err("could not retrieve entry")? { - let entry = entry?; - let filetype = entry.file_type().wrap_err("could not retrieve file type")?; + for entry in path + .read_dir() + .map_err(|_| Error262::InvalidPathToTestSuite)? + { + let entry = entry.map_err(|_| Error262::InvalidPathToTestSuite)?; + let filetype = entry + .file_type() + .map_err(|_| Error262::FailedToGetFileType { + path: entry.path().string(), + })?; if filetype.is_dir() { suites.push( - read_suite(entry.path().as_path(), ignored, ignore_suite).wrap_err_with(|| { - let path = entry.path(); - let suite = path.display(); - format!("error reading sub-suite {suite}") + read_suite(entry.path().as_path(), ignored, ignore_suite).map_err(|e| { + Error262::SubSuiteReadError { + path: entry.path().string(), + suite: path.string(), + error: Box::new(e), + } })?, ); continue; @@ -101,10 +111,10 @@ pub fn read_suite( continue; } - let mut test = read_test(&path).wrap_err_with(|| { - let path = entry.path(); - let suite = path.display(); - format!("error reading test {suite}") + let mut test = read_test(&path).map_err(|e| Error262::SubTestReadError { + path: entry.path().string(), + suite: path.string(), + error: Box::new(e), })?; if ignore_suite @@ -129,11 +139,11 @@ pub fn read_suite( } /// Reads information about a given test case. -pub fn read_test(path: &Path) -> Result { +pub fn read_test(path: &Path) -> Result { let name = path .file_stem() .and_then(OsStr::to_str) - .ok_or_eyre("invalid path for test")?; + .ok_or(Error262::InvalidPathToTest)?; let metadata = read_metadata(path)?; @@ -141,16 +151,25 @@ pub fn read_test(path: &Path) -> Result { } /// Reads the metadata from the input test code. -pub fn read_metadata(test: &Path) -> Result { - let code = fs::read_to_string(test)?; +pub fn read_metadata(test: &Path) -> Result { + let code = fs::read_to_string(test).map_err(|_| Error262::MetadateReadError { + path: test.string(), + })?; let (_, metadata) = code .split_once("/*---") - .ok_or_eyre("invalid test metadata")?; - let (metadata, _) = metadata - .split_once("---*/") - .ok_or_eyre("invalid test metadata")?; + .ok_or_else(|| Error262::MetadateParseError { + path: test.string(), + })?; + let (metadata, _) = + metadata + .split_once("---*/") + .ok_or_else(|| Error262::MetadateParseError { + path: test.string(), + })?; let metadata = metadata.replace('\r', "\n"); - serde_yaml::from_str(&metadata).map_err(Into::into) + serde_yaml::from_str(&metadata).map_err(|_| Error262::MetadateParseError { + path: test.string(), + }) } diff --git a/tools/test262/src/test_files.rs b/tools/test262/src/test_files.rs index 53e94112b97..86b0ba2bc8f 100644 --- a/tools/test262/src/test_files.rs +++ b/tools/test262/src/test_files.rs @@ -1,9 +1,9 @@ -use std::path::Path; use rustc_hash::{FxHashMap, FxHashSet}; use serde::Deserialize; +use std::path::Path; -use crate::{TestFlags, SpecEdition}; use super::structs::*; +use crate::{SpecEdition, TestFlags}; /// All the harness include files. #[allow(missing_docs)] @@ -44,13 +44,14 @@ pub struct Test { impl Test { /// Creates a new test. - pub fn new(name: N, path: C, metadata: MetaData) -> color_eyre::Result + pub fn new(name: N, path: C, metadata: MetaData) -> Result where N: Into>, C: Into>, { - let edition = SpecEdition::from_test_metadata(&metadata) - .map_err(|feats| color_eyre::eyre::eyre!("test metadata contained unknown features: {feats:?}"))?; + let edition = SpecEdition::from_test_metadata(&metadata).map_err(|feats| { + crate::Error262::MetadataUnknownFeatures(feats.into_iter().map(String::from).collect()) + })?; Ok(Self { edition, @@ -81,7 +82,6 @@ impl Test { } } - /// Represents a test suite. #[allow(missing_docs)] #[derive(Debug, Clone)] @@ -122,4 +122,3 @@ pub struct MetaData { pub struct Locale { locale: Box<[Box]>, } - From 6dc72cae59d73cd3f1b2928b62897c2985c08767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Jastrzebski?= Date: Fri, 13 Sep 2024 15:35:34 +0200 Subject: [PATCH 8/9] test262 hermetization --- tests/tester/src/main.rs | 8 +- tools/test262/src/git.rs | 2 +- tools/test262/src/lib.rs | 10 +- tools/test262/src/read.rs | 294 ++++++++++++++++++++------------------ 4 files changed, 163 insertions(+), 151 deletions(-) diff --git a/tests/tester/src/main.rs b/tests/tester/src/main.rs index 56234218fa2..ddb61dc6566 100644 --- a/tests/tester/src/main.rs +++ b/tests/tester/src/main.rs @@ -15,7 +15,7 @@ mod exec; mod results; use exec::{RunTest, RunTestSuite}; -use test262::{read, Ignored, SpecEdition, TestFlags}; +use test262::{Harness, Ignored, SpecEdition, TestFlags}; use self::results::{compare_results, write_json}; @@ -233,10 +233,10 @@ fn run_test_suite( if verbose != 0 { println!("Loading the test suite..."); } - let harness = read::read_harness(test262_path).wrap_err("could not read harness")?; + let harness = Harness::read(test262_path).wrap_err("could not read harness")?; if suite.to_string_lossy().ends_with(".js") { - let test = read::read_test(&test262_path.join(suite)).wrap_err_with(|| { + let test = test262::Test::read(&test262_path.join(suite)).wrap_err_with(|| { let suite = suite.display(); format!("could not read the test {suite}") })?; @@ -254,7 +254,7 @@ fn run_test_suite( println!(); } else { - let suite = read::read_suite(&test262_path.join(suite), config.ignored(), false) + let suite = test262::TestSuite::read(&test262_path.join(suite), config.ignored(), false) .wrap_err_with(|| { let suite = suite.display(); format!("could not read the suite {suite}") diff --git a/tools/test262/src/git.rs b/tools/test262/src/git.rs index b674e6d94c9..c61c7877f78 100644 --- a/tools/test262/src/git.rs +++ b/tools/test262/src/git.rs @@ -55,7 +55,7 @@ fn reset_commit(directory: &str, commit: &str, verbose: u8) -> Result<()> { } /// Clone repository -pub fn clone( +pub(super) fn clone( directory: &str, repor_url: &str, // "https://github.com/tc39/test262" baranch: &str, // "origin/main" diff --git a/tools/test262/src/lib.rs b/tools/test262/src/lib.rs index bd7e4a58df9..037b7f9abdc 100644 --- a/tools/test262/src/lib.rs +++ b/tools/test262/src/lib.rs @@ -3,10 +3,10 @@ mod edition; mod structs; mod test_files; mod test_flags; +mod git; +mod read; mod error; -pub mod git; -pub mod read; pub use error::Error262; pub use edition::SpecEdition; pub use structs::{ErrorType, Ignored, Outcome, Phase}; @@ -45,7 +45,7 @@ mod tests { #[test] #[ignore = "manual"] fn should_read_harness() { - let harness = super::read::read_harness(Path::new(TEST262_DIRECTORY)).unwrap(); + let harness = super::Harness::read(Path::new(TEST262_DIRECTORY)).unwrap(); assert!(harness.assert.path.is_file()); assert!(harness.sta.path.is_file()); assert!(harness.doneprint_handle.path.is_file()); @@ -58,12 +58,12 @@ mod tests { .join("test") .join("language") .join("import"); - let test_suite = super::read::read_suite(&path, &Ignored::default(), false).unwrap(); + let test_suite = super::TestSuite::read(&path, &Ignored::default(), false).unwrap(); assert!(!test_suite.name.is_empty()); assert!(!test_suite.tests.is_empty()); let test_path = &test_suite.tests[0].path; - let test = super::read::read_test(test_path); + let test = super::Test::read(test_path); assert!(test.is_ok()); } diff --git a/tools/test262/src/read.rs b/tools/test262/src/read.rs index aff63237b52..feb7086c67d 100644 --- a/tools/test262/src/read.rs +++ b/tools/test262/src/read.rs @@ -10,166 +10,178 @@ use std::{ path::{Path, PathBuf}, }; -/// Reads the Test262 defined bindings. -pub fn read_harness(test262_path: &Path) -> Result { - fn read_harness_file(path: PathBuf) -> Result { - let content = fs::read_to_string(&path).map_err(|_| Error262::HarnessFileReadError { - path: path.string(), - })?; - - Ok(HarnessFile { - content: content.into_boxed_str(), - path: path.into_boxed_path(), - }) - } - let mut includes = FxHashMap::default(); +impl Harness { + /// Reads the Test262 defined bindings. + pub fn read(test262_path: &Path) -> Result { + fn read_harness_file(path: PathBuf) -> Result { + let content = + fs::read_to_string(&path).map_err(|_| Error262::HarnessFileReadError { + path: path.string(), + })?; + + Ok(HarnessFile { + content: content.into_boxed_str(), + path: path.into_boxed_path(), + }) + } + let mut includes = FxHashMap::default(); - let harness_dir = test262_path.join("harness"); - for entry in fs::read_dir(&harness_dir).map_err(|_| Error262::InvalidHarnessDirecory { - path: harness_dir.string(), - })? { - let entry = entry.map_err(|_| Error262::InvalidHarnessDirecory { + let harness_dir = test262_path.join("harness"); + for entry in fs::read_dir(&harness_dir).map_err(|_| Error262::InvalidHarnessDirecory { path: harness_dir.string(), - })?; - let file_name = entry.file_name(); - let file_name = file_name.to_string_lossy(); - - if file_name == "assert.js" || file_name == "sta.js" || file_name == "doneprintHandle.js" { - continue; + })? { + let entry = entry.map_err(|_| Error262::InvalidHarnessDirecory { + path: harness_dir.string(), + })?; + let file_name = entry.file_name(); + let file_name = file_name.to_string_lossy(); + + if file_name == "assert.js" + || file_name == "sta.js" + || file_name == "doneprintHandle.js" + { + continue; + } + + includes.insert( + file_name.into_owned().into_boxed_str(), + read_harness_file(entry.path())?, + ); } - - includes.insert( - file_name.into_owned().into_boxed_str(), - read_harness_file(entry.path())?, - ); + let assert = read_harness_file(test262_path.join("harness/assert.js"))?; + let sta = read_harness_file(test262_path.join("harness/sta.js"))?; + let doneprint_handle = read_harness_file(test262_path.join("harness/doneprintHandle.js"))?; + + Ok(Harness { + assert, + sta, + doneprint_handle, + includes, + }) } - let assert = read_harness_file(test262_path.join("harness/assert.js"))?; - let sta = read_harness_file(test262_path.join("harness/sta.js"))?; - let doneprint_handle = read_harness_file(test262_path.join("harness/doneprintHandle.js"))?; - - Ok(Harness { - assert, - sta, - doneprint_handle, - includes, - }) } -/// Reads a test suite in the given path. -pub fn read_suite( - path: &Path, - ignored: &crate::structs::Ignored, - mut ignore_suite: bool, -) -> Result { - let name = path - .file_name() - .and_then(OsStr::to_str) - .ok_or(Error262::InvalidPathToTestSuite)?; - - ignore_suite |= ignored.contains_test(name); - - let mut suites = Vec::new(); - let mut tests = Vec::new(); - - // TODO: iterate in parallel - for entry in path - .read_dir() - .map_err(|_| Error262::InvalidPathToTestSuite)? - { - let entry = entry.map_err(|_| Error262::InvalidPathToTestSuite)?; - let filetype = entry - .file_type() - .map_err(|_| Error262::FailedToGetFileType { +impl TestSuite { + /// Reads a test suite in the given path. + pub fn read( + path: &Path, + ignored: &crate::structs::Ignored, + mut ignore_suite: bool, + ) -> Result { + let name = path + .file_name() + .and_then(OsStr::to_str) + .ok_or(Error262::InvalidPathToTestSuite)?; + + ignore_suite |= ignored.contains_test(name); + + let mut suites = Vec::new(); + let mut tests = Vec::new(); + + // TODO: iterate in parallel + for entry in path + .read_dir() + .map_err(|_| Error262::InvalidPathToTestSuite)? + { + let entry = entry.map_err(|_| Error262::InvalidPathToTestSuite)?; + let filetype = entry + .file_type() + .map_err(|_| Error262::FailedToGetFileType { + path: entry.path().string(), + })?; + + if filetype.is_dir() { + suites.push( + TestSuite::read(entry.path().as_path(), ignored, ignore_suite).map_err(|e| { + Error262::SubSuiteReadError { + path: entry.path().string(), + suite: path.string(), + error: Box::new(e), + } + })?, + ); + continue; + } + + let path = entry.path(); + + if path.extension() != Some(OsStr::new("js")) { + // Ignore files that aren't executable. + continue; + } + + if path + .file_stem() + .is_some_and(|stem| stem.as_encoded_bytes().ends_with(b"FIXTURE")) + { + // Ignore files that are fixtures. + continue; + } + + let mut test = Test::read(&path).map_err(|e| Error262::SubTestReadError { path: entry.path().string(), + suite: path.string(), + error: Box::new(e), })?; - if filetype.is_dir() { - suites.push( - read_suite(entry.path().as_path(), ignored, ignore_suite).map_err(|e| { - Error262::SubSuiteReadError { - path: entry.path().string(), - suite: path.string(), - error: Box::new(e), - } - })?, - ); - continue; - } - - let path = entry.path(); - - if path.extension() != Some(OsStr::new("js")) { - // Ignore files that aren't executable. - continue; + if ignore_suite + || ignored.contains_any_flag(test.flags) + || ignored.contains_test(&test.name) + || test + .features + .iter() + .any(|feat| ignored.contains_feature(feat)) + { + test.set_ignored(); + } + tests.push(test); } - if path - .file_stem() - .is_some_and(|stem| stem.as_encoded_bytes().ends_with(b"FIXTURE")) - { - // Ignore files that are fixtures. - continue; - } - - let mut test = read_test(&path).map_err(|e| Error262::SubTestReadError { - path: entry.path().string(), - suite: path.string(), - error: Box::new(e), - })?; - - if ignore_suite - || ignored.contains_any_flag(test.flags) - || ignored.contains_test(&test.name) - || test - .features - .iter() - .any(|feat| ignored.contains_feature(feat)) - { - test.set_ignored(); - } - tests.push(test); + Ok(TestSuite { + name: name.into(), + path: Box::from(path), + suites: suites.into_boxed_slice(), + tests: tests.into_boxed_slice(), + }) } - - Ok(TestSuite { - name: name.into(), - path: Box::from(path), - suites: suites.into_boxed_slice(), - tests: tests.into_boxed_slice(), - }) } -/// Reads information about a given test case. -pub fn read_test(path: &Path) -> Result { - let name = path - .file_stem() - .and_then(OsStr::to_str) - .ok_or(Error262::InvalidPathToTest)?; +impl Test { + /// Reads information about a given test case. + pub fn read(path: &Path) -> Result { + let name = path + .file_stem() + .and_then(OsStr::to_str) + .ok_or(Error262::InvalidPathToTest)?; - let metadata = read_metadata(path)?; + let metadata = MetaData::read(path)?; - Test::new(name, path, metadata) + Test::new(name, path, metadata) + } } -/// Reads the metadata from the input test code. -pub fn read_metadata(test: &Path) -> Result { - let code = fs::read_to_string(test).map_err(|_| Error262::MetadateReadError { - path: test.string(), - })?; - - let (_, metadata) = code - .split_once("/*---") - .ok_or_else(|| Error262::MetadateParseError { +impl MetaData { + /// Reads the metadata from the input test code. + pub fn read(test: &Path) -> Result { + let code = fs::read_to_string(test).map_err(|_| Error262::MetadateReadError { path: test.string(), })?; - let (metadata, _) = - metadata - .split_once("---*/") - .ok_or_else(|| Error262::MetadateParseError { - path: test.string(), - })?; - let metadata = metadata.replace('\r', "\n"); - serde_yaml::from_str(&metadata).map_err(|_| Error262::MetadateParseError { - path: test.string(), - }) + let (_, metadata) = + code.split_once("/*---") + .ok_or_else(|| Error262::MetadateParseError { + path: test.string(), + })?; + let (metadata, _) = + metadata + .split_once("---*/") + .ok_or_else(|| Error262::MetadateParseError { + path: test.string(), + })?; + let metadata = metadata.replace('\r', "\n"); + + serde_yaml::from_str(&metadata).map_err(|_| Error262::MetadateParseError { + path: test.string(), + }) + } } From 027fbb4f99d98cae5a9ec96a33ac542a181f8051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Jastrzebski?= Date: Fri, 20 Sep 2024 14:48:36 +0200 Subject: [PATCH 9/9] WIP - test262:read fn --- tests/tester/src/main.rs | 192 ++++++++++++++++++----------------- tools/test262/.gitignore | 1 + tools/test262/src/error.rs | 7 ++ tools/test262/src/git.rs | 3 + tools/test262/src/lib.rs | 166 +++++++++++++++++++++++++++++- tools/test262/src/structs.rs | 2 +- 6 files changed, 274 insertions(+), 97 deletions(-) diff --git a/tests/tester/src/main.rs b/tests/tester/src/main.rs index ddb61dc6566..6268355217a 100644 --- a/tests/tester/src/main.rs +++ b/tests/tester/src/main.rs @@ -175,7 +175,8 @@ fn main() -> Result<()> { let test262_path = if let Some(path) = test262_path.as_deref() { path } else { - test262::clone_test262(test262_commit, verbose)?; + // TODO remove test262 fn (unused) + // test262::clone_test262(test262_commit, verbose)?; Path::new(test262::TEST262_DIRECTORY) } .canonicalize(); @@ -185,6 +186,7 @@ fn main() -> Result<()> { &config, verbose, !disable_parallelism, + test262_commit, test262_path, suite.as_path(), output.as_deref(), @@ -212,6 +214,7 @@ fn run_test_suite( config: &Config, verbose: u8, parallel: bool, + test262_commit: Option<&str>, test262_path: &Path, suite: &Path, output: Option<&Path>, @@ -233,111 +236,118 @@ fn run_test_suite( if verbose != 0 { println!("Loading the test suite..."); } - let harness = Harness::read(test262_path).wrap_err("could not read harness")?; - if suite.to_string_lossy().ends_with(".js") { - let test = test262::Test::read(&test262_path.join(suite)).wrap_err_with(|| { - let suite = suite.display(); - format!("could not read the test {suite}") - })?; + let result = test262::read( + suite.to_owned(), + test262::ReadOptions { + test262_path: test262_path.to_owned(), + test262_commit, // TODO - Some(config.commit().to_string()) - priority config/arg?? + ignored: Some(config.ignored().clone()), + verbose, + ..Default::default() + }, + ) + .wrap_err_with(|| { + let suite = suite.display(); + format!("could not read the test {suite}") + })?; + + match result { + test262::ReadResult::Test(harness, test) => { + if test.edition <= edition { + if verbose != 0 { + println!("Test loaded, starting..."); + } + test.run(&harness, verbose, optimizer_options, console); + } else { + println!( + "Minimum spec edition of test is bigger than the specified edition. Skipping." + ); + } - if test.edition <= edition { + println!(); + Ok(()) + } + test262::ReadResult::TestSuite(harness, suite) => { if verbose != 0 { - println!("Test loaded, starting..."); + println!("Test suite loaded, starting tests..."); } - test.run(&harness, verbose, optimizer_options, console); - } else { - println!( - "Minimum spec edition of test is bigger than the specified edition. Skipping." + let results = suite.run( + &harness, + verbose, + parallel, + edition, + optimizer_options, + console, ); - } - - println!(); - } else { - let suite = test262::TestSuite::read(&test262_path.join(suite), config.ignored(), false) - .wrap_err_with(|| { - let suite = suite.display(); - format!("could not read the suite {suite}") - })?; - if verbose != 0 { - println!("Test suite loaded, starting tests..."); - } - let results = suite.run( - &harness, - verbose, - parallel, - edition, - optimizer_options, - console, - ); - - if versioned { - let mut table = comfy_table::Table::new(); - table.load_preset(comfy_table::presets::UTF8_HORIZONTAL_ONLY); - table.set_header(vec![ - "Edition", "Total", "Passed", "Ignored", "Failed", "Panics", "%", - ]); - for column in table.column_iter_mut().skip(1) { - column.set_cell_alignment(comfy_table::CellAlignment::Right); - } - for (v, stats) in SpecEdition::all_editions() - .filter(|v| *v <= edition) - .map(|v| { - let stats = results.versioned_stats.get(v).unwrap_or(results.stats); - (v, stats) - }) - { + if versioned { + let mut table = comfy_table::Table::new(); + table.load_preset(comfy_table::presets::UTF8_HORIZONTAL_ONLY); + table.set_header(vec![ + "Edition", "Total", "Passed", "Ignored", "Failed", "Panics", "%", + ]); + for column in table.column_iter_mut().skip(1) { + column.set_cell_alignment(comfy_table::CellAlignment::Right); + } + for (v, stats) in SpecEdition::all_editions() + .filter(|v| *v <= edition) + .map(|v| { + let stats = results.versioned_stats.get(v).unwrap_or(results.stats); + (v, stats) + }) + { + let Statistics { + total, + passed, + ignored, + panic, + } = stats; + let failed = total - passed - ignored; + let conformance = (passed as f64 / total as f64) * 100.0; + let conformance = format!("{conformance:.2}"); + table.add_row(vec![ + v.to_string(), + total.to_string(), + passed.to_string(), + ignored.to_string(), + failed.to_string(), + panic.to_string(), + conformance, + ]); + } + println!("\n\nResults\n"); + println!("{table}"); + } else { let Statistics { total, passed, ignored, panic, - } = stats; - let failed = total - passed - ignored; - let conformance = (passed as f64 / total as f64) * 100.0; - let conformance = format!("{conformance:.2}"); - table.add_row(vec![ - v.to_string(), - total.to_string(), - passed.to_string(), - ignored.to_string(), - failed.to_string(), - panic.to_string(), - conformance, - ]); + } = results.stats; + println!("\n\nResults ({edition}):"); + println!("Total tests: {total}"); + println!("Passed tests: {}", passed.to_string().green()); + println!("Ignored tests: {}", ignored.to_string().yellow()); + println!( + "Failed tests: {} ({})", + (total - passed - ignored).to_string().red(), + format!("{panic} panics").red() + ); + println!( + "Conformance: {:.2}%", + (passed as f64 / total as f64) * 100.0 + ); + + if let Some(output) = output { + write_json(results, output, verbose, test262_path) + .wrap_err("could not write the results to the output JSON file")?; + } } - println!("\n\nResults\n"); - println!("{table}"); - } else { - let Statistics { - total, - passed, - ignored, - panic, - } = results.stats; - println!("\n\nResults ({edition}):"); - println!("Total tests: {total}"); - println!("Passed tests: {}", passed.to_string().green()); - println!("Ignored tests: {}", ignored.to_string().yellow()); - println!( - "Failed tests: {} ({})", - (total - passed - ignored).to_string().red(), - format!("{panic} panics").red() - ); - println!( - "Conformance: {:.2}%", - (passed as f64 / total as f64) * 100.0 - ); - } - if let Some(output) = output { - write_json(results, output, verbose, test262_path) - .wrap_err("could not write the results to the output JSON file")?; + Ok(()) } } - - Ok(()) } /// Represents a tests statistic diff --git a/tools/test262/.gitignore b/tools/test262/.gitignore index 0f0c4b7486c..325b51d3910 100644 --- a/tools/test262/.gitignore +++ b/tools/test262/.gitignore @@ -1 +1,2 @@ /test262 +/test262_2 diff --git a/tools/test262/src/error.rs b/tools/test262/src/error.rs index 2cbe2166bbc..8c7a4613ce6 100644 --- a/tools/test262/src/error.rs +++ b/tools/test262/src/error.rs @@ -4,6 +4,13 @@ use std::path::{Path, PathBuf}; /// Errors #[allow(missing_docs)] pub enum Error262 { + InvalidRepo262Path { + path: PathBuf + }, + InvalidSuitePath { + path: PathBuf + }, + InvalidHarnessDirecory { path: String, }, diff --git a/tools/test262/src/git.rs b/tools/test262/src/git.rs index c61c7877f78..aceeffbf2a9 100644 --- a/tools/test262/src/git.rs +++ b/tools/test262/src/git.rs @@ -54,6 +54,9 @@ fn reset_commit(directory: &str, commit: &str, verbose: u8) -> Result<()> { Ok(()) } + +// Todo remove eyre + /// Clone repository pub(super) fn clone( directory: &str, diff --git a/tools/test262/src/lib.rs b/tools/test262/src/lib.rs index 037b7f9abdc..a44c7942d56 100644 --- a/tools/test262/src/lib.rs +++ b/tools/test262/src/lib.rs @@ -1,14 +1,16 @@ //! TC39 test262 mod edition; +mod error; +mod git; +mod read; mod structs; mod test_files; mod test_flags; -mod git; -mod read; -mod error; -pub use error::Error262; +use std::path::PathBuf; + pub use edition::SpecEdition; +pub use error::Error262; pub use structs::{ErrorType, Ignored, Outcome, Phase}; pub use test_files::{Harness, HarnessFile, MetaData, Test, TestSuite}; pub use test_flags::TestFlags; @@ -30,11 +32,109 @@ pub fn clone_test262(commit: Option<&str>, verbose: u8) -> color_eyre::Result<() ) } +/// Test Read Result +#[derive(Debug)] +pub enum ReadResult { + /// Single Test + Test(Harness, Test), + /// Test Suite + TestSuite(Harness, TestSuite) +} + +#[derive(Debug)] +/// Test Reading Options +pub struct ReadOptions<'a> { + /// Git 262 repository path: + /// Example: "/home/test262" or "./test262" + pub test262_path: PathBuf, + /// Example: "https://github.com/tc39/test262" + pub test262_repository: Option<&'a str>, + /// Example: "origin/main" + pub test262_branch: Option<&'a str>, + /// Example "de3a117f02e26a53f8b7cb41f6b4a0b8473c5db4" + pub test262_commit: Option<&'a str>, + /// Ignored configuration + pub ignored: Option, + /// Verbose + pub verbose: u8, +} + +impl Default for ReadOptions<'_> { + fn default() -> Self { + Self { + test262_path: PathBuf::from(TEST262_DIRECTORY), + test262_branch: Default::default(), + test262_repository: Default::default(), + test262_commit: Default::default(), + ignored: Default::default(), + verbose: 0, + } + } +} + +/// Read Test/TestSuite from tc38/test262 repository +/// Example: +/// suite: test/intl402/constructors-string-and-single-element-array.js +/// options: Default:default() +pub fn read(suite: PathBuf, options: ReadOptions<'_>) -> Result { + let test_262_path = options.test262_path; + let ignore = options.ignored.unwrap_or_default(); + let verbose = options.verbose; + + // Clone test262 repo + if !test_262_path.exists() { + let repo = options.test262_repository.unwrap_or(TEST262_REPOSITORY); + let branch = options.test262_branch.unwrap_or("origin/main"); + let commit = options.test262_commit; + git::clone( + &test_262_path.to_string_lossy(), + repo, + branch, + commit, + verbose, + ).expect("Failed to clone test262") // TODO + } + + // Repo path validation + if !test_262_path.is_dir() { + return Err(Error262::InvalidRepo262Path { + path: test_262_path, + }); + } + let Ok(test_262_path) = test_262_path.canonicalize() else { + return Err(Error262::InvalidRepo262Path { + path: test_262_path, + }); + }; + + // Test/TestSuite path validation + let absolute_or_relative = suite.canonicalize().or(test_262_path.join(&suite).canonicalize()); + let Ok(suite) = absolute_or_relative else { + return Err(Error262::InvalidSuitePath { path: suite }); + }; + if suite.is_absolute() && !suite.starts_with(&test_262_path) { + return Err(Error262::InvalidSuitePath { path: suite }); + } + + + let harness = Harness::read(&test_262_path).expect("Failed to read Harness"); // TOOD; + + if suite.to_string_lossy().ends_with(".js") { + let test = Test::read(&suite)?; + return Ok(ReadResult::Test(harness, test)); + } else if suite.is_dir() { + let test = TestSuite::read(&suite, &ignore, false)?; + return Ok(ReadResult::TestSuite(harness, test)); + } + + Err(Error262::InvalidSuitePath { path: suite }) +} + #[cfg(test)] mod tests { use crate::edition::SpecEdition; use crate::{Ignored, MetaData, TEST262_DIRECTORY}; - use std::path::Path; + use std::path::{Path, PathBuf}; #[test] #[ignore = "manual"] @@ -111,4 +211,60 @@ mod tests { SpecEdition::from_test_metadata(&metadata) ); } + + #[test] + #[ignore = "manual"] + fn should_read_test_with_path_relative_to_repository() { + let suite = PathBuf::from("test/intl402/constructors-string-and-single-element-array.js"); + let result = super::read(suite, Default::default()).expect("Test"); + match result { + crate::ReadResult::Test(_, test) => { + assert_eq!(test.name.as_ref(), "constructors-string-and-single-element-array"); + }, + _ => unreachable!("ReadResult::Test was expected"), + } + } + + #[test] + #[ignore = "manual"] + fn should_read_test_with_path_relative_to_current_directory() { + let suite = PathBuf::from("test262/test/intl402/constructors-string-and-single-element-array.js"); + let result = super::read(suite, Default::default()).expect("Test"); + match result { + crate::ReadResult::Test(_, test) => { + assert_eq!(test.name.as_ref(), "constructors-string-and-single-element-array"); + }, + _ => unreachable!("ReadResult::Test was expected"), + } + } + + #[test] + #[ignore = "manual"] + fn should_read_test_suite() { + let suite = PathBuf::from("test262/test/intl402/"); + let result = super::read(suite, Default::default()).expect("Test"); + match result { + crate::ReadResult::TestSuite(_, test) => { + assert_eq!(test.name.as_ref(), "intl402"); + }, + _ => unreachable!("ReadResult::TestSuite was expected"), + } + } + + #[test] + #[ignore = "manual"] + fn should_read_test_suite_custom_options() { + let suite = PathBuf::from("test262_2/test/intl402/"); + let result = super::read(suite, crate::ReadOptions { + test262_path: PathBuf::from("./test262_2"), + ..Default::default() + }).expect("Test"); + + match result { + crate::ReadResult::TestSuite(_, test) => { + assert_eq!(test.name.as_ref(), "intl402"); + }, + _ => unreachable!("ReadResult::TestSuite was expected"), + } + } } diff --git a/tools/test262/src/structs.rs b/tools/test262/src/structs.rs index 3a9634ec49c..4ebcfac4553 100644 --- a/tools/test262/src/structs.rs +++ b/tools/test262/src/structs.rs @@ -72,7 +72,7 @@ impl From> for Outcome { /// Structure to allow defining ignored tests, features and files that should /// be ignored even when reading. -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct Ignored { #[serde(default)] tests: FxHashSet>,