diff --git a/gulpfile.js b/gulpfile.js index 66b204c1..46f4705c 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -113,9 +113,9 @@ function inspect() { return { path: parsedPath, - level: level, - source: source, - program: program, + level, + source, + program, }; } catch (e) { if (parsedPath) { @@ -134,15 +134,15 @@ function benchmark() { const runs = data.level.examples.slice(0); - inboxGenerator.seed(123); + inboxGenerator.seed(Date.now()); - while (runs.length < 10) { + while (runs.length < 100) { const inbox = inboxGenerator.generate(data.level.number); const outbox = outboxGenerator.generate(data.level.number, inbox); runs.push({ - inbox: inbox, - outbox: outbox, + inbox, + outbox, }); } @@ -190,6 +190,14 @@ function benchmark() { throw 'Program always failing'; } + if (data.successRatio !== 1) { + if (runs[0].success && data.path.type !== 'specific') { + throw `Non-specific program failing on novel inputs (${Math.round( + 100 * data.successRatio, + )}% pass)`; + } + } + data.averageSteps = successfulRuns.length ? Math.round( successfulRuns.reduce((totalSteps, run) => { @@ -198,7 +206,7 @@ function benchmark() { ) : 0; } catch (e) { - console.error(chalk.red(data.path.full)); + console.error(chalk.red(file.data.path.full)); console.error(' ', chalk.red(e)); throw e; } diff --git a/solutions/04-Scrambler-Handler-7.21/14.19.unroll-ad-pro.asm b/solutions/04-Scrambler-Handler-7.21/14.19.unroll-ad-pro.asm.skip similarity index 100% rename from solutions/04-Scrambler-Handler-7.21/14.19.unroll-ad-pro.asm rename to solutions/04-Scrambler-Handler-7.21/14.19.unroll-ad-pro.asm.skip diff --git a/solutions/04-Scrambler-Handler-7.21/19.19.unroll-mrflip.asm b/solutions/04-Scrambler-Handler-7.21/19.19.unroll-mrflip.asm.skip similarity index 100% rename from solutions/04-Scrambler-Handler-7.21/19.19.unroll-mrflip.asm rename to solutions/04-Scrambler-Handler-7.21/19.19.unroll-mrflip.asm.skip diff --git a/solutions/04-Scrambler-Handler-7.21/20.18.unroll-viamodulo.asm b/solutions/04-Scrambler-Handler-7.21/20.18.unroll-viamodulo.asm.skip similarity index 100% rename from solutions/04-Scrambler-Handler-7.21/20.18.unroll-viamodulo.asm rename to solutions/04-Scrambler-Handler-7.21/20.18.unroll-viamodulo.asm.skip diff --git a/solutions/04-Scrambler-Handler-7.21/7.21-atesgoral.asm b/solutions/04-Scrambler-Handler-7.21/7.21-atesgoral.asm.skip similarity index 100% rename from solutions/04-Scrambler-Handler-7.21/7.21-atesgoral.asm rename to solutions/04-Scrambler-Handler-7.21/7.21-atesgoral.asm.skip diff --git a/solutions/28-Three-Sort-34.78/158.68.unroll-viamodulo.asm b/solutions/28-Three-Sort-34.78/158.68.unroll-viamodulo.asm.skip similarity index 100% rename from solutions/28-Three-Sort-34.78/158.68.unroll-viamodulo.asm rename to solutions/28-Three-Sort-34.78/158.68.unroll-viamodulo.asm.skip diff --git a/solutions/28-Three-Sort-34.78/28.115-eiTTio.asm b/solutions/28-Three-Sort-34.78/28.115-eiTTio.asm.skip similarity index 100% rename from solutions/28-Three-Sort-34.78/28.115-eiTTio.asm rename to solutions/28-Three-Sort-34.78/28.115-eiTTio.asm.skip diff --git a/solutions/28-Three-Sort-34.78/28.117.obsolete-esseger.asm b/solutions/28-Three-Sort-34.78/28.117.obsolete-esseger.asm.skip similarity index 100% rename from solutions/28-Three-Sort-34.78/28.117.obsolete-esseger.asm rename to solutions/28-Three-Sort-34.78/28.117.obsolete-esseger.asm.skip diff --git a/solutions/28-Three-Sort-34.78/29.117.obsolete-Eirik0.asm b/solutions/28-Three-Sort-34.78/29.117.obsolete-Eirik0.asm.skip similarity index 100% rename from solutions/28-Three-Sort-34.78/29.117.obsolete-Eirik0.asm rename to solutions/28-Three-Sort-34.78/29.117.obsolete-Eirik0.asm.skip diff --git a/solutions/28-Three-Sort-34.78/31.123.obsolete-Mygod.asm b/solutions/28-Three-Sort-34.78/31.123.obsolete-Mygod.asm.skip similarity index 100% rename from solutions/28-Three-Sort-34.78/31.123.obsolete-Mygod.asm rename to solutions/28-Three-Sort-34.78/31.123.obsolete-Mygod.asm.skip diff --git a/solutions/28-Three-Sort-34.78/32.128.obsolete-albertferras.asm b/solutions/28-Three-Sort-34.78/32.128.obsolete-albertferras.asm.skip similarity index 100% rename from solutions/28-Three-Sort-34.78/32.128.obsolete-albertferras.asm rename to solutions/28-Three-Sort-34.78/32.128.obsolete-albertferras.asm.skip diff --git a/solutions/28-Three-Sort-34.78/33.85-steambap.asm b/solutions/28-Three-Sort-34.78/33.85-steambap.asm.skip similarity index 100% rename from solutions/28-Three-Sort-34.78/33.85-steambap.asm rename to solutions/28-Three-Sort-34.78/33.85-steambap.asm.skip diff --git a/solutions/28-Three-Sort-34.78/34.139.obsolete-clarfonthey.asm b/solutions/28-Three-Sort-34.78/34.139.obsolete-clarfonthey.asm.skip similarity index 100% rename from solutions/28-Three-Sort-34.78/34.139.obsolete-clarfonthey.asm rename to solutions/28-Three-Sort-34.78/34.139.obsolete-clarfonthey.asm.skip diff --git a/solutions/28-Three-Sort-34.78/47.75-jdashton.asm b/solutions/28-Three-Sort-34.78/47.75-jdashton.asm.skip similarity index 100% rename from solutions/28-Three-Sort-34.78/47.75-jdashton.asm rename to solutions/28-Three-Sort-34.78/47.75-jdashton.asm.skip diff --git a/solutions/28-Three-Sort-34.78/49.77-eiTTio.asm b/solutions/28-Three-Sort-34.78/49.77-eiTTio.asm.skip similarity index 100% rename from solutions/28-Three-Sort-34.78/49.77-eiTTio.asm rename to solutions/28-Three-Sort-34.78/49.77-eiTTio.asm.skip diff --git a/solutions/28-Three-Sort-34.78/62.72-popq.asm b/solutions/28-Three-Sort-34.78/62.72-popq.asm.skip similarity index 100% rename from solutions/28-Three-Sort-34.78/62.72-popq.asm rename to solutions/28-Three-Sort-34.78/62.72-popq.asm.skip diff --git a/solutions/28-Three-Sort-34.78/62.73.obsolete-AlanDeSmet.asm b/solutions/28-Three-Sort-34.78/62.73.obsolete-AlanDeSmet.asm.skip similarity index 100% rename from solutions/28-Three-Sort-34.78/62.73.obsolete-AlanDeSmet.asm rename to solutions/28-Three-Sort-34.78/62.73.obsolete-AlanDeSmet.asm.skip diff --git a/solutions/28-Three-Sort-34.78/62.75.obsolete-clarfonthey.asm b/solutions/28-Three-Sort-34.78/62.75.obsolete-clarfonthey.asm.skip similarity index 100% rename from solutions/28-Three-Sort-34.78/62.75.obsolete-clarfonthey.asm rename to solutions/28-Three-Sort-34.78/62.75.obsolete-clarfonthey.asm.skip diff --git a/solutions/31-String-Reverse-11.122/14.103-FireGoblin.asm b/solutions/31-String-Reverse-11.122/14.103-FireGoblin.asm.skip similarity index 100% rename from solutions/31-String-Reverse-11.122/14.103-FireGoblin.asm rename to solutions/31-String-Reverse-11.122/14.103-FireGoblin.asm.skip diff --git a/solutions/35-Duplicate-Removal-17.167/27.96.hashtable-ironwallaby.asm b/solutions/35-Duplicate-Removal-17.167/27.96.hashtable-ironwallaby.asm.skip similarity index 100% rename from solutions/35-Duplicate-Removal-17.167/27.96.hashtable-ironwallaby.asm rename to solutions/35-Duplicate-Removal-17.167/27.96.hashtable-ironwallaby.asm.skip diff --git a/solutions/35-Duplicate-Removal-17.167/42.63-ocoss.asm b/solutions/35-Duplicate-Removal-17.167/42.63-ocoss.asm.skip similarity index 100% rename from solutions/35-Duplicate-Removal-17.167/42.63-ocoss.asm rename to solutions/35-Duplicate-Removal-17.167/42.63-ocoss.asm.skip diff --git a/solutions/35-Duplicate-Removal-17.167/48.70-WolfWings.asm b/solutions/35-Duplicate-Removal-17.167/48.70-WolfWings.asm.skip similarity index 100% rename from solutions/35-Duplicate-Removal-17.167/48.70-WolfWings.asm rename to solutions/35-Duplicate-Removal-17.167/48.70-WolfWings.asm.skip diff --git a/solutions/36-Alphabetizer-39.109/25.88-halchihal.asm b/solutions/36-Alphabetizer-39.109/25.88-halchihal.asm.skip similarity index 100% rename from solutions/36-Alphabetizer-39.109/25.88-halchihal.asm rename to solutions/36-Alphabetizer-39.109/25.88-halchihal.asm.skip diff --git a/solutions/39-Re-Coordinator-14.76/142.40.obsolete-viamodulo.asm b/solutions/39-Re-Coordinator-14.76/142.40.obsolete-viamodulo.asm.skip similarity index 100% rename from solutions/39-Re-Coordinator-14.76/142.40.obsolete-viamodulo.asm rename to solutions/39-Re-Coordinator-14.76/142.40.obsolete-viamodulo.asm.skip diff --git a/solutions/40-Prime-Factory-28.399/28.247-jdashton.asm b/solutions/40-Prime-Factory-28.399/28.247-jdashton.asm.skip similarity index 100% rename from solutions/40-Prime-Factory-28.399/28.247-jdashton.asm rename to solutions/40-Prime-Factory-28.399/28.247-jdashton.asm.skip diff --git a/solutions/41-Sorting-Floor-34.714/188.447.insertion-mrflip.asm b/solutions/41-Sorting-Floor-34.714/188.447.insertion-mrflip.asm.skip similarity index 100% rename from solutions/41-Sorting-Floor-34.714/188.447.insertion-mrflip.asm rename to solutions/41-Sorting-Floor-34.714/188.447.insertion-mrflip.asm.skip diff --git a/solutions/41-Sorting-Floor-34.714/19.734-marcinsmu.asm b/solutions/41-Sorting-Floor-34.714/19.734-marcinsmu.asm.skip similarity index 100% rename from solutions/41-Sorting-Floor-34.714/19.734-marcinsmu.asm rename to solutions/41-Sorting-Floor-34.714/19.734-marcinsmu.asm.skip diff --git a/solutions/41-Sorting-Floor-34.714/196.446.insertion-viamodulo.asm b/solutions/41-Sorting-Floor-34.714/196.446.insertion-viamodulo.asm.skip similarity index 100% rename from solutions/41-Sorting-Floor-34.714/196.446.insertion-viamodulo.asm rename to solutions/41-Sorting-Floor-34.714/196.446.insertion-viamodulo.asm.skip diff --git a/solutions/41-Sorting-Floor-34.714/20.648.selection-mrflip.asm b/solutions/41-Sorting-Floor-34.714/20.648.selection-mrflip.asm.skip similarity index 100% rename from solutions/41-Sorting-Floor-34.714/20.648.selection-mrflip.asm rename to solutions/41-Sorting-Floor-34.714/20.648.selection-mrflip.asm.skip diff --git a/solutions/41-Sorting-Floor-34.714/20.651.selection-sniperrifle2004.asm b/solutions/41-Sorting-Floor-34.714/20.651.selection-sniperrifle2004.asm.skip similarity index 100% rename from solutions/41-Sorting-Floor-34.714/20.651.selection-sniperrifle2004.asm rename to solutions/41-Sorting-Floor-34.714/20.651.selection-sniperrifle2004.asm.skip diff --git a/solutions/41-Sorting-Floor-34.714/20.689-polarathene.asm b/solutions/41-Sorting-Floor-34.714/20.689-polarathene.asm.skip similarity index 100% rename from solutions/41-Sorting-Floor-34.714/20.689-polarathene.asm rename to solutions/41-Sorting-Floor-34.714/20.689-polarathene.asm.skip diff --git a/solutions/41-Sorting-Floor-34.714/208.446.exploit-mrflip.asm b/solutions/41-Sorting-Floor-34.714/208.446.exploit-mrflip.asm.skip similarity index 100% rename from solutions/41-Sorting-Floor-34.714/208.446.exploit-mrflip.asm rename to solutions/41-Sorting-Floor-34.714/208.446.exploit-mrflip.asm.skip diff --git a/solutions/41-Sorting-Floor-34.714/27.639.insertion-sniperrifle2004.asm b/solutions/41-Sorting-Floor-34.714/27.639.insertion-sniperrifle2004.asm.skip similarity index 100% rename from solutions/41-Sorting-Floor-34.714/27.639.insertion-sniperrifle2004.asm rename to solutions/41-Sorting-Floor-34.714/27.639.insertion-sniperrifle2004.asm.skip diff --git a/solutions/41-Sorting-Floor-34.714/28.537.insertion-mrflip.asm b/solutions/41-Sorting-Floor-34.714/28.537.insertion-mrflip.asm.skip similarity index 100% rename from solutions/41-Sorting-Floor-34.714/28.537.insertion-mrflip.asm rename to solutions/41-Sorting-Floor-34.714/28.537.insertion-mrflip.asm.skip diff --git a/solutions/41-Sorting-Floor-34.714/28.705.selection-clarfonthey.asm b/solutions/41-Sorting-Floor-34.714/28.705.selection-clarfonthey.asm.skip similarity index 100% rename from solutions/41-Sorting-Floor-34.714/28.705.selection-clarfonthey.asm rename to solutions/41-Sorting-Floor-34.714/28.705.selection-clarfonthey.asm.skip diff --git a/solutions/41-Sorting-Floor-34.714/30.1061.gnomesort-eiTTio.asm b/solutions/41-Sorting-Floor-34.714/30.1061.gnomesort-eiTTio.asm.skip similarity index 100% rename from solutions/41-Sorting-Floor-34.714/30.1061.gnomesort-eiTTio.asm rename to solutions/41-Sorting-Floor-34.714/30.1061.gnomesort-eiTTio.asm.skip diff --git a/solutions/41-Sorting-Floor-34.714/30.536.insertion-viamodulo.asm b/solutions/41-Sorting-Floor-34.714/30.536.insertion-viamodulo.asm.skip similarity index 100% rename from solutions/41-Sorting-Floor-34.714/30.536.insertion-viamodulo.asm rename to solutions/41-Sorting-Floor-34.714/30.536.insertion-viamodulo.asm.skip diff --git a/solutions/41-Sorting-Floor-34.714/31.546-eiTTio.asm b/solutions/41-Sorting-Floor-34.714/31.546-eiTTio.asm.skip similarity index 100% rename from solutions/41-Sorting-Floor-34.714/31.546-eiTTio.asm rename to solutions/41-Sorting-Floor-34.714/31.546-eiTTio.asm.skip diff --git a/solutions/41-Sorting-Floor-34.714/31.938.insertion-clarfonthey.asm b/solutions/41-Sorting-Floor-34.714/31.938.insertion-clarfonthey.asm.skip similarity index 100% rename from solutions/41-Sorting-Floor-34.714/31.938.insertion-clarfonthey.asm rename to solutions/41-Sorting-Floor-34.714/31.938.insertion-clarfonthey.asm.skip diff --git a/solutions/41-Sorting-Floor-34.714/32.1404.bubblesort-hastebrot.asm b/solutions/41-Sorting-Floor-34.714/32.1404.bubblesort-hastebrot.asm.skip similarity index 100% rename from solutions/41-Sorting-Floor-34.714/32.1404.bubblesort-hastebrot.asm rename to solutions/41-Sorting-Floor-34.714/32.1404.bubblesort-hastebrot.asm.skip diff --git a/solutions/41-Sorting-Floor-34.714/33.570-polarathene.asm b/solutions/41-Sorting-Floor-34.714/33.570-polarathene.asm.skip similarity index 100% rename from solutions/41-Sorting-Floor-34.714/33.570-polarathene.asm rename to solutions/41-Sorting-Floor-34.714/33.570-polarathene.asm.skip diff --git a/solutions/41-Sorting-Floor-34.714/43.621.insertion-dubaaron.asm b/solutions/41-Sorting-Floor-34.714/43.621.insertion-dubaaron.asm.skip similarity index 100% rename from solutions/41-Sorting-Floor-34.714/43.621.insertion-dubaaron.asm rename to solutions/41-Sorting-Floor-34.714/43.621.insertion-dubaaron.asm.skip diff --git a/solutions/41-Sorting-Floor-34.714/69.1534.mergesort-IAmWave.asm b/solutions/41-Sorting-Floor-34.714/69.1534.mergesort-IAmWave.asm.skip similarity index 100% rename from solutions/41-Sorting-Floor-34.714/69.1534.mergesort-IAmWave.asm rename to solutions/41-Sorting-Floor-34.714/69.1534.mergesort-IAmWave.asm.skip diff --git a/solutions/41-Sorting-Floor-34.714/79.488.insertion-mrflip.asm b/solutions/41-Sorting-Floor-34.714/79.488.insertion-mrflip.asm.skip similarity index 100% rename from solutions/41-Sorting-Floor-34.714/79.488.insertion-mrflip.asm rename to solutions/41-Sorting-Floor-34.714/79.488.insertion-mrflip.asm.skip