From 340f62edbf52e3744a7f9ce4e0645e91c61c542d Mon Sep 17 00:00:00 2001 From: Greg Pfeil Date: Fri, 15 Apr 2022 12:39:57 -1000 Subject: [PATCH] Add a DynFlags plugin. This only has an effect on GHC 8.10 and later. Older versions need to manually set the relevant flags. TODO: warn when we had to override flags set by the user (ideally, we'd only warn when they explicitly differ, not when we're overriding behavior set by `-O2` or something. This also currently doesn't work, because some of our tests break when we don't ignore interface pragmas (I was hoping this would go away when we removed the presimplifier, but it didn't). Fixes #43. --- ghc/Categorifier/GHC/Driver.hs | 33 +++++++++++++++++++ ghc/categorifier-ghc.cabal | 1 - ...orifier-adjunctions-integration-test.cabal | 1 - ...gorifier-categories-integration-test.cabal | 1 - ...r-concat-extensions-integration-test.cabal | 1 - ...categorifier-concat-integration-test.cabal | 2 -- ...gorifier-ghc-bignum-integration-test.cabal | 1 - ...orifier-linear-base-integration-test.cabal | 1 - ...tegorifier-unconcat-integration-test.cabal | 1 - .../categorifier-vec-integration-test.cabal | 1 - plugin-test/categorifier-plugin-test.cabal | 1 - plugin/Categorifier.hs | 18 +++++----- plugin/Categorifier/Core/ErrorHandling.hs | 8 ++--- plugin/Categorifier/DynFlags.hs | 20 +++++++++++ plugin/README.md | 32 ++++-------------- plugin/categorifier-plugin.cabal | 2 +- th/categorifier-th.cabal | 1 - 17 files changed, 73 insertions(+), 52 deletions(-) create mode 100644 plugin/Categorifier/DynFlags.hs diff --git a/ghc/Categorifier/GHC/Driver.hs b/ghc/Categorifier/GHC/Driver.hs index 1f44e0ea..b70e4ebf 100644 --- a/ghc/Categorifier/GHC/Driver.hs +++ b/ghc/Categorifier/GHC/Driver.hs @@ -1,10 +1,14 @@ {-# LANGUAGE CPP #-} +{-# LANGUAGE PartialTypeSignatures #-} +{-# OPTIONS_GHC -Wno-partial-type-signatures #-} module Categorifier.GHC.Driver ( module DynFlags, module HscTypes, module Outputable, module Plugins, + defaultPurePlugin, + pureDynflagsAndCorePlugin, ) where @@ -28,3 +32,32 @@ import HscTypes hiding (InteractiveContext (..), InteractiveImport (..), ModGuts import Outputable hiding (Outputable (..), renderWithStyle) import Plugins #endif + +-- | Like `defaultPlugin`, but specifies that the plugin is pure (when GHC allows). +defaultPurePlugin :: Plugin +#if MIN_VERSION_ghc(8, 6, 0) +defaultPurePlugin = defaultPlugin {pluginRecompile = purePlugin} +#else +defaultPurePlugin = defaultPlugin +#endif + +-- | Builds a pure plugin that has both `DynFlags` and `Core` components. Prior to GHC 8.10, the +-- `DynFlags` component will be ignored and so the flags must be manually configured to match. +pureDynflagsAndCorePlugin :: + ([CommandLineOption] -> DynFlags -> IO DynFlags) -> + -- | @([CommandLineOption] -> [CoreToDo] -> CoreM [CoreToDo])@, but skipped to avoid import issues + _ -> + Plugin +#if MIN_VERSION_ghc(9, 2, 0) +pureDynflagsAndCorePlugin dynflags core = + defaultPurePlugin + { driverPlugin = + \opts hsc -> (\newFlags -> hsc {hsc_dflags = newFlags}) <$> dynflags opts (hsc_dflags hsc), + installCoreToDos = core + } +#elif MIN_VERSION_ghc(8, 10, 0) +pureDynflagsAndCorePlugin dynflags core = + defaultPurePlugin {dynflagsPlugin = dynflags, installCoreToDos = core} +#else +pureDynflagsAndCorePlugin _ core = defaultPurePlugin {installCoreToDos = core} +#endif diff --git a/ghc/categorifier-ghc.cabal b/ghc/categorifier-ghc.cabal index 31f1fc9c..695d8557 100644 --- a/ghc/categorifier-ghc.cabal +++ b/ghc/categorifier-ghc.cabal @@ -55,7 +55,6 @@ library Paths_categorifier_ghc ghc-options: -O2 - -fignore-interface-pragmas build-depends: , PyF ^>=0.9.0 || ^>=0.10.0 || ^>=0.11.0 , bytestring ^>=0.10.9 || ^>=0.11.0 diff --git a/integrations/adjunctions/integration-test/categorifier-adjunctions-integration-test.cabal b/integrations/adjunctions/integration-test/categorifier-adjunctions-integration-test.cabal index a6cf1730..17869d42 100644 --- a/integrations/adjunctions/integration-test/categorifier-adjunctions-integration-test.cabal +++ b/integrations/adjunctions/integration-test/categorifier-adjunctions-integration-test.cabal @@ -86,4 +86,3 @@ test-suite adjunctions-hierarchy-optimized main-is: Adjunctions/Main.hs ghc-options: -O2 - -fignore-interface-pragmas diff --git a/integrations/categories/integration-test/categorifier-categories-integration-test.cabal b/integrations/categories/integration-test/categorifier-categories-integration-test.cabal index 4e03ef9f..8e9bbdf1 100644 --- a/integrations/categories/integration-test/categorifier-categories-integration-test.cabal +++ b/integrations/categories/integration-test/categorifier-categories-integration-test.cabal @@ -78,4 +78,3 @@ test-suite categories-hierarchy-optimized main-is: Categories/Main.hs ghc-options: -O2 - -fignore-interface-pragmas diff --git a/integrations/concat-extensions/integration-test/categorifier-concat-extensions-integration-test.cabal b/integrations/concat-extensions/integration-test/categorifier-concat-extensions-integration-test.cabal index f78d9282..a5158762 100644 --- a/integrations/concat-extensions/integration-test/categorifier-concat-extensions-integration-test.cabal +++ b/integrations/concat-extensions/integration-test/categorifier-concat-extensions-integration-test.cabal @@ -86,4 +86,3 @@ test-suite concat-extensions-hierarchy-optimized main-is: ConCatExtensions/Main.hs ghc-options: -O2 - -fignore-interface-pragmas diff --git a/integrations/concat/integration-test/categorifier-concat-integration-test.cabal b/integrations/concat/integration-test/categorifier-concat-integration-test.cabal index 0ad027db..47258c86 100644 --- a/integrations/concat/integration-test/categorifier-concat-integration-test.cabal +++ b/integrations/concat/integration-test/categorifier-concat-integration-test.cabal @@ -89,7 +89,6 @@ test-suite concat-class-hierarchy-optimized main-is: ConCat/Main.hs ghc-options: -O2 - -fignore-interface-pragmas test-suite concat-function-hierarchy import: hierarchy-tests @@ -104,4 +103,3 @@ test-suite concat-function-hierarchy-optimized main-is: ConCat/Main.hs ghc-options: -O2 - -fignore-interface-pragmas diff --git a/integrations/ghc-bignum/integration-test/categorifier-ghc-bignum-integration-test.cabal b/integrations/ghc-bignum/integration-test/categorifier-ghc-bignum-integration-test.cabal index b54ccd98..a0adac5c 100644 --- a/integrations/ghc-bignum/integration-test/categorifier-ghc-bignum-integration-test.cabal +++ b/integrations/ghc-bignum/integration-test/categorifier-ghc-bignum-integration-test.cabal @@ -89,4 +89,3 @@ test-suite ghc-bignum-hierarchy-optimized main-is: GhcBignum/Main.hs ghc-options: -O2 - -fignore-interface-pragmas diff --git a/integrations/linear-base/integration-test/categorifier-linear-base-integration-test.cabal b/integrations/linear-base/integration-test/categorifier-linear-base-integration-test.cabal index e4abf99a..0e984e88 100644 --- a/integrations/linear-base/integration-test/categorifier-linear-base-integration-test.cabal +++ b/integrations/linear-base/integration-test/categorifier-linear-base-integration-test.cabal @@ -91,4 +91,3 @@ test-suite linear-base-hierarchy-optimized main-is: LinearBase/Main.hs ghc-options: -O2 - -fignore-interface-pragmas diff --git a/integrations/unconcat/integration-test/categorifier-unconcat-integration-test.cabal b/integrations/unconcat/integration-test/categorifier-unconcat-integration-test.cabal index 5e5d6330..d81d9bd0 100644 --- a/integrations/unconcat/integration-test/categorifier-unconcat-integration-test.cabal +++ b/integrations/unconcat/integration-test/categorifier-unconcat-integration-test.cabal @@ -84,4 +84,3 @@ test-suite unconcat-hierarchy-optimized main-is: UnconCat/Main.hs ghc-options: -O2 - -fignore-interface-pragmas diff --git a/integrations/vec/integration-test/categorifier-vec-integration-test.cabal b/integrations/vec/integration-test/categorifier-vec-integration-test.cabal index ef5bafda..8ea22f77 100644 --- a/integrations/vec/integration-test/categorifier-vec-integration-test.cabal +++ b/integrations/vec/integration-test/categorifier-vec-integration-test.cabal @@ -93,4 +93,3 @@ test-suite vec-hierarchy-optimized main-is: Vec/Main.hs ghc-options: -O2 - -fignore-interface-pragmas diff --git a/plugin-test/categorifier-plugin-test.cabal b/plugin-test/categorifier-plugin-test.cabal index a44984c2..90005de0 100644 --- a/plugin-test/categorifier-plugin-test.cabal +++ b/plugin-test/categorifier-plugin-test.cabal @@ -110,4 +110,3 @@ test-suite base-hierarchy-optimized main-is: Base/Main.hs ghc-options: -O2 - -fignore-interface-pragmas diff --git a/plugin/Categorifier.hs b/plugin/Categorifier.hs index 101d9414..c6503d9c 100644 --- a/plugin/Categorifier.hs +++ b/plugin/Categorifier.hs @@ -14,6 +14,7 @@ where import Categorifier.CommandLineOptions (OptionGroup, partitionOptions) import Categorifier.Common.IO.Exception (throwIOAsException) import qualified Categorifier.Core +import qualified Categorifier.DynFlags import qualified Categorifier.GHC.Core as GhcPlugins import qualified Categorifier.GHC.Driver as GhcPlugins import Control.Applicative (liftA2) @@ -30,15 +31,14 @@ import PyF (fmt) -- for more information. plugin :: GhcPlugins.Plugin plugin = - GhcPlugins.defaultPlugin - { GhcPlugins.installCoreToDos = - \opts -> - join - . GhcPlugins.liftIO - . liftA2 Categorifier.Core.install (partitionOptions' opts) - . pure, - GhcPlugins.pluginRecompile = GhcPlugins.flagRecompile - } + GhcPlugins.pureDynflagsAndCorePlugin + (\_opts -> pure . Categorifier.DynFlags.plugin) + ( \opts -> + join + . GhcPlugins.liftIO + . liftA2 Categorifier.Core.install (partitionOptions' opts) + . pure + ) partitionOptions' :: [GhcPlugins.CommandLineOption] -> IO (Map OptionGroup [Text]) partitionOptions' opts = diff --git a/plugin/Categorifier/Core/ErrorHandling.hs b/plugin/Categorifier/Core/ErrorHandling.hs index b31a2e55..b8f20408 100644 --- a/plugin/Categorifier/Core/ErrorHandling.hs +++ b/plugin/Categorifier/Core/ErrorHandling.hs @@ -232,11 +232,9 @@ required by {showE expr}.|] `inlinable` pragma to the definition or compiling with `-fexpose-all-unfoldings` to make _every_ operation inlinable. It's also important that the module containing the call to `categorify` is compiled - with `-fno-ignore-interface-pragmas` (also implied by `-O`). If the - unfolding that's missing is for `$j` (GHC-internal join points), you may - need to bump `-funfolding-creation-threshold` on the modules you're - depending on. If there is still no unfolding available, please file an issue - against the plugin.|] + with `-fno-ignore-interface-pragmas` (also implied by `-O` and enabled + automatically on GHC 8.10.1 or later). If there is still no unfolding + available, please file an issue against the plugin.|] Plugins.BootUnfolding -> [fmt| The identifier is defined in an hi-boot file, so can't be inlined.|] diff --git a/plugin/Categorifier/DynFlags.hs b/plugin/Categorifier/DynFlags.hs new file mode 100644 index 00000000..aebd2ba7 --- /dev/null +++ b/plugin/Categorifier/DynFlags.hs @@ -0,0 +1,20 @@ +module Categorifier.DynFlags + ( plugin, + ) +where + +import qualified Categorifier.GHC.Driver as GHC + +plugin :: GHC.DynFlags -> GHC.DynFlags +plugin = setUpDynFlags + +-- | This sets up flags that allow the plugin to do what it needs to. +-- +-- For a compiler that doesn't support this plugin (before GHC 8.10), the following GHC options +-- approximate it: +-- +-- [@-fno-ignore-interface-pragmas@]: Ensures we can inline definitions that we don't know how to +-- interpret directly to the target category. This flag is also +-- implied by @-O@ and @-O2@. +setUpDynFlags :: GHC.DynFlags -> GHC.DynFlags +setUpDynFlags = GHC.unSetGeneralFlag' GHC.Opt_IgnoreInterfacePragmas diff --git a/plugin/README.md b/plugin/README.md index 753985e6..0eed845d 100644 --- a/plugin/README.md +++ b/plugin/README.md @@ -11,8 +11,9 @@ directly, as well as ones that are depended on (transitively) by the ones that u ### targets you want to use `categorify` in -- enable the plugin with `-fplugin=Categorifier`, -- ensure inlining is available with `-fno-ignore-interface-pragmas` (implied by `-O` or `-O2`), and +- enable the plugin with `-fplugin=Categorifier`; +- if you are using a version of GHC older than 8.10.1, use the flags described in + [Categorifier.DynFlags](./Categorifier/DynFlags.hs); and - import `Categorifier.Categorify` to make `categorify` available (you must import the _entire_ module, but it may be qualified). @@ -32,26 +33,6 @@ directly, as well as ones that are depended on (transitively) by the ones that u - define `Categorifier.HasRep` instances for any types that you use in a converted function (the plugin will tell you if you are missing any when you try to convert) -### fine-tuning inlining sizes - -It can be difficult to find a reasonable setting for the various inlining thresholds. This attempts -to lay out an approach for identifying one. - -There are two significant GHC flags for adjusting inlining, `-funfolding-creation-threshold` and -`-funfolding-use-threshold`. They allow you to set an upper bound on the "size" of unfoldings that -will be considered for inlining. - -1. set the `creation` (globally) threshold high, say `10000`; -2. test to see if the inlining issue goes away (if so, skip to step 5); -3. set the `use` (in `categorify` modules) threshold to match the `creation` threshold; -4. do a binary search on the `use` thresholds to minimize them as much as possible; -5. do a binary search on the `creation` thresholds to minimize them as much as possible (the lower - bound here is probably the minimum of 750 (the default) and the `use` threshold). - -If either if these values is too small, you'll end up with errors complaining that some definition -couldn't be inlined. If they're too big, you'll get errors about "simplifier ticks exhausted" (in -which case, you can bump `-fsimpl-tick-factor`) and things will take a lot longer to compile. - ### defining `HasRep` instances You should use `Categorifier.Client.deriveHasRep` for all `HasRep` instances. However, for @@ -146,9 +127,10 @@ This is ostensibly a more correct approach, given the way GHC is structured, but There are a bunch of modules, this calls out the most important ones when diving in. -- [`Categorifier`](./Categorifier.hs) - this is the entrypoint of the plugin, everything that hooks into - GHC starts from here; -- [`Categorifier.Core.Categorify`](./Categorifier/Core/Categorify.hs) - the high-level logic of the +- [Categorifier](./Categorifier.hs) - this is the entrypoint of the plugin, everything that hooks into + GHC starts from here, with there being three immediate subcomponents: command-line option + handling, `DynFlags` settings, and the Core pass; +- [Categorifier.Core.Categorify](./Categorifier/Core/Categorify.hs) - the high-level logic of the categorical transformation as described in Conal's paper, it tries to define as clearly as possible the mapping from **Hask** to abstract categories; - [`Categorifier.Hierarchy`](./Categorifier/Hierarchy.hs) - the mappings from abstract diff --git a/plugin/categorifier-plugin.cabal b/plugin/categorifier-plugin.cabal index 2666555e..610f3f36 100644 --- a/plugin/categorifier-plugin.cabal +++ b/plugin/categorifier-plugin.cabal @@ -53,6 +53,7 @@ library -- __TODO__: move ...PrimOp to other-modules Categorifier.Core.PrimOp Categorifier.Core.Types + Categorifier.DynFlags Categorifier.Hierarchy other-modules: Categorifier.Benchmark @@ -67,7 +68,6 @@ library Paths_categorifier_plugin ghc-options: -O2 - -fignore-interface-pragmas build-depends: , PyF ^>=0.9.0 || ^>=0.10.0 || ^>=0.11.0 , barbies ^>=2.0.1 diff --git a/th/categorifier-th.cabal b/th/categorifier-th.cabal index 409ecc13..13f238e7 100644 --- a/th/categorifier-th.cabal +++ b/th/categorifier-th.cabal @@ -45,7 +45,6 @@ library Paths_categorifier_th ghc-options: -O2 - -fignore-interface-pragmas build-depends: , PyF ^>=0.9.0 || ^>=0.10.0 || ^>=0.11.0 , categorifier-common