Enable multiline and import|load support for a bunch of stuff (#311)

* chill debugger and get it to match better with lando4, also make security MLC * multiline support for v3 build steps * add multiline !load support to events * use sapi as a more holistic measure of the service api * uh * fix write-file to be more targetted re: ImportString * fix write-file to be more targetted re: ImportString part 2 * fix write-file to be more targetted re: ImportString part 3 * fix other SAPI related issues re: multipass * improve !load resolution when base is unknown * fix incorrect SAPI usage * #244: Variable for setting the network limit. * #244: Add test for the networkLimit config var. * #297: fixed incorrect -EncodedCommand fallback detection for powershell.exe script execution * cl * release v3.23.22 generated by @lando/prepare-release-action * multipass access to healthchecks * multipass access to tooling * multipass access for config --------- Co-authored-by: Alec Reynolds <[email protected]> Co-authored-by: Alec Reynolds <[email protected]> Co-authored-by: rtfm-47 <[email protected]>
lando · Dec 19, 2024 · 76c2446 · 76c2446
1 parent ed7e488
commit 76c2446
Show file tree

Hide file tree

Showing 49 changed files with 586 additions and 124 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -30,6 +30,11 @@
 * Fixed bug causing auto setup to not correctly reset the orchestrator binary path
 * Improved `lando init` so that it can auto setup if needed
 
+## v3.23.22 - [December 17, 2024](https://github.com/lando/core/releases/tag/v3.23.22)
+
+* Added ability to customize `networkLimit` [#245](https://github.com/lando/core/pull/245)
+* Fixed incorrect `-EncodedCommand` fallback detection for `powershell.exe` script execution [#297](https://github.com/lando/core/issues/297)
+
 ## v3.23.21 - [December 14, 2024](https://github.com/lando/core/releases/tag/v3.23.21)
 
 * Fixed `powershell` scripts from failing when user cannot set `-ExecutionPolicy` to `Bypass` for `Process` scope [#297](https://github.com/lando/core/issues/297)

diff --git a/app.js b/app.js
@@ -34,6 +34,7 @@ module.exports = async (app, lando) => {
       primary: app._defaultService,
       project: app.project,
       root: app.root,
+      sapis: require('./utils/get-service-apis')(app),
     }, {persist: true});
   };
 
@@ -54,15 +55,16 @@ module.exports = async (app, lando) => {
       compose: app.compose,
       containers: app.containers,
       info: _.cloneDeep(app.info).map(service => ({...service, hostname: [], urls: []})),
-      executors: require('./utils/get-executors')(_.get(app, 'v4.services', {})),
       name: app.name,
       mounts: require('./utils/get-mounts')(_.get(app, 'v4.services', {})),
+      primary: app._defaultService,
+      project: app.project,
       root: app.root,
+      sapis: require('./utils/get-service-apis')(app),
       overrides: {
         tooling: app._coreToolingOverrides,
       },
-      primary: app._defaultService,
-      project: app.project,
+
     }, {persist: true});
   };
 

diff --git a/builders/_lando.js b/builders/_lando.js
@@ -3,8 +3,12 @@
 // Modules
 const _ = require('lodash');
 const fs = require('fs');
+const os = require('os');
 const path = require('path');
+const write = require('../utils/write-file');
+
 const {color} = require('listr2');
+const {nanoid} = require('nanoid');
 
 /*
  * The lowest level lando service, this is where a lot of the deep magic lives
@@ -128,13 +132,29 @@ module.exports = {
         volumes.push(`${local}:${remote}`);
       });
 
+      // rebase remoteFiles
+      remoteFiles = _.merge({}, {'_lando_': '/tmp/rooster'}, remoteFiles);
+
       // Handle custom config files
-      _.forEach(config, (file, type) => {
-        if (_.has(remoteFiles, type)) {
-          const local = path.resolve(root, config[type]);
-          const remote = remoteFiles[type];
-          volumes.push(`${local}:${remote}`);
+      _.forEach(config, (local, remote) => {
+        // if this is special type then get it from remoteFile
+        remote = _.has(remoteFiles, remote) ? remoteFiles[remote] : path.resolve('/', remote);
+
+        // if file is an imported string lets just get the file path instead
+        if (local?.constructor?.name === 'ImportString') {
+          const meta = local.getMetadata();
+          if (meta.file) local = meta.file;
+          else local = local.toString();
         }
+
+        // if file is still a multiline string then dump to tmp and use that
+        if (typeof local === 'string' && local.split('\n').length > 1) {
+          const contents = local;
+          local = path.join(os.tmpdir(), nanoid());
+          write(local, contents, {forcePosixLineEndings: true});
+        }
+
+        volumes.push(`${path.resolve(root, local)}:${remote}`);
       });
 
       // Add named volumes and other thingz into our primary service

diff --git a/builders/lando-v4.js b/builders/lando-v4.js
@@ -151,6 +151,7 @@ module.exports = {
       this.addLSF(path.join(__dirname, '..', 'scripts', 'boot.sh'));
       this.addLSF(path.join(__dirname, '..', 'scripts', 'entrypoint.sh'));
       this.addLSF(path.join(__dirname, '..', 'scripts', 'exec.sh'));
+      this.addLSF(path.join(__dirname, '..', 'scripts', 'exec-multiliner.sh'));
       this.addLSF(path.join(__dirname, '..', 'scripts', 'run-hooks.sh'));
       this.addLSF(path.join(__dirname, '..', 'scripts', 'landorc.sh'), 'landorc');
       this.addLSF(path.join(__dirname, '..', 'scripts', 'utils.sh'));
@@ -303,7 +304,7 @@ module.exports = {
       super(id, merge({}, {stages}, {groups}, {states}, upstream), app, lando);
 
       // props
-      this.canExec = true;
+      this.api = 4;
       this.canHealthcheck = true;
       this.isInteractive = lando.config.isInteractive;
       this.generateCert = lando.generateCert.bind(lando);
@@ -401,7 +402,6 @@ module.exports = {
       this.setNPMRC(lando.config.pluginConfigFile);
 
       // add in top level things
-      this.debug('adding top level volumes %o and networks %o', this.tlvolumes, {networks: this.tlnetworks});
       this.addComposeData({networks: this.tlnetworks, volumes: this.tlvolumes});
 
       // environment
@@ -505,6 +505,10 @@ module.exports = {
     }
 
     addLSF(source, dest, {context = 'context'} = {}) {
+      // normalize file input
+      source = this.normalizeFileInput(source, {dest});
+
+      // then do the rest
       if (dest === undefined) dest = path.basename(source);
       this.addContext(`${source}:/etc/lando/${dest}`, context);
       return `/etc/lando/${dest}`;
@@ -673,15 +677,10 @@ module.exports = {
     }
 
     mountScript(contents, {dest = `tmp/${nanoid()}.sh`} = {}) {
-      // @TODO: check if contents is a string?
-
-      // compute hostside file
-      const file = path.join(this.tmpdir, `${nanoid()}.sh`);
-
-      // dump contents to service tmpdir and make executable
-      write(file, contents, {forcePosixLineEndings: true});
+      // normalize to a file
+      const file = this.normalizeFileInput(contents);
+      // make executable
       fs.chmodSync(file, '755');
-
       // now complete the final mapping for container injection
       return this.addLSF(file, dest, 'user');
     }

diff --git a/components/docker-engine.js b/components/docker-engine.js
@@ -128,7 +128,6 @@ class DockerEngine extends Dockerode {
     for (const source of sources) {
       try {
         fs.copySync(source.source, path.join(context, source.target), {dereference: true});
-        debug('copied %o into build context %o', source.source, path.join(context, source.target));
       } catch (error) {
         error.message = `Failed to copy ${source.source} into build context at ${source.target}!: ${error.message}`;
         throw error;
@@ -139,7 +138,6 @@ class DockerEngine extends Dockerode {
     // @NOTE: we do this last to ensure we overwrite any dockerfile that may happenstance end up in the build-context
     // from source above
     fs.copySync(dockerfile, path.join(context, 'Dockerfile'));
-    debug('copied Imagefile from %o to %o', dockerfile, path.join(context, 'Dockerfile'));
 
     // on windows we want to ensure the build context has linux line endings
     if (process.platform === 'win32') {
@@ -229,7 +227,6 @@ class DockerEngine extends Dockerode {
     for (const source of sources) {
       try {
         fs.copySync(source.source, path.join(context, source.target), {dereference: true});
-        debug('copied %o into build context %o', source.source, path.join(context, source.target));
       } catch (error) {
         error.message = `Failed to copy ${source.source} into build context at ${source.target}!: ${error.message}`;
         throw error;
@@ -245,7 +242,6 @@ class DockerEngine extends Dockerode {
 
     // copy the dockerfile to the correct place and reset
     fs.copySync(dockerfile, path.join(context, 'Dockerfile'));
-    debug('copied Imagefile from %o to %o', dockerfile, path.join(context, 'Dockerfile'));
     dockerfile = path.join(context, 'Dockerfile');
 
     // build initial buildx command

diff --git a/components/l337-v4.js b/components/l337-v4.js
@@ -3,6 +3,7 @@
 const fs = require('fs');
 const groupBy = require('lodash/groupBy');
 const isObject = require('lodash/isPlainObject');
+const isStringy = require('../utils/is-stringy');
 const os = require('os');
 const merge = require('lodash/merge');
 const path = require('path');
@@ -90,7 +91,6 @@ class L337ServiceV4 extends EventEmitter {
     }
     this.emit('state', this.#data.info);
     this.#app.v4.updateComposeCache();
-    this.debug('updated app info to %o', this.#data.info);
   }
 
   get info() {
@@ -127,11 +127,12 @@ class L337ServiceV4 extends EventEmitter {
 
     // set top level required stuff
     this.id = id;
+    this.api = 'l337';
     this.appRoot = appRoot;
     this.buildkit = true;
     this.config = config;
     this.context = context;
-    this.debug = debug;
+    this.debug = debug.extend(id);
     this.name = name ?? id;
     this.primary = primary;
     this.project = project;
@@ -253,7 +254,6 @@ class L337ServiceV4 extends EventEmitter {
 
     // update and log
     this.#app.v4.updateComposeCache();
-    this.debug('%o added top level compose data %o', this.id, JSON.parse(JSON.stringify(data)));
   }
 
   // adds files/dirs to the build context
@@ -327,7 +327,6 @@ class L337ServiceV4 extends EventEmitter {
         if (group) this.addSteps({group, instructions: file.instructions.join('\n'), contexted: true});
 
         // return normalized data
-        this.debug('%o added %o to the build context', this.id, file);
         return file;
       }));
     }
@@ -446,8 +445,6 @@ class L337ServiceV4 extends EventEmitter {
         if (step.offset && Number(step.offset)) step.weight = step.weight + step.offset;
         // and finally lets rewrite the group for better instruction grouping
         step.group = `${step.group}-${step.weight}-${step.user}`;
-        // log
-        this.debug('%o added build step %o', this.id, step);
         // push
         this.#data.steps.push(step);
       });
@@ -576,8 +573,8 @@ class L337ServiceV4 extends EventEmitter {
     // remove build contexts and tmp
     remove(this.context);
     remove(this.tmpdir);
-    this.debug('removed %o build-context %o', `${this.project}-${this.id}`, this.context);
-    this.debug('removed %o tmpdir %o', `${this.project}-${this.id}`, this.id, this.tmpdir);
+    this.debug('removed build-context %o', this.context);
+    this.debug('removed tmpdir %o', this.tmpdir);
   }
 
   generateBuildContext() {
@@ -706,6 +703,29 @@ class L337ServiceV4 extends EventEmitter {
     return candidates.length > 0 && candidates[0] !== data ? candidates[0] : false;
   }
 
+  normalizeFileInput(data, {dest = undefined} = {}) {
+    // if data is not a stringy then do something else?
+    if (!isStringy(data)) {
+      this.debug('%o does not seem to be valid file input data', data);
+      return data;
+    }
+
+    // if data is a single line string then just return it
+    if (data.split('\n').length === 1) return path.resolve(this.appRoot, data);
+
+    // if dest is undefined and we have ImportString then lets use that filename
+    if (dest === undefined && data?.constructor?.name === 'ImportString') {
+      const {file} = data.getMetadata();
+      if (file) dest = path.basename(file);
+    }
+
+    // if we are here it is multiline and lets dump to a tmp file and return that
+    const file = path.join(this.tmpdir, dest ?? nanoid());
+    write(file, data, {forcePosixLineEndings: true});
+
+    return file;
+  }
+
   normalizeVolumes(volumes = []) {
     if (!Array.isArray) return [];
 
@@ -791,7 +811,7 @@ class L337ServiceV4 extends EventEmitter {
     }
 
     // log
-    this.debug('%o set base image to %o with instructions %o', this.id, this.#data.image, this.#data.imageInstructions);
+    this.debug('set base image to %o with instructions %o', this.#data.image, this.#data.imageInstructions);
   }
 }
 

diff --git a/components/yaml.js b/components/yaml.js
@@ -22,6 +22,13 @@ const parseFileTypeInput = input => {
   };
 };
 
+// helper to find file
+const findFile = (file, base = undefined) => {
+  return require('../utils/traverse-up')([file], path.resolve(base))
+    .map(candidate => path.join(path.dirname(candidate), file))
+    .find(candidate => fs.existsSync(candidate));
+};
+
 // file loader options
 const fileloader = {
   kind: 'scalar',
@@ -33,7 +40,7 @@ const fileloader = {
     const input = parseFileTypeInput(data);
 
     // if data is not an absolute path then resolve with base
-    if (!path.isAbsolute(input.file)) input.file = path.resolve(this.base, input.file);
+    if (!path.isAbsolute(input.file)) input.file = findFile(input.file, this.base);
 
     // Otherwise check the path exists
     return fs.existsSync(input.file);
@@ -42,7 +49,7 @@ const fileloader = {
     // transform data
     data = {raw: data, ...parseFileTypeInput(data)};
     // normalize if needed
-    data.file = !path.isAbsolute(data.file) ? path.resolve(this.base, data.file) : data.file;
+    data.file = !path.isAbsolute(data.file) ? findFile(data.file, this.base) : data.file;
 
     // switch based on type
     switch (data.type) {
@@ -100,6 +107,13 @@ class ImportString extends String {
   getDumper() {
     return this.#metadata.raw;
   }
+
+  [Symbol.toPrimitive](hint) {
+    if (hint === 'string') {
+      return this.toString();
+    }
+    return this.toString();
+  }
 }
 
 class ImportObject extends Object {

diff --git a/docs/config/networking.md b/docs/config/networking.md
@@ -58,3 +58,9 @@ You can also use the environment variable `LANDO_HOST_IP`.
 ```sh
 lando exec my-service -- ping "\$LANDO_HOST_IP" -c 3
 ```
+
+## Network Limits
+
+By default Docker has a limit of 32 networks. If you're running a large number of sites, you'll see a message `Lando has detected you are at Docker's network limit`, after which Lando will attempt to clean up unused networks to put you below the network limit.
+
+If you've [modified your Docker daemon](https://discussion.fedoraproject.org/t/increase-limit-of-30-docker-networks-in-a-clean-way/96622/4) to allow more networks, you can set Lando's network limit to a higher number by setting the `networkLimit` variable in [Lando's global config](./global.html).
diff --git a/examples/build/.lando.yml b/examples/build/.lando.yml
@@ -14,6 +14,8 @@ events:
     - cat /var/www/run_internal.txt | grep www-data
     - cat /run_as_root.txt | grep root
     - cat /var/www/run.txt | grep www-data
+    - cat /tmp/famous-blogger | grep famous-blogger
+    - cat /tmp/paperback-writer | grep paperback-writer
     - /bin/sh -c 'echo "$LANDO_APP_PROJECT" | grep landobuild'
     # uncomment below to test out https://github.com/lando/core/issues/70
     # this is commented out by default because of https://github.com/actions/runner/issues/241
@@ -40,6 +42,9 @@ services:
       # - bash /app/build_as_root.bash
     build:
       - echo "$(id)" > /var/www/build.txt
+      - !load paperback-writer.sh
+      - /app/paperback-writer.sh --writer famous-blogger
+
     run_as_root:
       - echo "$(id)" > /run_as_root.txt
     run:

diff --git a/examples/build/paperback-writer.sh b/examples/build/paperback-writer.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+set -eo pipefail
+
+WRITER=paperback-writer
+
+# PARSE THE ARGZZ
+while (( "$#" )); do
+  case "$1" in
+    --writer)
+      WRITER="$2"
+      shift 2
+    ;;
+    --writer=*)
+      WRITER="${1#*=}"
+      shift
+    ;;
+    --)
+      shift
+      break
+    ;;
+    -*|--*=)
+      shift
+    ;;
+    *)
+      shift
+    ;;
+  esac
+done
+
+echo "$WRITER" > "/tmp/$WRITER"
+
+cat "/tmp/$WRITER"