diff --git a/builders/lando-v4.js b/builders/lando-v4.js index 187a11f00..132d6e553 100644 --- a/builders/lando-v4.js +++ b/builders/lando-v4.js @@ -1,19 +1,37 @@ 'use strict'; -const isObject = require('lodash/isPlainObject'); +const fs = require('fs'); const merge = require('lodash/merge'); +const path = require('path'); +const uniq = require('lodash/uniq'); -const isDisabled = require('../utils/is-disabled'); - -const normalizeAppMount = mount => { - // @TODO: are there any errors we should throw here? - // if mount is a string the parse into an object - if (typeof mount === 'string') mount = {target: mount.split(':')[0], type: mount.split(':')[1] || 'bind'}; - // if we have dest|destination and not target then map them over - if (!mount.target && mount.dest) mount.target = mount.dest; - if (!mount.target && mount.destination) mount.target = mount.destination; - // return - return mount; +const states = {APP: 'UNBUILT'}; +const groups = { + 'boot': { + description: 'Required packages that every subsequent group needs', + weight: 100, + user: 'root', + }, + 'system': { + description: 'System level packages', + weight: 200, + user: 'root', + }, + 'setup-user': { + description: 'Host/container user mapping considerations', + weight: 300, + user: 'root', + }, + 'tooling': { + description: 'Installation of tooling', + weight: 400, + user: 'root', + }, + 'config': { + description: 'Configuration file stuff', + weight: 500, + user: 'root', + }, }; /* @@ -22,120 +40,264 @@ const normalizeAppMount = mount => { module.exports = { api: 4, name: 'lando', + parent: 'l337', defaults: { - image: { - context: [], + config: { + 'app-mount': { + type: 'bind', + destination: '/app', + exclude: [], + }, }, - volumes: [], }, - parent: 'l337', + router: () => ({}), builder: (parent, defaults) => class LandoServiceV4 extends parent { - #groups - #stages - - static debug = require('debug')('lando-service-v4'); - - constructor(id, options) { - // merge configs ontop of defaults - const config = merge({}, defaults, options.config); - - // normalize image data - // @TODO: normalize image func? - const image = isObject(config.image) ? config.image : {imagefile: config.image}; - // @TODO: throw if imagefile/etc is not set - - // normalize app-mount - if (!config.appMount) config.appMount = config['app-mount']; - // set to false if its disabled otherwise normalize - const appMount = isDisabled(config.appMount) ? false : normalizeAppMount(config.appMount); - // @TODO: throw if appmount.target is a string and not an absolute path - // put together the stuff we can pass directly into super and leverage l337 - const l337 = {appMount, image}; - // if a command is set then lets pass that through as well - if (config.command) l337.command = config.command; - - // appmount must be an absolute path - - // if we have a string appmount then standardize it into object format - // 3. app-mount - // 4. ports? - // 5. groups/stages - - // l337me - super(id, Object.assign(options, {config: l337})); - - - // what stages do we - // how we run steps - // image - // app-build - // background-exec? - - // what build groups do we need? - // how we order steps - - // what is the purpose of boot? - // ensure some minimal set of dependencies and map the user - // boot.context - // boot.install - // boot.user - - // system.context - // system.install - - // user.context - // user.image - // user.app-build - // user.background-exec - - // add build groups - - // @TODO: - // 2. boot? - // 3. overrides - - // try to mock up some shit - // nginx/apache - // mariadb - // php - - // drupal - // webserver: user, config, appmount, certs, ports, build.exec - // mariadb: user, config, ports, storage, healthcheck - // php: user, config, appmount, ports, build.image, build.app - - // othershit - // parent stuff - - // ssh-keys? - - // console.log('hellp there') - // console.log(this); - // process.exit(1) - - - // Envvars & Labels - // const environment = { - // LANDO_SERVICE_API: 4, - // LANDO_SERVICE_NAME: name, - // LANDO_SERVICE_TYPE: type, - // }; - // // should be state data that lando uses? - // // user/team that "owns" the image - // const labels = { - // 'dev.lando.api': 4, - // }; - // // Set a reasonable log size - // const logging = {driver: 'json-file', options: {'max-file': '3', 'max-size': '10m'}}; - // // basic docker compose file - // this.addComposeData({ - // services: _.set({}, name, { - // environment, - // labels, - // logging, - // ports: ['80'], - // }), - // }); - }; + static debug = require('debug')('@lando/l337-service-v4'); + + #appMount = { + type: 'bind', + destination: '/app', + exclude: [], + volumes: [], + binds: [], + } + + #appBuildOpts = { + environment: [], + mounts: [], + } + + constructor(id, options, app, lando) { + // before we call super we need to separate things + const {config, ...upstream} = merge({}, defaults, options); + // @TODO: certs? + // @TODO: better appmount logix? + + // ger user info + const {gid, uid, username} = lando.config; + + // add some upstream stuff and legacy stuff + upstream.appMount = config['app-mount'].destination; + upstream.legacy = merge({}, {meUser: username}, upstream.legacy ?? {}); + + // add a user build group + groups.user = { + description: 'Catch all group for things that should be run as the user', + weight: 2000, + user: username, + }; + + // get this + super(id, {...upstream, groups, states}); + + // helpful + this.project = app.project; + this.router = options.router; + this.isInteractive = lando.config.isInteractive; + + // userstuff + this.gid = gid; + this.uid = uid; + this.username = username; + this.homevol = `${this.project}-${username}-home`; + this.datavol = `${this.project}-${this.id}-data`; + + // build script + // @TODO: handle array content? + this.buildScript = config?.build?.app ?? `true`; + + // set some other stuff + if (config['app-mount']) this.setAppMount(config['app-mount']); + + // auth stuff + this.setSSHAgent(); + this.setNPMRC(lando.config.pluginConfigFile); + + // @NOTE: uh? + this.addSteps({group: 'boot', instructions: ` + RUN rm /bin/sh && ln -s /bin/bash /bin/sh + `}); + + // @NOTE: setup dat user + this.addSteps({group: 'setup-user', instructions: ` + RUN sed -i '/UID_MIN/c\UID_MIN 500' /etc/login.defs + RUN sed -i '/UID_MAX/c\UID_MAX 600100000' /etc/login.defs + RUN sed -i '/GID_MIN/c\GID_MIN 20' /etc/login.defs + RUN sed -i '/GID_MAX/c\GID_MAX 600100000' /etc/login.defs + RUN getent group ${this.gid} > /dev/null || groupadd -g ${this.gid} ${this.username} + RUN useradd -u ${this.uid} -m -g ${this.gid} ${this.username} + RUN usermod -aG sudo ${this.username} + RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + `}); + + // inject global npmrc if we can + + // add a home folder persistent mount + this.addComposeData({volumes: {[this.homevol]: {external: true}}}); + // add the usual DC stuff + this.addServiceData({user: this.username, volumes: [`${this.homevol}:/home/${this.username}`]}); + } + + // buildapp + async buildApp() { + // bail if no script + if (!this.buildScript) this.debug('no build detected, skipping'); + + // get build func + const bengine = LandoServiceV4.getBengine(LandoServiceV4.bengineConfig, { + builder: LandoServiceV4.builder, + debug: this.debug, + orchestrator: LandoServiceV4.orchestrator, + }); + + // generate the build script + const buildScript = require('../utils/generate-build-script')(this.buildScript, this.username, this.appMount); + const buildScriptPath = path.join(this.context, 'app-build.sh'); + fs.writeFileSync(buildScriptPath, buildScript); + + try { + // set state + this.state = {APP: 'BUILDING'}; + + // stuff + const bs = `/home/${this.username}/app-build.sh`; + const command = `chmod +x ${bs} && sh ${bs}`; + + // run with the appropriate builder + const success = await bengine.run([command], { + image: this.tag, + attach: true, + interactive: this.isInteractive, + createOptions: { + User: this.username, + WorkingDir: this.appMount, + Entrypoint: ['/bin/sh', '-c'], + Env: uniq(this.#appBuildOpts.environment), + HostConfig: { + Binds: [ + `${this.homevol}:/home/${this.username}`, + ...uniq(this.#appBuildOpts.mounts), + `${buildScriptPath}:${bs}`, + ], + }, + }, + }); + + // // augment the success info + success.context = {script: fs.readFileSync(buildScriptPath, {encoding: 'utf-8'})}; + // state + this.state = {APP: 'BUILT'}; + // log + this.debug('app %o built successfully from %o', `${this.project}-${this.id}`, buildScriptPath); + return success; + + // failure + } catch (error) { + // augment error + error.id = this.id; + error.context = {script: fs.readFileSync(buildScriptPath, {encoding: 'utf-8'}), path: buildScriptPath}; + this.debug('app %o build failed with code %o error %o', `${this.project}-${this.id}`, error.code, error); + // set the build failure + this.state = {APP: 'BUILD FAILURE'}; + // then throw + throw error; + } + } + + setNPMRC(data) { + // if a string that exists as a path assume its json + if (typeof data === 'string' && fs.existsSync(data)) data = require(data); + + // convert to file contents + const contents = Object.entries(data).map(([key, value]) => `${key}=${value}`); + contents.push(''); + + // write to file + const npmauthfile = path.join(this.context, 'npmrc'); + fs.writeFileSync(npmauthfile, contents.join('\n')); + + // ensure mount + const mounts = [ + `${npmauthfile}:/home/${this.username}/.npmrc`, + `${npmauthfile}:/root/.npmrc`, + ]; + this.addServiceData({volumes: mounts}); + this.#appBuildOpts.mounts.push(...mounts); + this.npmrc = contents.join('\n'); + this.npmrcFile = npmauthfile; + } + + // sets ssh agent and prepares for socating + // DD ssh-agent is a bit strange and we wont use it in v4 plugin but its easiest for demoing purposes + // if you have issues with it its best to do the below + // 0. Close Docker Desktop + // 1. killall ssh-agent + // 2. Start Docker Desktop + // 3. Open a terminal (after Docker Desktop starts) + // 4. ssh-add (use the existing SSH agent, don't start a new one) + // 5. docker run --rm --mount type=bind,src=/run/host-services/ssh-auth.sock,target=/run/host-services/ssh-auth.sock -e SSH_AUTH_SOCK="/run/host-services/ssh-auth.sock" --entrypoint /usr/bin/ssh-add alpine/git -l + setSSHAgent() { + const socket = `/run/host-services/ssh-auth.sock`; + const socater = `/run/ssh-${this.username}.sock`; + + this.addComposeData({services: {[this.id]: { + environment: { + SSH_AUTH_SOCK: socater, + }, + volumes: [ + `${socket}:${socket}`, + ], + }}}); + + this.#appBuildOpts.environment.push(`SSH_AUTH_SOCK=${socater}`); + this.#appBuildOpts.mounts.push(`${socket}:${socket}`); + } + + // @TODO: more powerful syntax eg go as many levels as you want and maybe ! syntax? + setAppMount(config) { + // reset the destination + this.#appMount.destination = config.destination; + + // its easy if we dont have any excludes + if (config.exclude.length === 0) { + this.#appMount.binds = [`${this.appRoot}:${config.destination}`]; + this.#appBuildOpts.mounts.push(`${this.appRoot}:${config.destination}`); + + // if we have excludes then we need to compute somethings + } else { + // named volumes for excludes + this.#appMount.volumes = config.exclude.map(vol => `app-mount-${vol}`); + // get all paths to be considered + const binds = [ + ...fs.readdirSync(this.appRoot).filter(path => !config.exclude.includes(path)), + ...config.exclude, + ]; + // map into bind mounts + this.#appMount.binds = binds.map(path => { + if (config.exclude.includes(path)) return `app-mount-${path}:${this.#appMount.destination}/${path}`; + else return `${this.appRoot}/${path}:${this.#appMount.destination}/${path}`; + }); + // and again for appBuild stuff b w/ full mount name + binds.map(path => { + if (config.exclude.includes(path)) { + this.#appBuildOpts.mounts.push(`${this.project}_app-mount-${path}:${this.#appMount.destination}/${path}`); + } else { + this.#appBuildOpts.mounts.push(`${this.appRoot}/${path}:${this.#appMount.destination}/${path}`); + } + }); + } + + // add named volumes if we need to + if (this.#appMount.volumes.length > 0) { + this.addComposeData({volumes: Object.fromEntries(this.#appMount.volumes.map(vol => ([vol, {}])))}); + } + + // set bindz + this.addServiceData({volumes: this.#appMount.binds}); + + // set infp + this.appMount = config.destination; + this.info.appMount = this.appMount; + } }, }; diff --git a/components/docker-engine.js b/components/docker-engine.js index 9018649b6..d9e341825 100644 --- a/components/docker-engine.js +++ b/components/docker-engine.js @@ -432,18 +432,39 @@ class DockerEngine extends Dockerode { stderro = '', } = {}) { const awaitHandler = async () => { + // stdin helpers + const resizer = container => { + const dimensions = {h: process.stdout.rows, w: process.stderr.columns}; + if (dimensions.h != 0 && dimensions.w != 0) container.resize(dimensions, () => {}); + }; + const closer = (isRaw = process.isRaw) => { + process.stdout.removeListener('resize', resizer); + process.stdin.removeAllListeners(); + process.stdin.setRawMode(isRaw); + process.stdin.resume(); + }; + return new Promise((resolve, reject) => { + let prevkey; + const CTRL_P = '\u0010'; + const CTRL_Q = '\u0011'; + + const aopts = {stream: true, stdout: true, stderr: true, hijack: interactive, stdin: interactive}; + const isRaw = process.isRaw; + const stdout = new PassThrough(); + const stderr = new PassThrough(); + runner.on('container', container => { - runner.on('stream', stream => { - const stdout = new PassThrough(); - const stderr = new PassThrough(); + container.attach(aopts, (error, stream) => { + if (error) runner.emit('error', error); // handle attach dynamics if (attach) { // if tty and just pipe everthing to stdout - if (copts.Tty) stream.pipe(process.stdout); + if (copts.Tty) { + stream.pipe(process.stdout); // otherwise we should be able to pipe both - else { + } else { stdout.pipe(process.stdout); stderr.pipe(process.stderr); } @@ -458,6 +479,18 @@ class DockerEngine extends Dockerode { container.modem.demuxStream(stream, stdout, stderr); } + // handle interactive + if (interactive) { + process.stdin.resume(); + process.stdin.setEncoding('utf8'); + process.stdin.setRawMode(true); + process.stdin.pipe(stream); + process.stdin.on('data', key => { + if (prevkey === CTRL_P && key === CTRL_Q) closer(stream, isRaw); + prevkey = key; + }); + } + // make sure we close child streams when the parent is done stream.on('end', () => { try { @@ -485,32 +518,29 @@ class DockerEngine extends Dockerode { allo += String(buffer); if (!attach) debug.extend('stderr')(String(buffer)); }); - - runner.on('data', data => { - // emit error first - if (data.StatusCode !== 0) runner.emit('error', data); - // fire done no matter what? - runner.emit('done', data); - runner.emit('finished', data); - runner.emit('success', data); - }); }); // handle resolve/reject runner.on('done', data => { - // @TODO: what about data? + closer(stream, isRaw); resolve(makeSuccess(merge({}, data, {command: 'dockerode run', all: allo, stdout: stdouto, stderr: stderro}, {args: command}))); }); runner.on('error', error => { + closer(stream, isRaw); reject(makeError(merge({}, args, {command: 'dockerode run', all: allo, stdout: stdouto, stderr: stderro}, {args: command}, {error}))); }); }); }; // handles the callback to super.run - // we basically need this just to handle dockerode modem errors - const callbackHandler = error => { + const callbackHandler = (error, data) => { + // emit error first if (error) runner.emit('error', error); + if (data.StatusCode !== 0) runner.emit('error', data); + // fire done no matter what? + runner.emit('done', data); + runner.emit('finished', data); + runner.emit('success', data); }; // error if no command @@ -521,10 +551,14 @@ class DockerEngine extends Dockerode { // some good default createOpts const defaultCreateOptions = { AttachStdin: interactive, + AttachStdout: attach, + AttachStderr: attach, HostConfig: {AutoRemove: true}, Tty: false || interactive || attach, OpenStdin: true, + StdinOnce: true, }; + // merge our create options over the defaults const copts = merge({}, defaultCreateOptions, createOptions); // collect some args we can merge into promise resolution diff --git a/components/l337-v4.js b/components/l337-v4.js index 6dfca5392..79a951dc2 100644 --- a/components/l337-v4.js +++ b/components/l337-v4.js @@ -18,7 +18,7 @@ const hasInstructions = require('../utils/has-instructions'); class L337ServiceV4 extends EventEmitter { #data - static debug = require('debug')('l337-service-v4'); + static debug = require('debug')('@lando/l337-service-v4'); static bengineConfig = {}; static builder = require('../utils/get-docker-x')(); static orchestrator = require('../utils/get-compose-x')(); @@ -73,6 +73,10 @@ class L337ServiceV4 extends EventEmitter { return this.#data.states; } + get _data() { + return this.#data; + } + constructor(id, { appRoot = path.join(os.tmpdir(), nanoid(), id), buildArgs = {}, @@ -370,8 +374,9 @@ class L337ServiceV4 extends EventEmitter { volume.source = path.join(this.appRoot, volume.source); } - // if target exists then bind otherwise vol - volume.type = fs.existsSync(volume.source) ? 'bind' : 'volume'; + // we make an "exception" for any /run/host-services things that are in the docker vm + if (volume.source.startsWith('/run/host-services')) volume.type = 'bind'; + else volume.type = fs.existsSync(volume.source) ? 'bind' : 'volume'; // return return volume; diff --git a/hooks/app-add-v4-services.js b/hooks/app-add-v4-services.js index d441eea0d..8aed8e86f 100644 --- a/hooks/app-add-v4-services.js +++ b/hooks/app-add-v4-services.js @@ -24,18 +24,23 @@ module.exports = async (app, lando) => { // instantiate each service _.forEach(app.v4.parsedConfig, config => { - // Throw a warning if service is not supported - if (_.isEmpty(_.find(lando.factory.get(), {api: 4, name: config.type}))) { - app.log.warn('%s is not a supported v4 service type.', config.type); + // Throw a warning if builder is not supported + if (_.isEmpty(_.find(lando.factory.get(), {api: 4, name: config.builder}))) { + app.log.warn('%s is not a supported v4 builder.', config.builder); } + // @TODO: + // if we have routing information lets pass that through + // in v3 "type" was parsed into a builder and a version but in v4 we use a more generic "router" + // concept that lets the "entrypoint builder" + // get any cached info so we can set that as a base in the service const info = _(_.find(app.v4.cachedInfo, {service: config.name, api: 4})) .pick(['healthy', 'image', 'state', 'tag']) .value(); // retrieve the correct class and mimic-ish v4 patterns to ensure faster loads - const Service = lando.factory.get(config.type, config.api); + const Service = lando.factory.get(config.builder, config.api); Service.bengineConfig = lando.config.engineConfig; Service.builder = lando.config.dockerBin; Service.orchestrator = lando.config.orchestratorBin; diff --git a/hooks/app-run-v4-build-steps.js b/hooks/app-run-v4-build-steps.js index 1a81b2d0d..65342084c 100644 --- a/hooks/app-run-v4-build-steps.js +++ b/hooks/app-run-v4-build-steps.js @@ -18,7 +18,7 @@ module.exports = async (app, lando) => { if (_.isEmpty(data)) { lando.cache.remove(app.v4.preLockfile); lando.cache.remove(app.v4.postLockfile); - app.log.debug('removed v4 build locks'); + app.log.debug('removed v4 image build locks'); } }); }); @@ -31,7 +31,7 @@ module.exports = async (app, lando) => { .filter(service => _.includes(buildV4Services, service.id)) .value(); - app.log.debug('going to build v4 services', services.map(service => service.id)); + app.log.debug('going to build v4 images', services.map(service => service.id)); // now build an array of promises with our services const tasks = services.map(service => { @@ -91,6 +91,7 @@ module.exports = async (app, lando) => { const dir = service?.error?.context?.context ?? os.tmpdir(); service.error.logfile = path.join(dir, `error-${nanoid()}.log`); data.image = 'busybox'; + data.user = 'root'; data.command = require('../utils/get-v4-image-build-error-command')(service.error); data.volumes = [`${service.error.logfile}:/tmp/error.log`]; } @@ -109,4 +110,33 @@ module.exports = async (app, lando) => { // and reset the compose cache as well app.v4.updateComposeCache(); }); + + // run app build steps + app.events.on('pre-start', 110, async () => { + // get buildable services + const buildV4Services = _(app.v4.parsedConfig) + .filter(service => _.includes(_.get(app, 'opts.services', app.services), service.name)) + .map(service => service.name) + .value(); + + // filter out any services that dont need to be built + const services = _(app.v4.services) + .filter(service => _.includes(buildV4Services, service.id)) + .filter(service => typeof service.buildApp === 'function') + .filter(service => service?.info?.state?.IMAGE === 'BUILT') + .filter(service => service?.info?.state?.APP !== 'BUILT') + .value(); + + // and then run them in parallel + await Promise.all(services.map(async service => { + try { + await service.buildApp(); + } catch (error) { + app.addMessage(require('../messages/app-build-v4-error')(error), error, true); + } + })); + + // and reset the compose cache as well + app.v4.updateComposeCache(); + }); }; diff --git a/lib/factory.js b/lib/factory.js index 376a2b5e0..7694b9e82 100644 --- a/lib/factory.js +++ b/lib/factory.js @@ -108,7 +108,7 @@ module.exports = class Factory { // if service builder is a function and not a class then we need to pass the parent in until we get resolution if (_.isFunction(service.builder) && !isClass(service.builder)) { - service.builder = service.builder(this.get(service.parent, service.api), service.config); + service.builder = service.builder(this.get(service.parent, service.api), service.defaults ?? service.config); } // if we are here then we *should* have the class we need? diff --git a/messages/app-build-v4-error.js b/messages/app-build-v4-error.js new file mode 100644 index 000000000..282b46f95 --- /dev/null +++ b/messages/app-build-v4-error.js @@ -0,0 +1,15 @@ +'use strict'; + +const {bold} = require('color'); + +// checks to see if a setting is disabled +module.exports = error => ({ + title: `Could not build app in "${error.id}!"`, + type: 'warn', + detail: [ + `App build steps failed in ${bold(error.context.path)}`, + `Rerun with ${bold('lando rebuild --debug')} to see the entire build log and look for errors.`, + `When you've resolved the build issues you can then:`, + ], + command: 'lando rebuild', +}); diff --git a/utils/generate-build-script.js b/utils/generate-build-script.js new file mode 100644 index 000000000..e57b346ef --- /dev/null +++ b/utils/generate-build-script.js @@ -0,0 +1,14 @@ +/* eslint-disable max-len */ +'use strict'; + +module.exports = (contents, user, mount = '/app') => ` +#!/bin/sh +# clean up and setup ssh-auth +rm -rf /run/ssh-${user}.sock +sudo socat UNIX-LISTEN:/run/ssh-${user}.sock,fork,user=pirog,group=20,mode=777 UNIX-CONNECT:/run/host-services/ssh-auth.sock & + +# temp stuff for demo purposes +git config --global --add safe.directory ${mount} + +${contents} +`; diff --git a/utils/parse-v4-services.js b/utils/parse-v4-services.js index 6bc571c54..9ad3c45fa 100644 --- a/utils/parse-v4-services.js +++ b/utils/parse-v4-services.js @@ -8,6 +8,7 @@ module.exports = services => _(services) .map((service, name) => _.merge({}, { name, api: require('./get-service-api-version')(service.api), + builder: service.type.split(':')[0] ?? 'lando', config: _.omit(service, ['api', 'meUser', 'moreHttpPorts', 'primary', 'scanner', 'sport', 'type']), legacy: { meUser: service.meUser ?? 'www-data', @@ -15,7 +16,8 @@ module.exports = services => _(services) sport: service.sport ?? '443', }, primary: service.primary ?? false, + router: service.type.split(':')[1], scanner: service.scanner ?? false, - type: service.type ?? 'lando', + type: service.type, })) .value();